add support for computing relevant dates

This records the current time when the data is constructed, the date
of the last valid looking piece of data in elastic search, and how far
behind we seem to be on indexing. The json payload is adjusted to be
able to take additional metadata to support displaying this on the ER
page.

Change-Id: I0068ca0bbe72943d5d92dea704659ed865fea198

elastic_recheck/cmd/graph.py

@@ -21,6 +21,7 @@ import json
 import os
 
 from launchpadlib import launchpad
+import pytz
 import requests
 
 import elastic_recheck.elasticRecheck as er
@@ -81,9 +82,13 @@ def main():
 
     buglist = []
 
-    epoch = datetime.utcfromtimestamp(0)
-    ts = datetime.now()
-    ts = datetime(ts.year, ts.month, ts.day, ts.hour)
+    # if you don't hate timezones, you don't program enough
+    epoch = datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc)
+    ts = datetime.utcnow().replace(tzinfo=pytz.utc)
+    # rawnow is useful for sending to javascript
+    rawnow = int(((ts - epoch).total_seconds()) * 1000)
+
+    ts = datetime(ts.year, ts.month, ts.day, ts.hour).replace(tzinfo=pytz.utc)
     # ms since epoch
     now = int(((ts - epoch).total_seconds()) * 1000)
     # number of days to match to, this should be the same as we are
@@ -94,6 +99,18 @@ def main():
     # ER timeframe for search
     timeframe = days * 24 * STEP / 1000
 
+    last_indexed = int(
+        ((classifier.most_recent() - epoch).total_seconds()) * 1000)
+    behind = now - last_indexed
+
+    # the data we're going to return, including interesting headers
+    jsondata = {
+        'now': rawnow,
+        'last_indexed': last_indexed,
+        'behind': behind,
+        'buglist': []
+    }
+
     for query in classifier.queries:
         if args.queue:
             query['query'] = query['query'] + (' AND build_queue:"%s"' %
@@ -151,8 +168,9 @@ def main():
     buglist = sorted(buglist,
                      key=lambda bug: -(bug['fails24'] * 100000 + bug['fails']))
 
+    jsondata['buglist'] = buglist
     out = open(args.output, 'w')
-    out.write(json.dumps(buglist))
+    out.write(json.dumps(jsondata))
     out.close()
 
 

elastic_recheck/elasticRecheck.py

@@ -12,7 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-
+import dateutil.parser as dp
 import gerritlib.gerrit
 import pyelasticsearch
 
@@ -368,6 +368,15 @@ class Classifier():
             es_query = qb.generic(query, facet=facet)
         return self.es.search(es_query, size=size)
 
+    def most_recent(self):
+        """Return the datetime of the most recently indexed event."""
+        query = qb.most_recent_event()
+        results = self.es.search(query, size='1')
+        if len(results) > 0:
+            last = dp.parse(results[0].timestamp)
+            return last
+        return datetime.datetime.utcfromtimestamp(0)
+
     def classify(self, change_number, patch_number,
                  build_short_uuid, recent=False):
         """Returns either empty list or list with matched bugs."""

elastic_recheck/query_builder.py

@@ -110,3 +110,11 @@ def single_patch(query, review, patch, build_short_uuid):
                    'AND build_patchset:"%s"'
                    'AND build_short_uuid:%s' %
                    (query, review, patch, build_short_uuid))
+
+
+def most_recent_event():
+    return generic(
+        'filename:console.html '
+        'AND (build_queue:gate OR build_queue:check) '
+        'AND NOT tags:_grokparsefailure '
+        'AND NOT message:"%{logmessage}" ')