Add a cluster health indicator to the graph output

This uses the ES health API to get the cluster health status
and pretty it up in our index/gate graph pages.

The cluster health API is documented here:

https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html

Also add a note to the readme on how to view the openstack
ES cluster health.

Change-Id: I3df833cf5024af7282e2602e7dfa5db9c3384b6a

README.rst

@@ -199,9 +199,10 @@ You can execute an individual query locally and analyze the search results::
 Notes
 -----
 
-The html generation will generate links that work with Kibana3's logstash.json
+* The html generation will generate links that work with Kibana3's
-dashboard. If you want the links to work properly on these generated files
+  logstash.json dashboard. If you want the links to work properly on these
-you will need to host a Kibana3 with that dashboard.
+  generated files you will need to host a Kibana3 with that dashboard.
+* View the OpenStack ElasticSearch `cluster health here`_.
 
 Future Work
 -----------
@@ -215,3 +216,6 @@ Future Work
 - Move away from polling ElasticSearch to discover if its ready or not
 - Add nightly job to propose a patch to remove bug queries that return
   no hits -- Bug hasn't been seen in 2 weeks and must be closed
+
+
+.. _cluster health here: http://logstash.openstack.org/elasticsearch/_cluster/health?pretty=true

elastic_recheck/cmd/graph.py

@@ -159,6 +159,10 @@ def main():
         'buglist': []
     }
 
+    # Get the cluster health for the header
+    es = pyelasticsearch.ElasticSearch(es_url)
+    jsondata['status'] = es.health()['status']
+
     for query in classifier.queries:
         if args.queue:
             query['query'] = query['query'] + (' AND build_queue:%s' %

web/share/elastic-recheck.js

@@ -55,6 +55,20 @@ function update_critical_dates(data) {
     }
 }
 
+function update_health(data) {
+    var health = $('#health');
+    health.text(data['status']);
+    $('#health').text(data['status']);
+    if (data['status'] == 'red') {
+        // TODO(mriedem): link to the cluster health details
+        // http://logstash.openstack.org/elasticsearch/_cluster/health?pretty=true
+        health.css('font-weight', 'bold');
+    } else {
+        health.css('font-weight', 'normal');
+    }
+    health.css('color', data['status']);
+}
+
 function update() {
     var source   = $("#bug-template").html();
     var template = Handlebars.compile(source);
@@ -64,6 +78,7 @@ function update() {
         if ('buglist' in data) {
             buglist = data['buglist'];
             update_critical_dates(data);
+            update_health(data);
         }
 
         var main = $('#main-container');

web/share/gate.html

@@ -93,6 +93,7 @@
       Data Last Updated: <span id="last_updated"></span><br>
       Last Elastic Search Index Update: <span id="last_indexed"></span><br>
       Delay in Elastic Search: <span id="behind"></span><br>
+      Cluster Health: <span id="health"></span><br>
     </div>
 
     <div id="main-container" class="container">

web/share/index.html

@@ -90,6 +90,7 @@
       Data Last Updated: <span id="last_updated"></span><br>
       Last Elastic Search Index Update: <span id="last_indexed"></span><br>
       Delay in Elastic Search: <span id="behind"></span><br>
+      Cluster Health: <span id="health"></span><br>
     </div>
 
     <div id="main-container" class="container">