Merge "Update Elasticsearch health status expressions"

grafana/values.yaml

@@ -5805,14 +5805,14 @@ conf:
             show: true
           tableColumn: ''
           targets:
-          - expr: sum(elasticsearch_cluster_health_status{cluster=~"$cluster"})
+          - expr: (sum(elasticsearch_cluster_health_status{cluster=~"$cluster",color="green"})*2)+sum(elasticsearch_cluster_health_status{cluster=~"$cluster",color="yellow"})
             format: time_series
-            intervalFactor: 2
+            intervalFactor: 3
             legendFormat: ''
             metric: ''
             refId: A
             step: 40
-          thresholds: '0,1'
+          thresholds: '0,1,2'
           title: Cluster health status
           transparent: false
           type: singlestat
@@ -5820,6 +5820,9 @@ conf:
           valueMaps:
           - op: "="
             text: GREEN
+            value: '2'
+          - op: "="
+            text: YELLOW
             value: '1'
           - op: "="
             text: RED

nagios/values.yaml

@@ -851,7 +851,7 @@ conf:
       - check_es_cluster_health_status:
           use: generic-service
           service_description: ES_cluster-health-status
-          check_command: check_prom_alert!es_cluster_health_status_alert!CRITICAL- Elasticsearch Cluster is not green. One or more shards or replicas are unallocated!OK- Elasticsearch cluster health is green.
+          check_command: check_prom_alert!es_cluster_health_status_alert!CRITICAL- Elasticsearch cluster health status is not green. One or more shards or replicas are unallocated!OK- Elasticsearch cluster health is green.
           hostgroup_name: prometheus-hosts
       - check_es_cluster_number_nodes_running:
           use: generic-service

prometheus/values.yaml

@@ -2038,12 +2038,12 @@ conf:
               description: 'Elasticsearch cluster health status call timedout {{ $value }} times.'
               summary: 'Elasticsearch cluster health status calls are timing out.'
           - alert: es_cluster_health_status_alert
-            expr: elasticsearch_cluster_health_status > 0
+            expr: (sum(elasticsearch_cluster_health_status{color="green"})*2)+sum(elasticsearch_cluster_health_status{color="yellow"}) < 2
             for: 10m
             labels:
               severity: warning
             annotations:
-              description: 'Elasticsearch cluster health status is not green. One or more shards or replicas are unallocated.'
+              description: 'Elasticsearch cluster health status is {{ $value }}, not 2 (green). One or more shards or replicas are unallocated.'
               summary: 'Elasticsearch cluster health status is not green.'
           - alert: es_cluster_health_too_few_nodes_running
             expr: elasticsearch_cluster_health_number_of_nodes < 3
@@ -2061,6 +2061,14 @@ conf:
             annotations:
               description: 'There are only {{$value}} < 3 ElasticSearch data nodes running'
               summary: 'ElasticSearch running on less than 3 data nodes'
+          - alert: es_cluster_health_too_few_data_nodes_running
+            expr: elasticsearch_cluster_health_number_of_data_nodes < 3
+            for: 10m
+            labels:
+              severity: warning
+            annotations:
+              description: 'There are only {{$value}} < 3 ElasticSearch data nodes running'
+              summary: 'ElasticSearch running on less than 3 data nodes'
       mariadb:
         groups:
         - name: mariadb.rules