Get rid of auto_expand_replicas for Kibana indices

Setting auto_expand_replicas to "0-all" can make the Kibana indices
unavailable when scaling up the Elasticsearch cluster. As a consequence,
the Kibana service is unavailable and the operator needs to manually fix
the problem. This change applies the same replication settings as for
the log and notification indices leading to more predictable behavior.
It also enforces the replication settings in the provision_services.pp
manifest to deal with scale-down and scale-up operations.

Change-Id: I8979f3d006ccd908711cbe0862032dc7b73d9b62
Closes-Bug: #1552258

deployment_scripts/puppet/manifests/kibana_index_configuration.pp

@@ -17,6 +17,7 @@ notice('fuel-plugin-elasticsearch-kibana: kibana_index_configuration.pp')
 $vip = hiera('lma::elasticsearch::vip')
 $es_port = hiera('lma::elasticsearch::rest_port')
 $kibana_index = hiera('lma::elasticsearch::kibana_index')
+$number_of_replicas = hiera('lma::elasticsearch::number_of_replicas')
 
 # Elasticsearch must be reachable through HAproxy before the template creation.
 # This is due the fact that The Elasticsearch Puppet module miserably fails
@@ -34,10 +35,11 @@ haproxy_backend_status { 'elasticsearch':
 # Then, correct fields mapping are present before importing objects
 # (searches, visualizations and dashboards).
 lma_logging_analytics::es_template { 'kibana4':
-  host           => $vip,
-  port           => $es_port,
-  index_template => $kibana_index,
-  require        => Haproxy_backend_status['elasticsearch'],
+  number_of_replicas => $number_of_replicas,
+  host               => $vip,
+  port               => $es_port,
+  index_template     => $kibana_index,
+  require            => Haproxy_backend_status['elasticsearch'],
 }
 
 # Import all Kibana objects in one time by issuing a Bulk request

deployment_scripts/puppet/manifests/provision_services.pp

@@ -21,6 +21,7 @@ $kibana_vip = hiera('lma::kibana::vip')
 $kibana_viewer_port = hiera('lma::elasticsearch::kibana_frontend_viewer_port')
 $es_port = hiera('lma::elasticsearch::rest_port')
 $number_of_replicas = hiera('lma::elasticsearch::number_of_replicas')
+$kibana_index = hiera('lma::elasticsearch::kibana_index')
 
 $authnz = hiera_hash('lma::kibana::authnz')
 if $authnz['ldap_enabled'] and $authnz['ldap_authorization_enabled'] {
@@ -66,6 +67,12 @@ lma_logging_analytics::es_template { ['log', 'notification']:
   port               => $es_port,
 }
 
+# Adjust the number of replicas for the Kibana index
+exec { 'adjust_kibana_replicas':
+  command => "/usr/bin/curl -sL -XPUT http://${es_vip}:${es_port}/${kibana_index}/_settings \
+  -d '{\"index\": {\"number_of_replicas\": ${number_of_replicas}}}'"
+}
+
 $kibana_link_created_file = '/var/cache/kibana_link_created'
 exec { 'notify_kibana_url':
   creates => $kibana_link_created_file,

deployment_scripts/puppet/modules/lma_logging_analytics/templates/es_template_kibana4.json.erb

@@ -132,8 +132,7 @@
     },
     "settings": {
         "number_of_shards": <%= @number_of_shards %>,
-        "auto_expand_replicas": "0-all"
+        "number_of_replicas": <%= @number_of_replicas %>
     },
     "template": "<%= @template %>"
 }
-