From 4c29bafcbc8adf5a673b8951c4674999535b6b8d Mon Sep 17 00:00:00 2001
From: Steve Wilkerson <wilkers.steve@gmail.com>
Date: Fri, 19 Oct 2018 07:57:35 -0500
Subject: [PATCH] Gates: Update fluent-logging/elasticsearch configurations

This updates the fluentd buffer output configurations to account
for the restraints of the jobs deploying fluentd. This also
renames the fluentd configuration key from td_agent to fluentd to
reflect the fact we're no longer deploying td-agent

This also updates the Elasticsearch default replicas and overrides
the replica counts in each Elasticsearch deployment to account for
resource constraints

Change-Id: I55dee410eced99c3e1645f7452e4306ad646e601
---
 elasticsearch/values.yaml                               | 2 +-
 fluent-logging/templates/bin/_helm-tests.sh.tpl         | 2 +-
 fluent-logging/templates/configmap-etc.yaml             | 2 +-
 fluent-logging/values.yaml                              | 8 +-------
 tools/deployment/multinode/120-elasticsearch.sh         | 4 ++++
 tools/deployment/network-policy/120-elasticsearch.sh    | 1 +
 tools/deployment/osh-infra-logging/050-elasticsearch.sh | 1 +
 7 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml
index 7939fd953..6a36e6adc 100644
--- a/elasticsearch/values.yaml
+++ b/elasticsearch/values.yaml
@@ -106,7 +106,7 @@ pod:
           default: kubernetes.io/hostname
   replicas:
     master: 3
-    data: 2
+    data: 3
     client: 3
   lifecycle:
     upgrades:
diff --git a/fluent-logging/templates/bin/_helm-tests.sh.tpl b/fluent-logging/templates/bin/_helm-tests.sh.tpl
index e30b9ae8f..12bbf3786 100644
--- a/fluent-logging/templates/bin/_helm-tests.sh.tpl
+++ b/fluent-logging/templates/bin/_helm-tests.sh.tpl
@@ -34,7 +34,7 @@ function check_output_indexes_exist () {
     Elasticsearch outputs should match. This means we can safely use the last item
     in fluentd's configuration to get the Fluentd flush output interval
   */}}
-  {{- $fluentdConf := last .Values.conf.td_agent -}}
+  {{- $fluentdConf := last .Values.conf.fluentd -}}
   {{- $fluentdElasticsearchConf := index $fluentdConf "elasticsearch" -}}
   {{- $fluentdFlush := index $fluentdElasticsearchConf "flush_interval" -}}
   fluentdFlush={{$fluentdFlush}}
diff --git a/fluent-logging/templates/configmap-etc.yaml b/fluent-logging/templates/configmap-etc.yaml
index 723526ecb..80c3f964d 100644
--- a/fluent-logging/templates/configmap-etc.yaml
+++ b/fluent-logging/templates/configmap-etc.yaml
@@ -25,7 +25,7 @@ type: Opaque
 data:
   fluent-bit.conf: {{ include "fluent_logging.utils.to_fluentbit_conf" .Values.conf.fluentbit | b64enc }}
   parsers.conf: {{ include "fluent_logging.utils.to_fluentbit_conf" .Values.conf.parsers | b64enc }}
-  fluent.conf: {{ include "fluent_logging.utils.to_fluentd_conf" .Values.conf.td_agent | b64enc }}
+  fluent.conf: {{ include "fluent_logging.utils.to_fluentd_conf" .Values.conf.fluentd | b64enc }}
 {{ range $template, $fields := .Values.conf.templates }}
   {{ $template }}.json: {{ toJson $fields | b64enc }}
 {{ end }}
diff --git a/fluent-logging/values.yaml b/fluent-logging/values.yaml
index ae30b78cb..3124f6a70 100644
--- a/fluent-logging/values.yaml
+++ b/fluent-logging/values.yaml
@@ -202,7 +202,7 @@ conf:
         Time_Key: time
         Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
         Time_Keep: On
-  td_agent:
+  fluentd:
     - metrics_agent:
         header: source
         type: monitor_agent
@@ -304,10 +304,8 @@ conf:
         buffer_chunk_limit: 2M
         buffer_queue_limit: 8
         flush_interval: "5"
-        flush_thread_count: 2
         max_retry_wait: 300
         disable_retry_limit: ""
-        num_threads: 8
     - kernel_elasticsearch:
         header: match
         type: elasticsearch
@@ -322,10 +320,8 @@ conf:
         buffer_chunk_limit: 2M
         buffer_queue_limit: 8
         flush_interval: "5"
-        flush_thread_count: 2
         max_retry_wait: 300
         disable_retry_limit: ""
-        num_threads: 8
     # NOTE(srwilkers): This configuration entry should always be the last output
     # defined, as it is used to determine the total flush cycle time for fluentbit
     # and fluentd
@@ -342,10 +338,8 @@ conf:
         buffer_chunk_limit: 2M
         buffer_queue_limit: 8
         flush_interval: "5"
-        flush_thread_count: 2
         max_retry_wait: 300
         disable_retry_limit: ""
-        num_threads: 8
         type_name: fluent
   fluentd_exporter:
     log:
diff --git a/tools/deployment/multinode/120-elasticsearch.sh b/tools/deployment/multinode/120-elasticsearch.sh
index c35224ff5..a0ae6d18a 100755
--- a/tools/deployment/multinode/120-elasticsearch.sh
+++ b/tools/deployment/multinode/120-elasticsearch.sh
@@ -21,6 +21,10 @@ make elasticsearch
 
 #NOTE: Deploy command
 tee /tmp/elasticsearch.yaml << EOF
+pod:
+  replicas:
+    data: 1
+    master: 2
 conf:
   elasticsearch:
     env:
diff --git a/tools/deployment/network-policy/120-elasticsearch.sh b/tools/deployment/network-policy/120-elasticsearch.sh
index 9161d9496..02a408a04 100755
--- a/tools/deployment/network-policy/120-elasticsearch.sh
+++ b/tools/deployment/network-policy/120-elasticsearch.sh
@@ -27,6 +27,7 @@ monitoring:
 pod:
   replicas:
     data: 1
+    master: 2
 manifests:
   network_policy: true
 network_policy:
diff --git a/tools/deployment/osh-infra-logging/050-elasticsearch.sh b/tools/deployment/osh-infra-logging/050-elasticsearch.sh
index 8ec6c1d23..3f6da7010 100755
--- a/tools/deployment/osh-infra-logging/050-elasticsearch.sh
+++ b/tools/deployment/osh-infra-logging/050-elasticsearch.sh
@@ -27,6 +27,7 @@ monitoring:
 pod:
   replicas:
     data: 1
+    master: 2
 conf:
   elasticsearch:
     snapshots: