From 181d7ebb3454c6ec82c129882db3d54ad0aac3dd Mon Sep 17 00:00:00 2001
From: Steve Wilkerson <wilkers.steve@gmail.com>
Date: Thu, 10 Jan 2019 13:09:58 -0600
Subject: [PATCH] Fluentd: Update buffer output settings for Elasticsearch

This updates the fluentd configuration to use 8 threads for the
Elasticsearch output configuration by default. This uses the
correct buffer output settings for the fluent-elasticsearch
plugin

This also updates the buffer output settings to the defaults used
for fluentd

Change-Id: I976cddaa973e850dabe4de495cd3bf1a4acdd4e7
---
 fluent-logging/values.yaml | 75 +++++++++++++++++++++++---------------
 1 file changed, 46 insertions(+), 29 deletions(-)

diff --git a/fluent-logging/values.yaml b/fluent-logging/values.yaml
index aa808e381..d70a9287c 100644
--- a/fluent-logging/values.yaml
+++ b/fluent-logging/values.yaml
@@ -359,11 +359,15 @@ conf:
         port: "#{ENV['ELASTICSEARCH_PORT']}"
         logstash_format: true
         logstash_prefix: libvirt
-        buffer_chunk_limit: 2M
-        buffer_queue_limit: 8
-        flush_interval: "5"
-        max_retry_wait: 300
-        disable_retry_limit: ""
+        buffer:
+          -
+            - header: buffer
+              flush_thread_count: 8
+              flush_interval: "15"
+              chunk_limit_size: 8MB
+              queue_limit_length: 256
+              retry_max_interval: 30
+              retry_forever: false
     - qemu_elasticsearch:
         header: match
         type: elasticsearch
@@ -375,11 +379,15 @@ conf:
         port: "#{ENV['ELASTICSEARCH_PORT']}"
         logstash_format: true
         logstash_prefix: qemu
-        buffer_chunk_limit: 2M
-        buffer_queue_limit: 8
-        flush_interval: "5"
-        max_retry_wait: 300
-        disable_retry_limit: ""
+        buffer:
+          -
+            - header: buffer
+              flush_thread_count: 8
+              flush_interval: "15"
+              chunk_limit_size: 8MB
+              queue_limit_length: 256
+              retry_max_interval: 30
+              retry_forever: false
     - journal_elasticsearch:
         header: match
         type: elasticsearch
@@ -391,11 +399,15 @@ conf:
         port: "#{ENV['ELASTICSEARCH_PORT']}"
         logstash_format: true
         logstash_prefix: journal
-        buffer_chunk_limit: 2M
-        buffer_queue_limit: 8
-        flush_interval: "5"
-        max_retry_wait: 300
-        disable_retry_limit: ""
+        buffer:
+          -
+            - header: buffer
+              flush_thread_count: 8
+              flush_interval: "15"
+              chunk_limit_size: 8MB
+              queue_limit_length: 256
+              retry_max_interval: 30
+              retry_forever: false
     - kernel_elasticsearch:
         header: match
         type: elasticsearch
@@ -407,14 +419,15 @@ conf:
         port: "#{ENV['ELASTICSEARCH_PORT']}"
         logstash_format: true
         logstash_prefix: kernel
-        buffer_chunk_limit: 2M
-        buffer_queue_limit: 8
-        flush_interval: "5"
-        max_retry_wait: 300
-        disable_retry_limit: ""
-    # NOTE(srwilkers): This configuration entry should always be the last output
-    # defined, as it is used to determine the total flush cycle time for fluentbit
-    # and fluentd
+        buffer:
+          -
+            - header: buffer
+              flush_thread_count: 8
+              flush_interval: "15"
+              chunk_limit_size: 8MB
+              queue_limit_length: 256
+              retry_max_interval: 30
+              retry_forever: false
     - elasticsearch:
         header: match
         type: elasticsearch
@@ -425,12 +438,16 @@ conf:
         host: "#{ENV['ELASTICSEARCH_HOST']}"
         port: "#{ENV['ELASTICSEARCH_PORT']}"
         logstash_format: true
-        buffer_chunk_limit: 2M
-        buffer_queue_limit: 8
-        flush_interval: "5"
-        max_retry_wait: 300
-        disable_retry_limit: ""
-        type_name: fluent
+        buffer:
+          -
+            - header: buffer
+              flush_thread_count: 8
+              flush_interval: "15"
+              chunk_limit_size: 8MB
+              queue_limit_length: 256
+              retry_max_interval: 30
+              retry_forever: false
+        flush_interval: "15"
   fluentd_exporter:
     log:
       format: "logger:stdout?json=true"