From c8a22f1090d2d55baaa6aec49d2fc52ab00f252c Mon Sep 17 00:00:00 2001
From: Doug Szumski <doug@stackhpc.com>
Date: Thu, 7 Mar 2019 10:01:30 +0000
Subject: [PATCH] Support customising Fluentd formatting

In some scenarios it may be useful to perform custom formatting of logs
before forwarding them. For example, the JSON formatter plugin can be
used to convert an event to JSON.

Change-Id: I3dd9240c5910a9477456283b392edc9566882dcd
---
 ansible/roles/common/handlers/main.yml        |  1 +
 ansible/roles/common/tasks/config.yml         | 21 +++++++++++++++++++
 .../roles/common/templates/fluentd.json.j2    | 12 ++++-------
 .../central-logging-guide.rst                 | 11 ++++++++++
 ...custom-format-config-b6fc3a4883efc213.yaml | 10 +++++++++
 5 files changed, 47 insertions(+), 8 deletions(-)
 create mode 100644 releasenotes/notes/fluentd-custom-format-config-b6fc3a4883efc213.yaml

diff --git a/ansible/roles/common/handlers/main.yml b/ansible/roles/common/handlers/main.yml
index 0887c18b92..e279b079e4 100644
--- a/ansible/roles/common/handlers/main.yml
+++ b/ansible/roles/common/handlers/main.yml
@@ -23,6 +23,7 @@
       or fluentd_output.changed | bool
       or fluentd_output_custom.changed | bool
       or fluentd_format.changed | bool
+      or fluentd_format_custom.changed | bool
       or fluentd_filter.changed | bool
       or fluentd_filter_custom.changed | bool
       or fluentd_td_agent.changed | bool
diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml
index 5a8eb174a7..aa7676ae80 100644
--- a/ansible/roles/common/tasks/config.yml
+++ b/ansible/roles/common/tasks/config.yml
@@ -158,6 +158,27 @@
   notify:
     - Restart fluentd container
 
+- name: Find custom fluentd format config files
+  local_action:
+    module: find
+    path: "{{ node_custom_config }}/fluentd/format"
+    pattern: "*.conf"
+  run_once: True
+  register: find_custom_fluentd_format
+  when:
+    - enable_fluentd | bool
+
+- name: Copying over custom fluentd format config files
+  template:
+    src: "{{ item.path }}"
+    dest: "{{ node_config_directory }}/fluentd/format/{{ item.path | basename }}"
+  register: fluentd_format_custom
+  when:
+    - enable_fluentd | bool
+  with_items: "{{ find_custom_fluentd_format.files }}"
+  notify:
+    - Restart fluentd container
+
 - name: Copying over fluentd filter config files
   template:
     src: "conf/filter/{{ item.src }}.conf.j2"
diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2
index 3b4580c069..8d6b6cba28 100644
--- a/ansible/roles/common/templates/fluentd.json.j2
+++ b/ansible/roles/common/templates/fluentd.json.j2
@@ -27,15 +27,11 @@
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
         },
+        {# Copy all configuration files in format/ directory to include #}
+        {# custom format configs. #}
         {
-            "source": "{{ container_config_directory }}/format/apache_access.conf",
-            "dest": "{{ fluentd_dir }}/format/apache_access.conf",
-            "owner": "{{ fluentd_user }}",
-            "perm": "0600"
-        },
-        {
-            "source": "{{ container_config_directory }}/format/wsgi_access.conf",
-            "dest": "{{ fluentd_dir }}/format/wsgi_access.conf",
+            "source": "{{ container_config_directory }}/format/*.conf",
+            "dest": "{{ fluentd_dir }}/format/",
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
         },
diff --git a/doc/source/reference/logging-and-monitoring/central-logging-guide.rst b/doc/source/reference/logging-and-monitoring/central-logging-guide.rst
index 91c60852a3..e5058514a4 100644
--- a/doc/source/reference/logging-and-monitoring/central-logging-guide.rst
+++ b/doc/source/reference/logging-and-monitoring/central-logging-guide.rst
@@ -182,6 +182,17 @@ Configuration of custom fluentd filters is possible by placing filter
 configuration files in ``/etc/kolla/config/fluentd/filter/*.conf`` on the
 control host.
 
+Custom log formatting
+---------------------
+
+In some scenarios it may be useful to perform custom formatting of logs before
+forwarding them. For example, the JSON formatter plugin can be used to convert
+an event to JSON.
+
+Configuration of custom fluentd formatting is possible by placing filter
+configuration files in ``/etc/kolla/config/fluentd/format/*.conf`` on the
+control host.
+
 Custom log forwarding
 ---------------------
 
diff --git a/releasenotes/notes/fluentd-custom-format-config-b6fc3a4883efc213.yaml b/releasenotes/notes/fluentd-custom-format-config-b6fc3a4883efc213.yaml
new file mode 100644
index 0000000000..9cfd1daee3
--- /dev/null
+++ b/releasenotes/notes/fluentd-custom-format-config-b6fc3a4883efc213.yaml
@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Adds ability to configure custom fluentd formatting.
+
+    In some scenarios it may be useful to configure custom fluentd formatting
+    to, for example, convert events to JSON.
+
+    Configuration of custom fluentd formatting is possible by placing
+    output configuration files in /etc/kolla/config/fluentd/format/\*.conf.