Support customising Fluentd formatting

In some scenarios it may be useful to perform custom formatting of logs
before forwarding them. For example, the JSON formatter plugin can be
used to convert an event to JSON.

Change-Id: I3dd9240c5910a9477456283b392edc9566882dcd

ansible/roles/common/handlers/main.yml

@@ -23,6 +23,7 @@
       or fluentd_output.changed | bool
       or fluentd_output_custom.changed | bool
       or fluentd_format.changed | bool
+      or fluentd_format_custom.changed | bool
       or fluentd_filter.changed | bool
       or fluentd_filter_custom.changed | bool
       or fluentd_td_agent.changed | bool

ansible/roles/common/tasks/config.yml

@@ -158,6 +158,27 @@
   notify:
     - Restart fluentd container
 
+- name: Find custom fluentd format config files
+  local_action:
+    module: find
+    path: "{{ node_custom_config }}/fluentd/format"
+    pattern: "*.conf"
+  run_once: True
+  register: find_custom_fluentd_format
+  when:
+    - enable_fluentd | bool
+
+- name: Copying over custom fluentd format config files
+  template:
+    src: "{{ item.path }}"
+    dest: "{{ node_config_directory }}/fluentd/format/{{ item.path | basename }}"
+  register: fluentd_format_custom
+  when:
+    - enable_fluentd | bool
+  with_items: "{{ find_custom_fluentd_format.files }}"
+  notify:
+    - Restart fluentd container
+
 - name: Copying over fluentd filter config files
   template:
     src: "conf/filter/{{ item.src }}.conf.j2"

ansible/roles/common/templates/fluentd.json.j2

@@ -27,15 +27,11 @@
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
         },
+        {# Copy all configuration files in format/ directory to include #}
+        {# custom format configs. #}
         {
-            "source": "{{ container_config_directory }}/format/apache_access.conf",
-            "dest": "{{ fluentd_dir }}/format/apache_access.conf",
-            "owner": "{{ fluentd_user }}",
-            "perm": "0600"
-        },
-        {
-            "source": "{{ container_config_directory }}/format/wsgi_access.conf",
-            "dest": "{{ fluentd_dir }}/format/wsgi_access.conf",
+            "source": "{{ container_config_directory }}/format/*.conf",
+            "dest": "{{ fluentd_dir }}/format/",
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
         },

doc/source/reference/logging-and-monitoring/central-logging-guide.rst

@@ -182,6 +182,17 @@ Configuration of custom fluentd filters is possible by placing filter
 configuration files in ``/etc/kolla/config/fluentd/filter/*.conf`` on the
 control host.
 
+Custom log formatting
+---------------------
+
+In some scenarios it may be useful to perform custom formatting of logs before
+forwarding them. For example, the JSON formatter plugin can be used to convert
+an event to JSON.
+
+Configuration of custom fluentd formatting is possible by placing filter
+configuration files in ``/etc/kolla/config/fluentd/format/*.conf`` on the
+control host.
+
 Custom log forwarding
 ---------------------
 

releasenotes/notes/fluentd-custom-format-config-b6fc3a4883efc213.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Adds ability to configure custom fluentd formatting.
+
+    In some scenarios it may be useful to configure custom fluentd formatting
+    to, for example, convert events to JSON.
+
+    Configuration of custom fluentd formatting is possible by placing
+    output configuration files in /etc/kolla/config/fluentd/format/\*.conf.