From f12e72440a687142116d8e5b82a2d5935e9bf8fb Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 17 Oct 2017 16:09:14 +0100
Subject: [PATCH] Support configuration of custom fluentd filters

In some scenarios it may be useful to apply custom filters to logs
before forwarding them.  This may be useful to add additional tags to
the messages or to modify the tags to conform to a log format that
differs from the one defined by kolla-ansible.

Configuration of custom fluentd filters is possible by placing filter
configuration files in ``/etc/kolla/config/fluentd/filter/*.conf`` on
the ansible control host.

Change-Id: I29a20efb0df4bf3564ef009616c786e928aa26d9
Implements: blueprint fluentd-custom-filters
---
 ansible/roles/common/handlers/main.yml          |  1 +
 ansible/roles/common/tasks/config.yml           | 17 +++++++++++++++++
 ansible/roles/common/templates/fluentd.json.j2  | 12 ++++--------
 doc/source/reference/central-logging-guide.rst  | 12 ++++++++++++
 ...d-custom-filter-config-26ee8801c7782b08.yaml | 13 +++++++++++++
 5 files changed, 47 insertions(+), 8 deletions(-)
 create mode 100644 releasenotes/notes/fluentd-custom-filter-config-26ee8801c7782b08.yaml

diff --git a/ansible/roles/common/handlers/main.yml b/ansible/roles/common/handlers/main.yml
index c580ab5246..f1c155d886 100644
--- a/ansible/roles/common/handlers/main.yml
+++ b/ansible/roles/common/handlers/main.yml
@@ -21,6 +21,7 @@
       or fluentd_output_custom.changed | bool
       or fluentd_format.changed | bool
       or fluentd_filter.changed | bool
+      or fluentd_filter_custom.changed | bool
       or fluentd_td_agent.changed | bool
       or fluentd_container.changed | bool
 
diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml
index fe39f8e46f..d3b0e2bd61 100644
--- a/ansible/roles/common/tasks/config.yml
+++ b/ansible/roles/common/tasks/config.yml
@@ -117,6 +117,23 @@
   notify:
     - Restart fluentd container
 
+- name: Find custom fluentd filter config files
+  local_action:
+    module: find
+    path: "{{ node_custom_config }}/fluentd/filter"
+    pattern: "*.conf"
+  run_once: True
+  register: find_custom_fluentd_filters
+
+- name: Copying over custom fluentd filter config files
+  template:
+    src: "{{ item.path }}"
+    dest: "{{ node_config_directory }}/fluentd/filter/{{ item.path | basename }}"
+  register: fluentd_filter_custom
+  with_items: "{{ find_custom_fluentd_filters.files }}"
+  notify:
+    - Restart fluentd container
+
 - name: Copying over td-agent.conf
   template:
     src: "td-agent.conf.j2"
diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2
index e12daa3129..f711699444 100644
--- a/ansible/roles/common/templates/fluentd.json.j2
+++ b/ansible/roles/common/templates/fluentd.json.j2
@@ -47,15 +47,11 @@
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
         },
+        {# Copy all configuration files in filter/ directory to include #}
+        {# custom filter configs. #}
         {
-            "source": "{{ container_config_directory }}/filter/00-record_transformer.conf",
-            "dest": "{{ fluentd_dir }}/filter/00-record_transformer.conf",
-            "owner": "{{ fluentd_user }}",
-            "perm": "0600"
-        },
-        {
-            "source": "{{ container_config_directory }}/filter/01-rewrite.conf",
-            "dest": "{{ fluentd_dir }}/filter/01-rewrite.conf",
+            "source": "{{ container_config_directory }}/filter/*.conf",
+            "dest": "{{ fluentd_dir }}/filter/",
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
         },
diff --git a/doc/source/reference/central-logging-guide.rst b/doc/source/reference/central-logging-guide.rst
index 25e39eeee8..e4cc40e4fc 100644
--- a/doc/source/reference/central-logging-guide.rst
+++ b/doc/source/reference/central-logging-guide.rst
@@ -155,6 +155,18 @@ can also be exported at once by choosing *export everything* option.
 In the same tab (Settings - Objects) one can also import saved items by
 choosing *import* option.
 
+Custom log filtering
+====================
+
+In some scenarios it may be useful to apply custom filters to logs before
+forwarding them.  This may be useful to add additional tags to the messages
+or to modify the tags to conform to a log format that differs from the one
+defined by kolla-ansible.
+
+Configuration of custom fluentd filters is possible by placing filter
+configuration files in ``/etc/kolla/config/fluentd/filter/*.conf`` on the
+control host.
+
 Custom log forwarding
 =====================
 
diff --git a/releasenotes/notes/fluentd-custom-filter-config-26ee8801c7782b08.yaml b/releasenotes/notes/fluentd-custom-filter-config-26ee8801c7782b08.yaml
new file mode 100644
index 0000000000..a56770873b
--- /dev/null
+++ b/releasenotes/notes/fluentd-custom-filter-config-26ee8801c7782b08.yaml
@@ -0,0 +1,13 @@
+---
+features:
+  - |
+    Adds ability to configure custom fluentd filters.
+
+    In some scenarios it may be useful to apply custom filters to logs before
+    forwarding them.  This may be useful to add additional tags to the messages
+    or to modify the tags to conform to a log format that differs from the one
+    defined by kolla-ansible.
+
+    Configuration of custom fluentd filters is possible by placing filter
+    configuration files in ``/etc/kolla/config/fluentd/filter/*.conf`` on the
+    control host.