From 3676c8958ee31f3f9bfc17c4217748934f46e356 Mon Sep 17 00:00:00 2001
From: Juan Badia Payno <jbadiapa@redhat.com>
Date: Tue, 10 Jul 2018 12:17:27 +0200
Subject: [PATCH] Fluentd:Add extra transformation for non containerized
 services

Containerized an non-containerized services can only be mixed
on Pike.
Currently at Pike,all the services are expected to be containerized
or non containerized but not mixed.

This patch adds an extra param so log paths are able to be
tranformed accordingly.

Added some units test

Change-Id: I3e104fbaba0b7944deccf39fd87f3efaf1e21649
Related-Bug: #1742651
---
 Puppetfile_extras                             |   4 +
 manifests/profile/base/logging/fluentd.pp     |  25 ++-
 ...ipleo_profile_base_logging_fluentd_spec.rb | 173 ++++++++++++++++++
 3 files changed, 196 insertions(+), 6 deletions(-)
 create mode 100644 spec/classes/tripleo_profile_base_logging_fluentd_spec.rb

diff --git a/Puppetfile_extras b/Puppetfile_extras
index ab94601d9..324618ee3 100644
--- a/Puppetfile_extras
+++ b/Puppetfile_extras
@@ -33,6 +33,10 @@ mod 'fdio',
   :git => 'https://git.fd.io/puppet-fdio',
   :ref => 'master'
 
+mod 'fluentd',
+  :git => 'https://github.com/soylent/konstantin-fluentd',
+  :ref => 'master'
+
 mod 'certmonger',
   :git => 'https://github.com/earsdown/puppet-certmonger',
   :ref => 'v1.1.1'
diff --git a/manifests/profile/base/logging/fluentd.pp b/manifests/profile/base/logging/fluentd.pp
index 2e41942de..e0809d7ef 100644
--- a/manifests/profile/base/logging/fluentd.pp
+++ b/manifests/profile/base/logging/fluentd.pp
@@ -63,6 +63,11 @@
 #   (Optional) List. Specifies [find, replace] arguments that will be
 #   used to transform the 'path' value for logging sources using puppet's
 #   regsubst function.
+#
+# [*non_containerized_logs*]
+#   (Optional) List. Log path that are not containerized so the log would
+#   be at path_transformation[2].
+#
 class tripleo::profile::base::logging::fluentd (
   $step = Integer(hiera('step')),
   $fluentd_sources = undef,
@@ -75,7 +80,8 @@ class tripleo::profile::base::logging::fluentd (
   $fluentd_shared_key = undef,
   $fluentd_listen_syslog = true,
   $fluentd_syslog_port = 42185,
-  $fluentd_path_transform = undef
+  $fluentd_path_transform = undef,
+  $non_containerized_logs = []
 ) {
 
   if $step >= 4 {
@@ -108,12 +114,19 @@ class tripleo::profile::base::logging::fluentd (
       if $fluentd_path_transform {
         $_fluentd_sources = map($fluentd_sources) |$source| {
           if $source['path'] {
-            $newpath = {
-              'path' => regsubst($source['path'],
-                        $fluentd_path_transform[0],
-                        $fluentd_path_transform[1])
+            if ($source['path'] in $non_containerized_logs ){
+                $newpath = {
+                'path' => regsubst($source['path'],
+                          $fluentd_path_transform[0],
+                          $fluentd_path_transform[2])
+                }
+            } else {
+              $newpath = {
+                'path' => regsubst($source['path'],
+                          $fluentd_path_transform[0],
+                          $fluentd_path_transform[1])
+              }
             }
-
             $source + $newpath
           } else {
             $source
diff --git a/spec/classes/tripleo_profile_base_logging_fluentd_spec.rb b/spec/classes/tripleo_profile_base_logging_fluentd_spec.rb
new file mode 100644
index 000000000..77c4edff2
--- /dev/null
+++ b/spec/classes/tripleo_profile_base_logging_fluentd_spec.rb
@@ -0,0 +1,173 @@
+#
+# Copyright (C) 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+require 'spec_helper'
+
+describe 'tripleo::profile::base::logging::fluentd' do
+  shared_examples_for 'tripleo::profile::base::logging::fluentd' do
+
+    context 'with step less than 4' do
+      let(:params) { { :step => 3 } }
+
+      it 'should do nothing' do
+        is_expected.to_not contain_class('fluentd')
+        is_expected.to_not contain_fluentd__plugin('rubygem-fluent-plugin-add')
+      end
+    end
+
+    context 'with defaults and step greater than 3' do
+      let(:params) { { :step => 4 } }
+
+      it { is_expected.to contain_class('fluentd') }
+      it { is_expected.to contain_fluentd__plugin('rubygem-fluent-plugin-add').with(
+        :plugin_provider => 'yum',
+      ) }
+    end
+
+    context 'step greater than 3 and a fluentd source' do
+      let(:params) { {
+        :step => 4,
+        :fluentd_sources => [ {
+          'format' => '/(?<time>\\d{4}-\\d{2}-\\d{2} \\d{2} =>\\d{2}:\\d{2}.\\d+) (?<pid>\\d+) (?<priority>\\S+) (?<message>.*)$/',
+          'path' => '/var/log/keystone/keystone.log',
+          'pos_file' => '/var/cache/fluentd/openstack.keystone.pos',
+          'tag' => 'openstack.keystone',
+          'type' => 'tail'
+        } ],
+      } }
+
+      it { is_expected.to contain_class('fluentd') }
+      it { is_expected.to contain_fluentd__plugin('rubygem-fluent-plugin-add').with(
+        :plugin_provider => 'yum',
+      ) }
+      it { is_expected.to contain_fluentd__config('100-openstack-sources.conf').with(
+        :config => {
+          'source' => params[:fluentd_sources]
+        }
+      ) }
+    end
+
+    context 'step greater than 3 and a fluentd source with transformation' do
+      let(:params) { {
+        :step => 4,
+        :fluentd_sources => [ {
+          'format' => '/(?<time>\\d{4}-\\d{2}-\\d{2} \\d{2} =>\\d{2}:\\d{2}.\\d+) (?<pid>\\d+) (?<priority>\\S+) (?<message>.*)$/',
+          'path' => '/var/log/keystone/keystone.log',
+          'pos_file' => '/var/cache/fluentd/openstack.keystone.pos',
+          'tag' => 'openstack.keystone',
+          'type' => 'tail'
+        } ],
+        :fluentd_path_transform => [
+          '/var/log/',
+          '/var/log/containers/',
+        ]
+      } }
+
+      it { is_expected.to contain_class('fluentd') }
+      it { is_expected.to contain_fluentd__plugin('rubygem-fluent-plugin-add').with(
+        :plugin_provider => 'yum',
+      ) }
+      it { is_expected.to contain_fluentd__config('100-openstack-sources.conf').with(
+        :config => {
+          'source' => [ {
+          'format' => '/(?<time>\\d{4}-\\d{2}-\\d{2} \\d{2} =>\\d{2}:\\d{2}.\\d+) (?<pid>\\d+) (?<priority>\\S+) (?<message>.*)$/',
+          'path' => '/var/log/containers/keystone/keystone.log',
+          'pos_file' => '/var/cache/fluentd/openstack.keystone.pos',
+          'tag' => 'openstack.keystone',
+          'type' => 'tail'
+        } ]
+        }
+      ) }
+    end
+
+   context 'step greater than 3 and a fluentd source with transformation' do
+      let(:params) { {
+        :step => 4,
+        :fluentd_sources => [ {
+          'format' => '/(?<time>\\d{4}-\\d{2}-\\d{2} \\d{2} =>\\d{2}:\\d{2}.\\d+) (?<pid>\\d+) (?<priority>\\S+) (?<message>.*)$/',
+          'path' => '/var/log/keystone/keystone.log',
+          'pos_file' => '/var/cache/fluentd/openstack.keystone.pos',
+          'tag' => 'openstack.keystone',
+          'type' => 'tail'
+        } ],
+        :fluentd_path_transform => [
+          '/var/log/',
+          '/var/log/containers/'
+        ]
+      } }
+
+      it { is_expected.to contain_class('fluentd') }
+      it { is_expected.to contain_fluentd__plugin('rubygem-fluent-plugin-add').with(
+        :plugin_provider => 'yum',
+      ) }
+      it { is_expected.to contain_fluentd__config('100-openstack-sources.conf').with(
+        :config => {
+          'source' => [ {
+          'format' => '/(?<time>\\d{4}-\\d{2}-\\d{2} \\d{2} =>\\d{2}:\\d{2}.\\d+) (?<pid>\\d+) (?<priority>\\S+) (?<message>.*)$/',
+          'path' => '/var/log/containers/keystone/keystone.log',
+          'pos_file' => '/var/cache/fluentd/openstack.keystone.pos',
+          'tag' => 'openstack.keystone',
+          'type' => 'tail'
+        } ]
+        }
+      ) }
+   end
+
+  context 'step greater than 3 and a fluentd source for a non-containerized service' do
+      let(:params) { {
+        :step => 4,
+        :fluentd_sources => [ {
+          'format' => '/(?<time>\\d{4}-\\d{2}-\\d{2} \\d{2} =>\\d{2}:\\d{2}.\\d+) (?<pid>\\d+) (?<priority>\\S+) (?<message>.*)$/',
+          'path' => '/var/log/keystone/keystone.log',
+          'pos_file' => '/var/cache/fluentd/openstack.keystone.pos',
+          'tag' => 'openstack.keystone',
+          'type' => 'tail'
+        } ],
+        :non_containerized_logs =>['/var/log/keystone/keystone.log'],
+        :fluentd_path_transform => [
+          '/var/log/',
+          '/var/log/containers/',
+          '/var/log/containers/host/',
+        ]
+      } }
+
+      it { is_expected.to contain_class('fluentd') }
+      it { is_expected.to contain_fluentd__plugin('rubygem-fluent-plugin-add').with(
+        :plugin_provider => 'yum',
+      ) }
+      it { is_expected.to contain_fluentd__config('100-openstack-sources.conf').with(
+        :config => {
+          'source' => [ {
+          'format' => '/(?<time>\\d{4}-\\d{2}-\\d{2} \\d{2} =>\\d{2}:\\d{2}.\\d+) (?<pid>\\d+) (?<priority>\\S+) (?<message>.*)$/',
+          'path' => '/var/log/containers/host/keystone/keystone.log',
+          'pos_file' => '/var/cache/fluentd/openstack.keystone.pos',
+          'tag' => 'openstack.keystone',
+          'type' => 'tail'
+        } ]
+        }
+      ) }
+    end
+  end
+
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let (:facts) {
+        facts
+      }
+      it_behaves_like 'tripleo::profile::base::logging::fluentd'
+    end
+  end
+end