Fluentd: Transform path by service

Currently, the configuration delivered via
service_config_settings does not provide the
transformation path regarding the logs for
containerized deployments. So logs are not
correctly configured on fluentd on containerized
deployments.

This path added this feature so the logs are
properly located on fluentd configuration.

Added unit test for this issue

Partial-bug: #1715187
Partial-bug: #1716427
Change-Id: Ie7df4b8b94cb0ae38096ab95800f211ef1cd8455

manifests/profile/base/logging/fluentd.pp

@@ -100,8 +100,9 @@ class tripleo::profile::base::logging::fluentd (
     # Load per-service plugin configuration
     ::tripleo::profile::base::logging::fluentd::fluentd_service {
       $service_names:
-        pos_file_path  => $fluentd_pos_file_path,
-        default_format => $fluentd_default_format
+        pos_file_path     => $fluentd_pos_file_path,
+        default_format    => $fluentd_default_format,
+        fluentd_transform => $fluentd_path_transform
     }
 
     if $fluentd_manage_groups {

manifests/profile/base/logging/fluentd/fluentd_service.pp

@@ -7,9 +7,15 @@
 #
 # [*default_format*]
 #   Default regular expression against which to match log messages.
+#
+# [*fluentd_transform*]
+#   Two value array where the first value is a regular expresion to
+#   be replaced and the second one is the replacement.
+#
 define tripleo::profile::base::logging::fluentd::fluentd_service (
   $pos_file_path  = undef,
-  $default_format = undef
+  $default_format = undef,
+  $fluentd_transform = undef
 ) {
   $sources = hiera("tripleo_fluentd_sources_${title}", [])
   $filters = hiera("tripleo_fluentd_filters_${title}", [])
@@ -23,14 +29,29 @@ define tripleo::profile::base::logging::fluentd::fluentd_service (
   # Check that we have something to configure to avoid creating
   # stub config files.
   if !empty($sources) or !empty($filters) or !empty($matches) {
+    if $fluentd_transform and !empty($sources) {
+      $new_source = map($sources) |$source| {
+          if $source['path'] {
+            $newpath = {
+              'path' => regsubst($source['path'],
+                        $fluentd_transform[0],
+                        $fluentd_transform[1])
+            }
 
+            $source + $newpath
+          } else {
+            $source
+          }
+      }
+    }else{
+      $new_source = $sources
+    }
     # Insert default values into list of sources.
-    $_sources = $sources.map |$src| {
+    $_sources = $new_source.map |$src| {
       $default_source
       + {pos_file => "${pos_file_path}/${src['tag']}.pos"}
       + $src
     }
-
     ::fluentd::config { "100-openstack-${title}.conf":
       config => {
         'source' => $_sources,

spec/classes/tripleo_profile_base_logging_fluentd_spec.rb

@@ -62,6 +62,9 @@ describe 'tripleo::profile::base::logging::fluentd' do
           'source' => params[:fluentd_sources]
         }
       ) }
+      it { is_expected.to contain_file('/etc/fluentd/config.d/100-openstack-sources.conf').with_content(
+      /^\s*path \/var\/log\/keystone\/keystone\.log$/
+      ) }
     end
 
     context 'step greater than 3 and a fluentd source with transformation' do
@@ -112,7 +115,42 @@ describe 'tripleo::profile::base::logging::fluentd' do
        :default_format => '/(?<time>\\d{4}-\\d{2}-\\d{2} \\d{2} =>\\d{2}:\\d{2}.\\d+) (?<pid>\\d+) (?<priority>\\S+) (?<message>.*)$/'
       ) }
       it { is_expected.to contain_fluentd__config('100-openstack-ceilometer_agent_central.conf') }
-      it { is_expected.to contain_file('/etc/fluentd/config.d/100-openstack-ceilometer_agent_central.conf') }
+      it { is_expected.to contain_file('/etc/fluentd/config.d/100-openstack-ceilometer_agent_central.conf').with_content(
+      /^\s*path \/var\/log\/ceilometer\/central\.log$/
+      ) }
+    end
+
+    context 'Config by service -- ceilometer_agent_central with path trasnformation' do
+      let(:params) { {
+        :step => 4,
+        :fluentd_default_format => '/(?<time>\\d{4}-\\d{2}-\\d{2} \\d{2} =>\\d{2}:\\d{2}.\\d+) (?<pid>\\d+) (?<priority>\\S+) (?<message>.*)$/',
+        :fluentd_manage_groups => false,
+        :fluentd_pos_file_path => '/var/cache/fluentd/',
+        :service_names => [ 'ceilometer_agent_central' ],
+        :fluentd_path_transform => [
+          '/var/log/',
+          '/var/log/containers/',
+        ]
+      } }
+      it { is_expected.to contain_class('fluentd') }
+      it { is_expected.to contain_file('/var/cache/fluentd/') }
+      it { is_expected.to contain_tripleo__profile__base__logging__fluentd__fluentd_service('ceilometer_agent_central').with(
+       :pos_file_path => '/var/cache/fluentd/',
+       :default_format => '/(?<time>\\d{4}-\\d{2}-\\d{2} \\d{2} =>\\d{2}:\\d{2}.\\d+) (?<pid>\\d+) (?<priority>\\S+) (?<message>.*)$/'
+      ) }
+      it { is_expected.to contain_fluentd__config('100-openstack-ceilometer_agent_central.conf').with(
+        :config => {
+          'source' =>  [ {
+          'format' => '/(?<time>\\d{4}-\\d{2}-\\d{2} \\d{2} =>\\d{2}:\\d{2}.\\d+) (?<pid>\\d+) (?<priority>\\S+) (?<message>.*)$/',
+          'path' => '/var/log/containers/ceilometer/central.log',
+          'pos_file' => '/var/cache/fluentd//openstack.ceilometer.agent.central.pos',
+          'tag' => 'openstack.ceilometer.agent.central',
+          'type' => 'tail'
+        } ],
+          'filter' => [],
+          'match' => []
+        }
+      ) }
     end
 
     context 'Groups by service -- ceilometer_agent_central added ceilometer' do