Specify explicitly Neutron log file names parsed by Hekad

Fixes-bug: #1546424
Change-Id: Icacfb8f7c6b81817856df468aeb592978a8d26e8

deployment_scripts/puppet/manifests/controller.pp

@@ -207,7 +207,14 @@ service { $heat_engine_service:
 # OpenStack logs are useful for deriving HTTP metrics, so we enable them even
 # if Elasticsearch is disabled.
 lma_collector::logs::openstack { 'nova': }
-lma_collector::logs::openstack { 'neutron': }
+
+# For every virtual network that exists, Neutron spawns one metadata proxy
+# service that will log to a separate file in the Neutron log directory.
+# Eventually it may be hundreds of these files and Heka will have trouble
+# coping with the situation. See bug #1547402 for details.
+lma_collector::logs::openstack { 'neutron':
+  service_match => '(dhcp-agent|l3-agent|metadata-agent|neutron-netns-cleanup|openvswitch-agent|server)',
+}
 lma_collector::logs::openstack { 'cinder': }
 lma_collector::logs::openstack { 'glance': }
 lma_collector::logs::openstack { 'heat': }

deployment_scripts/puppet/modules/lma_collector/README.md

@@ -57,7 +57,7 @@ To make the collector collect logs created by an OpenStack service declare the
 `lma_collector::logs::openstack` define. This is an example for the Nova logs:
 
 ```puppet
-lma_collector::logs::openstack { 'nova': }
+lma_collector::logs::openstack { 'nova': }
 ```
 
 This configures Heka to read the Nova logs from the log files located in
@@ -726,6 +726,14 @@ The define doesn't work for Swift, as Swift only writes its logs to Syslog.
 See the specific [`lma_collector::logs::swift`](#class-lma_collectorlogsswift)
 class for Swift.
 
+##### Parameters
+
+* `service_match`: *Optional*. The regular expression portion which matches the
+  log file names excluding the suffix `.log`. This is generally used to
+  explicitly specify the file name(s) within the directory.
+  Valid options: a regexp string supported by the [Go programming
+  language](https://golang.org/pkg/regexp/). Default: `.+`.
+
 #### Define: `lma_collector::collectd::openstack`
 
 Declare this define to make collectd collect statistics from an OpenStack

deployment_scripts/puppet/modules/lma_collector/manifests/logs/openstack.pp

@@ -23,7 +23,9 @@
 # It works for "standard" OpenStack services that write their logs into log files
 # located in /var/log/{service}, where {service} is the service name.
 #
-define lma_collector::logs::openstack {
+define lma_collector::logs::openstack (
+  $service_match = '.+',
+) {
 
   # Note: $log_directory could be made configurable in the future.
 
@@ -36,7 +38,7 @@ define lma_collector::logs::openstack {
     log_directory  => "/var/log/${title}",
     decoder        => 'openstack',
     splitter       => 'openstack',
-    file_match     => '(?P<Service>.+)\.log\.?(?P<Seq>\d*)$',
+    file_match     => "(?P<Service>${service_match})\\.log\\.?(?P<Seq>\\d*)$",
     differentiator => "['${title}', '_', 'Service']",
     priority       => '["^Seq"]',
     require        => Class['lma_collector::logs::openstack_decoder_splitter'],

deployment_scripts/puppet/modules/lma_collector/spec/defines/lma_collector_logs_openstack_spec.rb

@@ -24,6 +24,18 @@ describe "lma_collector::logs::openstack" do
     describe "with title" do
         let(:title) { :nova }
         it { is_expected.to contain_heka__input__logstreamer('nova') \
-             .with_differentiator("['nova', '_', 'Service']") }
+             .with_differentiator("['nova', '_', 'Service']") \
+             .with_file_match('(?P<Service>.+)\.log\.?(?P<Seq>\d*)$')
+        }
+    end
+    describe "with title and service_match" do
+        let(:title) { :nova }
+        let(:params) do
+          { :service_match => '(dhcp-agent|l3-agent|metadata-agent|neutron-netns-cleanup|openvswitch-agent|server)' }
+        end
+
+        it { is_expected.to contain_heka__input__logstreamer('nova') \
+             .with_file_match('(?P<Service>(dhcp-agent|l3-agent|metadata-agent|neutron-netns-cleanup|openvswitch-agent|server))\.log\.?(?P<Seq>\d*)$')
+        }
     end
 end