From 049a6e2b87f31a9a2d7f971af760e080dde6f4ee Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Mon, 25 Apr 2016 13:03:53 -0700 Subject: [PATCH] Properly support logstash conf.d Add in proper support for conf.d configs. Create input at 00-input, output at 99-output, and filters at user specifiable levels in between using a new defined type for filters. Co-Authored-By: Jonathan Harker Change-Id: Icbca7a6ba0c5a94a273ef158f707311b588483fd --- manifests/filter.pp | 34 ++++++++++++++++++++ manifests/indexer.pp | 58 +++++++++++++++++++++++++++-------- spec/acceptance/basic_spec.rb | 9 +++++- templates/input.conf.erb | 12 ++++++++ templates/output.conf.erb | 5 +++ 5 files changed, 105 insertions(+), 13 deletions(-) create mode 100644 manifests/filter.pp create mode 100644 templates/input.conf.erb create mode 100644 templates/output.conf.erb diff --git a/manifests/filter.pp b/manifests/filter.pp new file mode 100644 index 0000000..cf8880b --- /dev/null +++ b/manifests/filter.pp @@ -0,0 +1,34 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# = Class: logstash::filter +# +# Class to install logstash filter configs +# +# == Parameters +# +# [*level*] +# String. Conf level prefix for stitching files together in order. +# [*target*] +# String. Path to actual config location will be symlinked to. +define logstash::filter ( + $level, + $target, +) { + include ::logstash + + file { "/etc/logstash/conf.d/${level}-${name}.conf": + ensure => link, + target => $target, + require => Class['logstash'], + } +} diff --git a/manifests/indexer.pp b/manifests/indexer.pp index 89e087a..7627e6d 100644 --- a/manifests/indexer.pp +++ b/manifests/indexer.pp @@ -19,21 +19,56 @@ # == Parameters # # [*conf_template*] -# String. Path to indexer config template. -# Default: 'logstash/indexer.conf.erb' +# String. (deprecated) Path to indexer config template. +# Default: undef +# [*input_template*] +# String. Path to indexer input config template. +# Default: 'logstash/input.conf.erb' +# [*output_template*] +# String. Path to indexer output config template. +# Default: 'logstash/output.conf.erb' class logstash::indexer ( - $conf_template = 'logstash/indexer.conf.erb', + $conf_template = undef, + $input_template = 'logstash/input.conf.erb', + $output_template = 'logstash/output.conf.erb', ) { include ::logstash - file { '/etc/logstash/conf.d/indexer.conf': - ensure => present, - content => template($conf_template), - replace => true, - owner => 'logstash', - group => 'logstash', - mode => '0644', - require => Class['logstash'], + if $conf_template != undef { + notify { 'Using $conf_template is deprecated, please switch to $input_template, $output_template and ::logstash::filter defines.': } + + file { '/etc/logstash/conf.d/indexer.conf': + ensure => present, + content => template($conf_template), + replace => true, + owner => 'logstash', + group => 'logstash', + mode => '0644', + require => Class['logstash'], + notify => Service['logstash'], + } + } else { + file { '/etc/logstash/conf.d/00-input.conf': + ensure => present, + content => template($input_template), + replace => true, + owner => 'logstash', + group => 'logstash', + mode => '0644', + require => Class['logstash'], + notify => Service['logstash'], + } + + file { '/etc/logstash/conf.d/99-output.conf': + ensure => present, + content => template($output_template), + replace => true, + owner => 'logstash', + group => 'logstash', + mode => '0644', + require => Class['logstash'], + notify => Service['logstash'], + } } file { '/etc/default/logstash': @@ -50,7 +85,6 @@ class logstash::indexer ( ensure => running, enable => true, subscribe => [ - File['/etc/logstash/conf.d/indexer.conf'], File['/etc/default/logstash'], ], require => Class['logstash'], diff --git a/spec/acceptance/basic_spec.rb b/spec/acceptance/basic_spec.rb index 3bf7908..58f939f 100644 --- a/spec/acceptance/basic_spec.rb +++ b/spec/acceptance/basic_spec.rb @@ -30,7 +30,7 @@ describe 'puppet-logstash module', :if => ['debian', 'ubuntu'].include?(os[:fami end describe 'module logstash::indexer' do - describe file('/etc/logstash/conf.d/indexer.conf') do + describe file('/etc/logstash/conf.d/00-input.conf') do it { should be_file } it { should be_owned_by 'logstash' } it { should be_grouped_into 'logstash' } @@ -38,6 +38,13 @@ describe 'puppet-logstash module', :if => ['debian', 'ubuntu'].include?(os[:fami its(:content) { should include 'host => "127.0.0.1"' } end + describe file('/etc/logstash/conf.d/99-output.conf') do + it { should be_file } + it { should be_owned_by 'logstash' } + it { should be_grouped_into 'logstash' } + its(:content) { should include 'host => "127.0.0.1"' } + end + describe file('/etc/default/logstash') do its(:content) { should include 'LS_OPTS="-w 1"' } end diff --git a/templates/input.conf.erb b/templates/input.conf.erb new file mode 100644 index 0000000..7621a8e --- /dev/null +++ b/templates/input.conf.erb @@ -0,0 +1,12 @@ +input { + redis { + host => "127.0.0.1" + type => "redis-input" + # these settings should match the output of the agent + data_type => "list" + key => "logstash" + + # We use json_event here since the sender is a logstash agent + format => "json_event" + } +} diff --git a/templates/output.conf.erb b/templates/output.conf.erb new file mode 100644 index 0000000..14a803f --- /dev/null +++ b/templates/output.conf.erb @@ -0,0 +1,5 @@ +output { + elasticsearch { + host => "127.0.0.1" + } +}