From 8491d10d265222e6b6c26c9a6f9f407ad82faf51 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Thu, 19 Nov 2015 16:33:22 -0800 Subject: [PATCH] Decouple log processing from logstash As part of the move to logstash 2.0 we are relying on upstream packaging for logstash. This packaging replaces a lot of the micromanagement of users and groups and dirs that was done in puppet for logstash. This is great news because its less work for us but means that the log processors can't rely on puppet resources for those items and we don't actually want to install logstash package everywhere we run log processor daemons. Since the log processors don't need a logstash service running and actually don't need any of the logstash stuff at all decouple them completely and have the log processor daemons use their own user, group, log dir, config dir, etc. With this in place we can easily switch to using the logstash packages only where we actually need logstash to be running. Change-Id: I2354fbe9d3ab25134c52bfe58f562dfdf9ff6786 --- files/jenkins-log-client.init | 4 +-- manifests/client.pp | 10 ++++---- manifests/init.pp | 35 ++++++++++++++++++++++++--- manifests/worker.pp | 15 +++++------- metadata.json | 1 - templates/jenkins-log-worker.init.erb | 4 +-- 6 files changed, 47 insertions(+), 22 deletions(-) diff --git a/files/jenkins-log-client.init b/files/jenkins-log-client.init index 49f9d65..e9379db 100755 --- a/files/jenkins-log-client.init +++ b/files/jenkins-log-client.init @@ -17,9 +17,9 @@ DESC="Jenkins Log Client" NAME=jenkins-log-client DAEMON=/usr/local/bin/log-gearman-client.py PIDFILE=/var/run/$NAME/$NAME.pid -DAEMON_ARGS="-c /etc/logstash/jenkins-log-client.yaml -d /var/log/logstash/log-client-debug.log -p $PIDFILE" +DAEMON_ARGS="-c /etc/logprocessor/jenkins-log-client.yaml -d /var/log/logprocessor/log-client-debug.log -p $PIDFILE" SCRIPTNAME=/etc/init.d/$NAME -USER=logstash +USER=logprocessor # Exit if the package is not installed [ -x "$DAEMON" ] || exit 0 diff --git a/manifests/client.pp b/manifests/client.pp index 999a4af..ff1288f 100644 --- a/manifests/client.pp +++ b/manifests/client.pp @@ -20,13 +20,13 @@ class log_processor::client ( $statsd_host = undef, ) { - file { '/etc/logstash/jenkins-log-client.yaml': + file { '/etc/logprocessor/jenkins-log-client.yaml': ensure => present, owner => 'root', group => 'root', mode => '0555', source => $config_file, - require => File['/etc/logstash'], + require => File['/etc/logprocessor'], } file { '/etc/init.d/jenkins-log-client': @@ -37,7 +37,7 @@ class log_processor::client ( source => 'puppet:///modules/log_processor/jenkins-log-client.init', require => [ File['/usr/local/bin/log-gearman-client.py'], - File['/etc/logstash/jenkins-log-client.yaml'], + File['/etc/logprocessor/jenkins-log-client.yaml'], File['/etc/default/jenkins-log-client'], ], } @@ -53,13 +53,13 @@ class log_processor::client ( service { 'jenkins-log-client': enable => true, hasrestart => true, - subscribe => File['/etc/logstash/jenkins-log-client.yaml'], + subscribe => File['/etc/logprocessor/jenkins-log-client.yaml'], require => File['/etc/init.d/jenkins-log-client'], } include ::logrotate logrotate::file { 'log-client-debug.log': - log => '/var/log/logstash/log-client-debug.log', + log => '/var/log/logprocessor/log-client-debug.log', options => [ 'compress', 'copytruncate', diff --git a/manifests/init.pp b/manifests/init.pp index e260ed4..7ceb9c2 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -23,6 +23,35 @@ class log_processor ( } } + group { 'logprocessor': + ensure => present, + } + user { 'logprocessor': + ensure => present, + comment => 'Log Processor User', + home => '/etc/logprocessor', + gid => 'logprocessor', + shell => '/bin/bash', + membership => 'minimum', + require => Group['logprocessor'], + } + + file { '/etc/logprocessor': + ensure => directory, + owner => 'logprocessor', + group => 'logprocessor', + mode => '0755', + require => User['logprocessor'], + } + + file { '/var/log/logprocessor': + ensure => directory, + owner => 'logprocessor', + group => 'logprocessor', + mode => '0755', + require => User['logprocessor'], + } + package { 'python-zmq': ensure => present, } @@ -50,9 +79,9 @@ class log_processor ( file { '/var/lib/crm114': ensure => directory, - owner => 'logstash', - group => 'logstash', - require => User['logstash'], + owner => 'logprocessor', + group => 'logprocessor', + require => User['logprocessor'], } file { '/usr/local/bin/classify-log.crm': diff --git a/manifests/worker.pp b/manifests/worker.pp index cfe173d..427fee2 100644 --- a/manifests/worker.pp +++ b/manifests/worker.pp @@ -20,13 +20,13 @@ define log_processor::worker ( ) { $suffix = "-${name}" - file { "/etc/logstash/jenkins-log-worker${suffix}.yaml": + file { "/etc/logprocessor/jenkins-log-worker${suffix}.yaml": ensure => present, owner => 'root', group => 'root', mode => '0555', source => $config_file, - require => Class['logstash::indexer'], + require => User['logprocessor'], } file { "/etc/init.d/jenkins-log-worker${suffix}": @@ -37,23 +37,20 @@ define log_processor::worker ( content => template('log_processor/jenkins-log-worker.init.erb'), require => [ File['/usr/local/bin/log-gearman-worker.py'], - File["/etc/logstash/jenkins-log-worker${suffix}.yaml"], + File["/etc/logprocessor/jenkins-log-worker${suffix}.yaml"], ], } service { "jenkins-log-worker${suffix}": enable => true, hasrestart => true, - subscribe => File["/etc/logstash/jenkins-log-worker${suffix}.yaml"], - require => [ - Class['logstash::indexer'], - File["/etc/init.d/jenkins-log-worker${suffix}"], - ], + subscribe => File["/etc/logprocessor/jenkins-log-worker${suffix}.yaml"], + require => File["/etc/init.d/jenkins-log-worker${suffix}"], } include ::logrotate logrotate::file { "log-worker${suffix}-debug.log": - log => "/var/log/logstash/log-worker${suffix}-debug.log", + log => "/var/log/logprocessor/log-worker${suffix}-debug.log", options => [ 'compress', 'copytruncate', diff --git a/metadata.json b/metadata.json index 2a1aed9..be93385 100644 --- a/metadata.json +++ b/metadata.json @@ -8,7 +8,6 @@ "project_page": "http://docs.openstack.org/infra/system-config/", "issues_url": "https://storyboard.openstack.org/#!/project/801", "dependencies": [ - {"name":"openstackci/logstash","version_requirement":">= 0.0.1"}, {"name":"openstackci/pip","version_requirement":">= 0.0.1"}, {"name":"openstackci/logrotate","version_requirement":">= 0.0.1"} ] diff --git a/templates/jenkins-log-worker.init.erb b/templates/jenkins-log-worker.init.erb index f122623..86e0f26 100755 --- a/templates/jenkins-log-worker.init.erb +++ b/templates/jenkins-log-worker.init.erb @@ -17,9 +17,9 @@ DESC="Jenkins Log Worker" NAME=jenkins-log-worker<%= suffix %> DAEMON=/usr/local/bin/log-gearman-worker.py PIDFILE=/var/run/$NAME/$NAME.pid -DAEMON_ARGS="-c /etc/logstash/jenkins-log-worker<%= suffix %>.yaml -d /var/log/logstash/log-worker<%= suffix %>-debug.log -p $PIDFILE" +DAEMON_ARGS="-c /etc/logprocessor/jenkins-log-worker<%= suffix %>.yaml -d /var/log/logprocessor/log-worker<%= suffix %>-debug.log -p $PIDFILE" SCRIPTNAME=/etc/init.d/$NAME -USER=logstash +USER=logprocessor # Exit if the package is not installed [ -x "$DAEMON" ] || exit 0