Decouple log processing from logstash

As part of the move to logstash 2.0 we are relying on upstream packaging
for logstash. This packaging replaces a lot of the micromanagement of
users and groups and dirs that was done in puppet for logstash. This is
great news because its less work for us but means that the log
processors can't rely on puppet resources for those items and we don't
actually want to install logstash package everywhere we run log
processor daemons.

Since the log processors don't need a logstash service running and
actually don't need any of the logstash stuff at all decouple them
completely and have the log processor daemons use their own user, group,
log dir, config dir, etc. With this in place we can easily switch to
using the logstash packages only where we actually need logstash to be
running.

Change-Id: I2354fbe9d3ab25134c52bfe58f562dfdf9ff6786
This commit is contained in:
Clark Boylan 2015-11-19 16:33:22 -08:00
parent 08274f5f35
commit 8491d10d26
6 changed files with 47 additions and 22 deletions

View File

@ -17,9 +17,9 @@ DESC="Jenkins Log Client"
NAME=jenkins-log-client
DAEMON=/usr/local/bin/log-gearman-client.py
PIDFILE=/var/run/$NAME/$NAME.pid
DAEMON_ARGS="-c /etc/logstash/jenkins-log-client.yaml -d /var/log/logstash/log-client-debug.log -p $PIDFILE"
DAEMON_ARGS="-c /etc/logprocessor/jenkins-log-client.yaml -d /var/log/logprocessor/log-client-debug.log -p $PIDFILE"
SCRIPTNAME=/etc/init.d/$NAME
USER=logstash
USER=logprocessor
# Exit if the package is not installed
[ -x "$DAEMON" ] || exit 0

View File

@ -20,13 +20,13 @@ class log_processor::client (
$statsd_host = undef,
) {
file { '/etc/logstash/jenkins-log-client.yaml':
file { '/etc/logprocessor/jenkins-log-client.yaml':
ensure => present,
owner => 'root',
group => 'root',
mode => '0555',
source => $config_file,
require => File['/etc/logstash'],
require => File['/etc/logprocessor'],
}
file { '/etc/init.d/jenkins-log-client':
@ -37,7 +37,7 @@ class log_processor::client (
source => 'puppet:///modules/log_processor/jenkins-log-client.init',
require => [
File['/usr/local/bin/log-gearman-client.py'],
File['/etc/logstash/jenkins-log-client.yaml'],
File['/etc/logprocessor/jenkins-log-client.yaml'],
File['/etc/default/jenkins-log-client'],
],
}
@ -53,13 +53,13 @@ class log_processor::client (
service { 'jenkins-log-client':
enable => true,
hasrestart => true,
subscribe => File['/etc/logstash/jenkins-log-client.yaml'],
subscribe => File['/etc/logprocessor/jenkins-log-client.yaml'],
require => File['/etc/init.d/jenkins-log-client'],
}
include ::logrotate
logrotate::file { 'log-client-debug.log':
log => '/var/log/logstash/log-client-debug.log',
log => '/var/log/logprocessor/log-client-debug.log',
options => [
'compress',
'copytruncate',

View File

@ -23,6 +23,35 @@ class log_processor (
}
}
group { 'logprocessor':
ensure => present,
}
user { 'logprocessor':
ensure => present,
comment => 'Log Processor User',
home => '/etc/logprocessor',
gid => 'logprocessor',
shell => '/bin/bash',
membership => 'minimum',
require => Group['logprocessor'],
}
file { '/etc/logprocessor':
ensure => directory,
owner => 'logprocessor',
group => 'logprocessor',
mode => '0755',
require => User['logprocessor'],
}
file { '/var/log/logprocessor':
ensure => directory,
owner => 'logprocessor',
group => 'logprocessor',
mode => '0755',
require => User['logprocessor'],
}
package { 'python-zmq':
ensure => present,
}
@ -50,9 +79,9 @@ class log_processor (
file { '/var/lib/crm114':
ensure => directory,
owner => 'logstash',
group => 'logstash',
require => User['logstash'],
owner => 'logprocessor',
group => 'logprocessor',
require => User['logprocessor'],
}
file { '/usr/local/bin/classify-log.crm':

View File

@ -20,13 +20,13 @@ define log_processor::worker (
) {
$suffix = "-${name}"
file { "/etc/logstash/jenkins-log-worker${suffix}.yaml":
file { "/etc/logprocessor/jenkins-log-worker${suffix}.yaml":
ensure => present,
owner => 'root',
group => 'root',
mode => '0555',
source => $config_file,
require => Class['logstash::indexer'],
require => User['logprocessor'],
}
file { "/etc/init.d/jenkins-log-worker${suffix}":
@ -37,23 +37,20 @@ define log_processor::worker (
content => template('log_processor/jenkins-log-worker.init.erb'),
require => [
File['/usr/local/bin/log-gearman-worker.py'],
File["/etc/logstash/jenkins-log-worker${suffix}.yaml"],
File["/etc/logprocessor/jenkins-log-worker${suffix}.yaml"],
],
}
service { "jenkins-log-worker${suffix}":
enable => true,
hasrestart => true,
subscribe => File["/etc/logstash/jenkins-log-worker${suffix}.yaml"],
require => [
Class['logstash::indexer'],
File["/etc/init.d/jenkins-log-worker${suffix}"],
],
subscribe => File["/etc/logprocessor/jenkins-log-worker${suffix}.yaml"],
require => File["/etc/init.d/jenkins-log-worker${suffix}"],
}
include ::logrotate
logrotate::file { "log-worker${suffix}-debug.log":
log => "/var/log/logstash/log-worker${suffix}-debug.log",
log => "/var/log/logprocessor/log-worker${suffix}-debug.log",
options => [
'compress',
'copytruncate',

View File

@ -8,7 +8,6 @@
"project_page": "http://docs.openstack.org/infra/system-config/",
"issues_url": "https://storyboard.openstack.org/#!/project/801",
"dependencies": [
{"name":"openstackci/logstash","version_requirement":">= 0.0.1"},
{"name":"openstackci/pip","version_requirement":">= 0.0.1"},
{"name":"openstackci/logrotate","version_requirement":">= 0.0.1"}
]

View File

@ -17,9 +17,9 @@ DESC="Jenkins Log Worker"
NAME=jenkins-log-worker<%= suffix %>
DAEMON=/usr/local/bin/log-gearman-worker.py
PIDFILE=/var/run/$NAME/$NAME.pid
DAEMON_ARGS="-c /etc/logstash/jenkins-log-worker<%= suffix %>.yaml -d /var/log/logstash/log-worker<%= suffix %>-debug.log -p $PIDFILE"
DAEMON_ARGS="-c /etc/logprocessor/jenkins-log-worker<%= suffix %>.yaml -d /var/log/logprocessor/log-worker<%= suffix %>-debug.log -p $PIDFILE"
SCRIPTNAME=/etc/init.d/$NAME
USER=logstash
USER=logprocessor
# Exit if the package is not installed
[ -x "$DAEMON" ] || exit 0