Add a simple watchdog for logstash-indexer service

The logstash-indexer services sometimes fall out of the elasticsearch
cluster when it is under heavy load and unable to respond to pings.
Logstash doesn't do anything to reconnect :( so restart the service if a
node detects that it has fallen out of the cluster.

Upstream bug submitted at https://logstash.jira.com/browse/LOGSTASH-1951

Change-Id: I2e7767c5fe20cff279366fec2ddadd7710dbb4a9
This commit is contained in:
Clark Boylan 2014-02-24 11:23:48 -08:00
parent f45fe600fb
commit e6d78af997
3 changed files with 62 additions and 0 deletions

View File

@ -0,0 +1,17 @@
#!/bin/bash
#
# This is a work around for https://logstash.jira.com/browse/LOGSTASH-1951
# Logstash disconnects from the cluster and will not rejoin under
# its own power.
ES_ADDRESS=$1
JSON_OUT=$(curl -sf "http://${ES_ADDRESS}:9200/_cluster/nodes/${HOSTNAME}")
CURL_RET=$?
RESULT=$(echo $JSON_OUT | jq '.nodes == {}')
if [ "$CURL_RET" == "0" ] && [ "$RESULT" == "true" ] ;
then
stop --quiet logstash-indexer
start --quiet logstash-indexer
fi

View File

@ -0,0 +1,41 @@
# Copyright 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# Class to install a simple watchdog for the logstash-indexer service.
# es_api_node is the address to access the elasticsearch api at (should
# be a 'host:port' string).
class logstash::watchdog (
$es_api_node = 'localhost'
) {
package { 'jq':
ensure => present,
}
file { '/usr/local/bin/logstash-watchdog':
ensure => present,
source => 'puppet:///modules/logstash/logstash-watchdog.sh',
replace => true,
owner => 'root',
group => 'root',
mode => '0555',
}
cron { 'logstash-watchdog':
minute => '*/10',
environment => 'PATH=/bin:/usr/bin:/usr/local/bin',
command => "sleep $((RANDOM\%60)) && /usr/local/bin/logstash-watchdog ${es_api_node}",
require => Service['logstash-indexer']
}
}

View File

@ -39,6 +39,10 @@ class openstack_project::logstash_worker (
conf_template => 'openstack_project/logstash/indexer.conf.erb', conf_template => 'openstack_project/logstash/indexer.conf.erb',
} }
class { 'logstash::watchdog':
es_api_node => $discover_node,
}
include log_processor include log_processor
log_processor::worker { 'A': log_processor::worker { 'A':
config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml', config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',