Increase the Heka poolsize on controllers

On controller nodes, the Heka poolsize must be increased to handle the load
generated by derived metrics from logs otherwise a deadlock
can happen in the filter plugins and block heka.

Fixes-bug: #1557388
(cherry picked from commit 96df47af73)

Conflicts:
	deployment_scripts/puppet/manifests/base.pp
	deployment_scripts/puppet/modules/lma_collector/manifests/init.pp

Change-Id: I74362011d32d413f244c6cdb6e4625ed96759df0
This commit is contained in:
Swann Croiset 2016-04-05 09:53:24 +02:00
parent 451ab33a69
commit 346425f04d
4 changed files with 22 additions and 0 deletions

View File

@ -83,6 +83,11 @@ if $is_controller{
$rabbitmq_password = $rabbit['password']
$wait_delay = 30
# On controller nodes the log parsing can generate a lot of http_metrics
# which can block heka (idle packs). It was observed that a poolsize set to 200
# solves the issue.
$poolsize = 200
# file { $pre_script:
# ensure => present,
# owner => 'root',
@ -93,6 +98,8 @@ if $is_controller{
# }
} else {
$pre_script = undef
# For other nodes, the poolsize is set to 100 (the Heka default value)
$poolsize = 100
}
class { 'lma_collector':
@ -100,6 +107,7 @@ class { 'lma_collector':
groups => $additional_groups,
pacemaker_managed => $pacemaker_managed,
rabbitmq_resource => $rabbitmq_resource,
poolsize => $poolsize,
}
if $elasticsearch_mode != 'disabled' {

View File

@ -53,6 +53,9 @@
# [*dashboard_port*]
# The listening port for the Heka dashboard (default: 4352).
#
# [*poolsize*]
# The pool size of maximum messages that can exist (default: 100).
#
# [*internal_statistics*]
# Whether or not to dump Heka internal statistics to stdout at a regular
# interval (currently every hour).
@ -84,10 +87,15 @@ class heka (
$max_timer_inject = $heka::params::max_timer_inject,
$dashboard_address = $heka::params::dashboard_address,
$dashboard_port = $heka::params::dashboard_port,
$poolsize = undef,
$pre_script = undef,
$internal_statistics = $heka::params::internal_statistics,
) inherits heka::params {
if $poolsize {
validate_integer($poolsize)
}
$hekad_wrapper = "/usr/local/bin/${service_name}_wrapper"
$base_dir = "/var/cache/${service_name}"
$log_file = "/var/log/${service_name}.log"

View File

@ -13,3 +13,6 @@ max_process_inject = <%= @max_process_inject %>
<% if @max_timer_inject -%>
max_timer_inject = <%= @max_timer_inject %>
<% end -%>
<% if @poolsize -%>
poolsize = <%= @poolsize %>
<% end -%>

View File

@ -33,6 +33,7 @@ class lma_collector (
$tags = $lma_collector::params::tags,
$groups = [],
$pre_script = undef,
$poolsize = 100,
$pacemaker_managed = $lma_collector::params::pacemaker_managed,
$rabbitmq_resource = undef,
$aggregator_address = undef,
@ -41,6 +42,7 @@ class lma_collector (
include heka::params
validate_hash($tags)
validate_integer($poolsize)
$service_name = $lma_collector::params::service_name
$config_dir = $lma_collector::params::config_dir
@ -58,6 +60,7 @@ class lma_collector (
max_message_size => $lma_collector::params::hekad_max_message_size,
max_process_inject => $lma_collector::params::hekad_max_process_inject,
max_timer_inject => $lma_collector::params::hekad_max_timer_inject,
poolsize => $poolsize,
}
if $pacemaker_managed {