From 346425f04d20526b9e8c23be9360e5456e79c9bb Mon Sep 17 00:00:00 2001
From: Swann Croiset <scroiset@mirantis.com>
Date: Tue, 5 Apr 2016 09:53:24 +0200
Subject: [PATCH] Increase the Heka poolsize on controllers

On controller nodes, the Heka poolsize must be increased to handle the load
generated by derived metrics from logs otherwise a deadlock
can happen in the filter plugins and block heka.

Fixes-bug: #1557388
(cherry picked from commit 96df47af7385578280a68863b55416da7dd8bfdb)

Conflicts:
	deployment_scripts/puppet/manifests/base.pp
	deployment_scripts/puppet/modules/lma_collector/manifests/init.pp

Change-Id: I74362011d32d413f244c6cdb6e4625ed96759df0
---
 deployment_scripts/puppet/manifests/base.pp               | 8 ++++++++
 deployment_scripts/puppet/modules/heka/manifests/init.pp  | 8 ++++++++
 .../puppet/modules/heka/templates/global.toml.erb         | 3 +++
 .../puppet/modules/lma_collector/manifests/init.pp        | 3 +++
 4 files changed, 22 insertions(+)

diff --git a/deployment_scripts/puppet/manifests/base.pp b/deployment_scripts/puppet/manifests/base.pp
index 5e9b675a1..8adea9780 100644
--- a/deployment_scripts/puppet/manifests/base.pp
+++ b/deployment_scripts/puppet/manifests/base.pp
@@ -83,6 +83,11 @@ if $is_controller{
   $rabbitmq_password = $rabbit['password']
   $wait_delay        = 30
 
+  # On controller nodes the log parsing can generate a lot of http_metrics
+  # which can block heka (idle packs). It was observed that a poolsize set to 200
+  # solves the issue.
+  $poolsize = 200
+
   #  file { $pre_script:
   #    ensure  => present,
   #    owner   => 'root',
@@ -93,6 +98,8 @@ if $is_controller{
   #  }
 } else {
   $pre_script = undef
+  # For other nodes, the poolsize is set to 100 (the Heka default value)
+  $poolsize = 100
 }
 
 class { 'lma_collector':
@@ -100,6 +107,7 @@ class { 'lma_collector':
   groups            => $additional_groups,
   pacemaker_managed => $pacemaker_managed,
   rabbitmq_resource => $rabbitmq_resource,
+  poolsize          => $poolsize,
 }
 
 if $elasticsearch_mode != 'disabled' {
diff --git a/deployment_scripts/puppet/modules/heka/manifests/init.pp b/deployment_scripts/puppet/modules/heka/manifests/init.pp
index 152a15d06..de9a9e028 100644
--- a/deployment_scripts/puppet/modules/heka/manifests/init.pp
+++ b/deployment_scripts/puppet/modules/heka/manifests/init.pp
@@ -53,6 +53,9 @@
 # [*dashboard_port*]
 #   The listening port for the Heka dashboard (default: 4352).
 #
+# [*poolsize*]
+#   The pool size of maximum messages that can exist (default: 100).
+#
 # [*internal_statistics*]
 #   Whether or not to dump Heka internal statistics to stdout at a regular
 #   interval (currently every hour).
@@ -84,10 +87,15 @@ class heka (
   $max_timer_inject = $heka::params::max_timer_inject,
   $dashboard_address = $heka::params::dashboard_address,
   $dashboard_port = $heka::params::dashboard_port,
+  $poolsize = undef,
   $pre_script = undef,
   $internal_statistics = $heka::params::internal_statistics,
 ) inherits heka::params {
 
+  if $poolsize {
+    validate_integer($poolsize)
+  }
+
   $hekad_wrapper = "/usr/local/bin/${service_name}_wrapper"
   $base_dir      = "/var/cache/${service_name}"
   $log_file      = "/var/log/${service_name}.log"
diff --git a/deployment_scripts/puppet/modules/heka/templates/global.toml.erb b/deployment_scripts/puppet/modules/heka/templates/global.toml.erb
index 2ad5cf40e..b09671bc4 100644
--- a/deployment_scripts/puppet/modules/heka/templates/global.toml.erb
+++ b/deployment_scripts/puppet/modules/heka/templates/global.toml.erb
@@ -13,3 +13,6 @@ max_process_inject = <%= @max_process_inject %>
 <% if @max_timer_inject -%>
 max_timer_inject = <%= @max_timer_inject %>
 <% end -%>
+<% if @poolsize -%>
+poolsize = <%= @poolsize %>
+<% end -%>
diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/init.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/init.pp
index a2390d8f1..a861f4fe3 100644
--- a/deployment_scripts/puppet/modules/lma_collector/manifests/init.pp
+++ b/deployment_scripts/puppet/modules/lma_collector/manifests/init.pp
@@ -33,6 +33,7 @@ class lma_collector (
   $tags = $lma_collector::params::tags,
   $groups = [],
   $pre_script = undef,
+  $poolsize = 100,
   $pacemaker_managed = $lma_collector::params::pacemaker_managed,
   $rabbitmq_resource = undef,
   $aggregator_address = undef,
@@ -41,6 +42,7 @@ class lma_collector (
   include heka::params
 
   validate_hash($tags)
+  validate_integer($poolsize)
 
   $service_name = $lma_collector::params::service_name
   $config_dir = $lma_collector::params::config_dir
@@ -58,6 +60,7 @@ class lma_collector (
     max_message_size    => $lma_collector::params::hekad_max_message_size,
     max_process_inject  => $lma_collector::params::hekad_max_process_inject,
     max_timer_inject    => $lma_collector::params::hekad_max_timer_inject,
+    poolsize            => $poolsize,
   }
 
   if $pacemaker_managed {