summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSwann Croiset <scroiset@mirantis.com>2016-04-05 09:53:24 +0200
committerSwann Croiset <scroiset@mirantis.com>2016-04-06 11:05:42 +0200
commit346425f04d20526b9e8c23be9360e5456e79c9bb (patch)
tree32aa7ee41fedc66dcca3874a5e809f8c2eafa559
parent451ab33a69953f2971cf39a75114bd075f43422e (diff)
Increase the Heka poolsize on controllers0.8.1rc3
On controller nodes, the Heka poolsize must be increased to handle the load generated by derived metrics from logs otherwise a deadlock can happen in the filter plugins and block heka. Fixes-bug: #1557388 (cherry picked from commit 96df47af7385578280a68863b55416da7dd8bfdb) Conflicts: deployment_scripts/puppet/manifests/base.pp deployment_scripts/puppet/modules/lma_collector/manifests/init.pp Change-Id: I74362011d32d413f244c6cdb6e4625ed96759df0
Notes
Notes (review): Verified+1: Plugins CI <plugins.ci@mirantis.com> Code-Review+2: Simon Pasquier <spasquier@mirantis.com> Workflow+1: Swann Croiset <scroiset@mirantis.com> Verified+2: Jenkins Submitted-by: Jenkins Submitted-at: Wed, 06 Apr 2016 09:11:32 +0000 Reviewed-on: https://review.openstack.org/301821 Project: openstack/fuel-plugin-lma-collector Branch: refs/heads/stable/0.8
-rw-r--r--deployment_scripts/puppet/manifests/base.pp8
-rw-r--r--deployment_scripts/puppet/modules/heka/manifests/init.pp8
-rw-r--r--deployment_scripts/puppet/modules/heka/templates/global.toml.erb3
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/manifests/init.pp3
4 files changed, 22 insertions, 0 deletions
diff --git a/deployment_scripts/puppet/manifests/base.pp b/deployment_scripts/puppet/manifests/base.pp
index 5e9b675..8adea97 100644
--- a/deployment_scripts/puppet/manifests/base.pp
+++ b/deployment_scripts/puppet/manifests/base.pp
@@ -83,6 +83,11 @@ if $is_controller{
83 $rabbitmq_password = $rabbit['password'] 83 $rabbitmq_password = $rabbit['password']
84 $wait_delay = 30 84 $wait_delay = 30
85 85
86 # On controller nodes the log parsing can generate a lot of http_metrics
87 # which can block heka (idle packs). It was observed that a poolsize set to 200
88 # solves the issue.
89 $poolsize = 200
90
86 # file { $pre_script: 91 # file { $pre_script:
87 # ensure => present, 92 # ensure => present,
88 # owner => 'root', 93 # owner => 'root',
@@ -93,6 +98,8 @@ if $is_controller{
93 # } 98 # }
94} else { 99} else {
95 $pre_script = undef 100 $pre_script = undef
101 # For other nodes, the poolsize is set to 100 (the Heka default value)
102 $poolsize = 100
96} 103}
97 104
98class { 'lma_collector': 105class { 'lma_collector':
@@ -100,6 +107,7 @@ class { 'lma_collector':
100 groups => $additional_groups, 107 groups => $additional_groups,
101 pacemaker_managed => $pacemaker_managed, 108 pacemaker_managed => $pacemaker_managed,
102 rabbitmq_resource => $rabbitmq_resource, 109 rabbitmq_resource => $rabbitmq_resource,
110 poolsize => $poolsize,
103} 111}
104 112
105if $elasticsearch_mode != 'disabled' { 113if $elasticsearch_mode != 'disabled' {
diff --git a/deployment_scripts/puppet/modules/heka/manifests/init.pp b/deployment_scripts/puppet/modules/heka/manifests/init.pp
index 152a15d..de9a9e0 100644
--- a/deployment_scripts/puppet/modules/heka/manifests/init.pp
+++ b/deployment_scripts/puppet/modules/heka/manifests/init.pp
@@ -53,6 +53,9 @@
53# [*dashboard_port*] 53# [*dashboard_port*]
54# The listening port for the Heka dashboard (default: 4352). 54# The listening port for the Heka dashboard (default: 4352).
55# 55#
56# [*poolsize*]
57# The pool size of maximum messages that can exist (default: 100).
58#
56# [*internal_statistics*] 59# [*internal_statistics*]
57# Whether or not to dump Heka internal statistics to stdout at a regular 60# Whether or not to dump Heka internal statistics to stdout at a regular
58# interval (currently every hour). 61# interval (currently every hour).
@@ -84,10 +87,15 @@ class heka (
84 $max_timer_inject = $heka::params::max_timer_inject, 87 $max_timer_inject = $heka::params::max_timer_inject,
85 $dashboard_address = $heka::params::dashboard_address, 88 $dashboard_address = $heka::params::dashboard_address,
86 $dashboard_port = $heka::params::dashboard_port, 89 $dashboard_port = $heka::params::dashboard_port,
90 $poolsize = undef,
87 $pre_script = undef, 91 $pre_script = undef,
88 $internal_statistics = $heka::params::internal_statistics, 92 $internal_statistics = $heka::params::internal_statistics,
89) inherits heka::params { 93) inherits heka::params {
90 94
95 if $poolsize {
96 validate_integer($poolsize)
97 }
98
91 $hekad_wrapper = "/usr/local/bin/${service_name}_wrapper" 99 $hekad_wrapper = "/usr/local/bin/${service_name}_wrapper"
92 $base_dir = "/var/cache/${service_name}" 100 $base_dir = "/var/cache/${service_name}"
93 $log_file = "/var/log/${service_name}.log" 101 $log_file = "/var/log/${service_name}.log"
diff --git a/deployment_scripts/puppet/modules/heka/templates/global.toml.erb b/deployment_scripts/puppet/modules/heka/templates/global.toml.erb
index 2ad5cf4..b09671b 100644
--- a/deployment_scripts/puppet/modules/heka/templates/global.toml.erb
+++ b/deployment_scripts/puppet/modules/heka/templates/global.toml.erb
@@ -13,3 +13,6 @@ max_process_inject = <%= @max_process_inject %>
13<% if @max_timer_inject -%> 13<% if @max_timer_inject -%>
14max_timer_inject = <%= @max_timer_inject %> 14max_timer_inject = <%= @max_timer_inject %>
15<% end -%> 15<% end -%>
16<% if @poolsize -%>
17poolsize = <%= @poolsize %>
18<% end -%>
diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/init.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/init.pp
index a2390d8..a861f4f 100644
--- a/deployment_scripts/puppet/modules/lma_collector/manifests/init.pp
+++ b/deployment_scripts/puppet/modules/lma_collector/manifests/init.pp
@@ -33,6 +33,7 @@ class lma_collector (
33 $tags = $lma_collector::params::tags, 33 $tags = $lma_collector::params::tags,
34 $groups = [], 34 $groups = [],
35 $pre_script = undef, 35 $pre_script = undef,
36 $poolsize = 100,
36 $pacemaker_managed = $lma_collector::params::pacemaker_managed, 37 $pacemaker_managed = $lma_collector::params::pacemaker_managed,
37 $rabbitmq_resource = undef, 38 $rabbitmq_resource = undef,
38 $aggregator_address = undef, 39 $aggregator_address = undef,
@@ -41,6 +42,7 @@ class lma_collector (
41 include heka::params 42 include heka::params
42 43
43 validate_hash($tags) 44 validate_hash($tags)
45 validate_integer($poolsize)
44 46
45 $service_name = $lma_collector::params::service_name 47 $service_name = $lma_collector::params::service_name
46 $config_dir = $lma_collector::params::config_dir 48 $config_dir = $lma_collector::params::config_dir
@@ -58,6 +60,7 @@ class lma_collector (
58 max_message_size => $lma_collector::params::hekad_max_message_size, 60 max_message_size => $lma_collector::params::hekad_max_message_size,
59 max_process_inject => $lma_collector::params::hekad_max_process_inject, 61 max_process_inject => $lma_collector::params::hekad_max_process_inject,
60 max_timer_inject => $lma_collector::params::hekad_max_timer_inject, 62 max_timer_inject => $lma_collector::params::hekad_max_timer_inject,
63 poolsize => $poolsize,
61 } 64 }
62 65
63 if $pacemaker_managed { 66 if $pacemaker_managed {