From 65393651e26ea4aeac8d80a5d484345c04e5a369 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Thu, 16 Apr 2015 10:25:44 +0200 Subject: [PATCH] Add collectd for CentOS platforms This change adds collectd packages for CentOS. In addition, a few fixes are needed to ensure successful deployment of the LMA collector on CentOS platforms: - Install collectd-dbi package on controller nodes. - Modify Python scripts using subprocess to workaround issues with Python 2.6. - configure a valid database name for the MySQL plugin. - set hasstatus and hasrestart parameters to true for all OpenStack services otherwise Puppet fails to restart them. Change-Id: I06891b069ba6047dfa40bcef7937b52964bda268 --- .../puppet/manifests/controller.pp | 1 + .../files/collectd/rabbitmq_info.py | 17 ++++++++++--- .../lma_collector/manifests/collectd/base.pp | 24 ++++++++++++++----- .../lma_collector/manifests/collectd/dbi.pp | 11 ++++++--- .../lma_collector/manifests/collectd/mysql.pp | 3 ++- .../manifests/notifications/cinder.pp | 2 ++ .../manifests/notifications/compute.pp | 2 ++ .../manifests/notifications/controller.pp | 12 ++++++++++ .../modules/lma_collector/manifests/params.pp | 1 + environment_config.yaml | 21 ++++++++-------- pre_build_hook | 10 +++++++- 11 files changed, 80 insertions(+), 24 deletions(-) diff --git a/deployment_scripts/puppet/manifests/controller.pp b/deployment_scripts/puppet/manifests/controller.pp index d37d0a189..5d4f692aa 100644 --- a/deployment_scripts/puppet/manifests/controller.pp +++ b/deployment_scripts/puppet/manifests/controller.pp @@ -100,6 +100,7 @@ if $lma_collector['influxdb_mode'] != 'disabled' { } class { 'lma_collector::collectd::mysql': + database => 'nova', username => 'nova', password => $nova['db_password'], } diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/rabbitmq_info.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/rabbitmq_info.py index f1e08c3dc..02693ee74 100644 --- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/rabbitmq_info.py +++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/rabbitmq_info.py @@ -19,8 +19,10 @@ # limitations under the License. import collectd -import subprocess import re +import signal +import subprocess +import sys NAME = 'rabbitmq_info' @@ -28,8 +30,6 @@ NAME = 'rabbitmq_info' RABBITMQCTL_BIN = '/usr/sbin/rabbitmqctl' # Override in config by specifying 'PmapBin' PMAP_BIN = '/usr/bin/pmap' -# Override in config by specifying 'PidofBin'. -PIDOF_BIN = '/bin/pidof' # Override in config by specifying 'PidFile. PID_FILE = "/var/run/rabbitmq/pid" # Override in config by specifying 'Vhost'. @@ -42,6 +42,16 @@ CLUSTER_STATUS = re.compile('.*disc,\[([^\]]+)\].*running_nodes,\[([^\]]+)\]', re.S) +def restore_sigchld(): + """ + Restore SIGCHLD handler for python <= v2.6 + It will BREAK the exec plugin!!! + See https://github.com/deniszh/collectd-iostat-python/issues/2 for details + """ + if sys.version_info[0] == 2 and sys.version_info[1] <= 6: + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + + # Obtain the interesting statistical info def get_stats(): stats = {} @@ -206,6 +216,7 @@ def logger(t, msg): # Runtime +collectd.register_init(restore_sigchld) collectd.register_config(configure_callback) collectd.warning('Initialising rabbitmq_info') collectd.register_read(read_callback) diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/base.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/base.pp index f250d902f..8a024e2a8 100644 --- a/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/base.pp +++ b/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/base.pp @@ -30,12 +30,24 @@ class lma_collector::collectd::base { log_file => $lma_collector::params::collectd_logfile, } - class { 'collectd::plugin::write_http': - urls => { - "http://127.0.0.1:${port}" => { - 'format' => 'JSON', - storerates => true - } + $urls = { + "http://127.0.0.1:${port}" => { + 'format' => 'JSON', + storerates => true + } + } + if $::osfamily == 'RedHat' { + # collectd Puppet manifest is broken for RedHat derivatives as it tries to + # install the collectd-write_http package which doesn't exist (for CentOS + # at least) + collectd::plugin {'write_http': + ensure => present, + content => template('collectd/plugin/write_http.conf.erb'), + } + } + else { + class { 'collectd::plugin::write_http': + urls => $urls, } } diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/dbi.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/dbi.pp index cb813dfeb..b58423284 100644 --- a/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/dbi.pp +++ b/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/dbi.pp @@ -16,12 +16,17 @@ class lma_collector::collectd::dbi { include lma_collector::params include lma_collector::collectd::service + if $::osfamily == 'RedHat' { + package { 'collectd-dbi': + ensure => present, + } + } + package { $lma_collector::params::collectd_dbi_package: - ensure => present, - name => $lma_collector::params::collectd_dbi_package, + ensure => present, } collectd::plugin { 'dbi': - require => Package[$lma_collector::params::collectd_dbi_package], + require => Package[$lma_collector::params::collectd_dbi_package], } } diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/mysql.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/mysql.pp index abc64a214..abfd79015 100644 --- a/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/mysql.pp +++ b/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/mysql.pp @@ -13,12 +13,13 @@ # under the License. # class lma_collector::collectd::mysql ( + $database = $lma_collector::params::mysql_database, $username = $lma_collector::params::mysql_username, $password = $lma_collector::params::mysql_password, ) inherits lma_collector::params { include lma_collector::collectd::service - collectd::plugin::mysql::database { 'openstack': + collectd::plugin::mysql::database { $database: host => 'localhost', username => $username, password => $password, diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/notifications/cinder.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/notifications/cinder.pp index dda7fae39..ee03115bd 100644 --- a/deployment_scripts/puppet/modules/lma_collector/manifests/notifications/cinder.pp +++ b/deployment_scripts/puppet/modules/lma_collector/manifests/notifications/cinder.pp @@ -33,5 +33,7 @@ class lma_collector::notifications::cinder ( } service { $::cinder::params::volume_service: + hasstatus => true, + hasrestart => true, } } diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/notifications/compute.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/notifications/compute.pp index 4b10bb0f2..ec00e9ab7 100644 --- a/deployment_scripts/puppet/modules/lma_collector/manifests/notifications/compute.pp +++ b/deployment_scripts/puppet/modules/lma_collector/manifests/notifications/compute.pp @@ -32,5 +32,7 @@ class lma_collector::notifications::compute ( } service { $::nova::params::compute_service_name: + hasstatus => true, + hasrestart => true, } } diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/notifications/controller.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/notifications/controller.pp index 3ae35c72a..ce2e40903 100644 --- a/deployment_scripts/puppet/modules/lma_collector/manifests/notifications/controller.pp +++ b/deployment_scripts/puppet/modules/lma_collector/manifests/notifications/controller.pp @@ -104,6 +104,8 @@ class lma_collector::notifications::controller ( } service { [$::nova::params::api_service_name, $::nova::params::conductor_service_name, $::nova::params::scheduler_service_name]: + hasstatus => true, + hasrestart => true, } # Cinder @@ -119,6 +121,8 @@ class lma_collector::notifications::controller ( } service { [$::cinder::params::api_service, $::cinder::params::scheduler_service]: + hasstatus => true, + hasrestart => true, } # Keystone @@ -134,6 +138,8 @@ class lma_collector::notifications::controller ( } service { $::keystone::params::service_name: + hasstatus => true, + hasrestart => true, } # Neutron @@ -149,6 +155,8 @@ class lma_collector::notifications::controller ( } service { $::neutron::params::server_service: + hasstatus => true, + hasrestart => true, } # Glance @@ -183,6 +191,8 @@ class lma_collector::notifications::controller ( } service { [$::glance::params::api_service_name, $::glance::params::registry_service_name]: + hasstatus => true, + hasrestart => true, } # Heat @@ -198,5 +208,7 @@ class lma_collector::notifications::controller ( } service { [$::heat::params::api_service_name, $::heat::params::engine_service_name]: + hasstatus => true, + hasrestart => true, } } diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/params.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/params.pp index 731eedd02..232549d3d 100644 --- a/deployment_scripts/puppet/modules/lma_collector/manifests/params.pp +++ b/deployment_scripts/puppet/modules/lma_collector/manifests/params.pp @@ -68,6 +68,7 @@ class lma_collector::params { } } $additional_packages = [ 'python-dateutil' ] + $mysql_database = '' $mysql_username = '' $mysql_password = '' $rabbitmq_pid_file = '/var/run/rabbitmq/pid' diff --git a/environment_config.yaml b/environment_config.yaml index c10ed6ff4..c71268849 100644 --- a/environment_config.yaml +++ b/environment_config.yaml @@ -63,11 +63,6 @@ attributes: label: "Local node" - data: "remote" label: "Remote server" -# Hide all InfluxDB parameters for now -# Change action from "hide" to "none" to change this - restrictions: &hide_control - - condition: true - action: "hide" influxdb_node_name: value: 'influxdb' @@ -75,7 +70,9 @@ attributes: description: 'Label of the node running the InfluxDB/Grafana plugin that is deployed in the environment.' weight: 65 type: "text" - restrictions: *hide_control + restrictions: + - condition: "settings:lma_collector.influxdb_mode.value != 'local'" + action: "disable" influxdb_address: value: '' @@ -84,7 +81,9 @@ attributes: weight: 70 type: "text" regex: *node_address_regex - restrictions: *hide_control + restrictions: + - condition: "settings:lma_collector.influxdb_mode.value != 'remote'" + action: "disable" influxdb_database: value: 'lma' @@ -95,7 +94,9 @@ attributes: regex: ¬_empty_parameter source: '\S' error: "Invalid value" - restrictions: *hide_control + restrictions: &disable_influxdb_control + - condition: "settings:lma_collector.influxdb_mode.value == 'disabled'" + action: "disable" influxdb_user: value: 'lma' label: 'InfluxDB user' @@ -103,7 +104,7 @@ attributes: weight: 80 type: "text" regex: *not_empty_parameter - restrictions: *hide_control + restrictions: *disable_influxdb_control influxdb_password: value: 'lmapass' label: 'InfluxDB password' @@ -111,4 +112,4 @@ attributes: weight: 85 type: "password" regex: *not_empty_parameter - restrictions: *hide_control + restrictions: *disable_influxdb_control diff --git a/pre_build_hook b/pre_build_hook index 0f51dc870..ca54c8eee 100755 --- a/pre_build_hook +++ b/pre_build_hook @@ -51,7 +51,15 @@ download_packages \ http://mirrors.kernel.org/ubuntu/pool/main/libd/libdbi/libdbi1_0.9.0-1_amd64.deb \ http://mirrors.kernel.org/ubuntu/pool/main/r/rrdtool/librrd4_1.4.7-2ubuntu5_amd64.deb \ http://mirrors.kernel.org/ubuntu/pool/universe/c/collectd/collectd_5.4.0-3ubuntu2_amd64.deb \ - http://mirrors.kernel.org/ubuntu/pool/main/libd/libdbi-drivers/libdbd-mysql_0.9.0-2ubuntu2_amd64.deb + http://mirrors.kernel.org/ubuntu/pool/main/libd/libdbi-drivers/libdbd-mysql_0.9.0-2ubuntu2_amd64.deb \ + http://mirror.centos.org/centos/6/os/x86_64/Packages/libdbi-0.8.3-4.el6.x86_64.rpm \ + http://mirror.centos.org/centos/6/os/x86_64/Packages/libdbi-drivers-0.8.3-5.1.el6.x86_64.rpm \ + http://mirror.centos.org/centos/6/os/x86_64/Packages/libdbi-dbd-mysql-0.8.3-5.1.el6.x86_64.rpm \ + http://fuel-repository.mirantis.com/fuel-plugins/fuel-plugin-lma-collector/centos/collectd-5.4.1-5.mira1.mira1.x86_64.rpm \ + http://fuel-repository.mirantis.com/fuel-plugins/fuel-plugin-lma-collector/centos/collectd-apache-5.4.1-5.mira1.mira1.x86_64.rpm \ + http://fuel-repository.mirantis.com/fuel-plugins/fuel-plugin-lma-collector/centos/collectd-dbi-5.4.1-5.mira1.mira1.x86_64.rpm \ + http://fuel-repository.mirantis.com/fuel-plugins/fuel-plugin-lma-collector/centos/collectd-mysql-5.4.1-5.mira1.mira1.x86_64.rpm + # Extract dependent manifests from fuel-library rm -rf ${MODULES_DIR}/{cinder,glance,heat,inifile,keystone,neutron,nova,openstack,stdlib,concat}