Configure Nagios to monitor all nodes

All hosts are configured with their management IP with a ping check and
have a basic SSH service checks for all IPs (except for public network).

Note: each Nagios Hosts inherit from a Nagios Host Template to add Custom
Variables per host. These custom variables are used within commands (and
could be used elsewhere if it's useful like notification commands)

implements blueprint lma-infra-alerting-plugin

Change-Id: Ia65b0da45a8dd299406dc434a1dcdb8099c62b25

deployment_scripts/puppet/manifests/nagios.pp

@@ -51,4 +51,70 @@ if $plugin['node_name'] == hiera('user_node_name') {
     notify_recovery => $notify_recovery,
     notify_unknown => $notify_unknown,
   }
+
+  $nodes_hash = hiera('nodes', {})
+  $controller_nodes = hiera('controllers')
+  $compute_nodes = filter_nodes($nodes_hash,'role','compute')
+  $cinder_nodes = filter_nodes($nodes_hash,'role','cinder')
+  $base_os_nodes = filter_nodes($nodes_hash,'role','base-os')
+  $osd_nodes = filter_nodes($nodes_hash, 'role', 'ceph-osd')
+
+  $all_nodes = {}
+  if !empty($controller_nodes){
+    $all_nodes['controller'] = $controller_nodes
+  }
+
+  if !empty($compute_nodes){
+    $all_nodes['compute'] = $compute_nodes
+  }
+  if !empty($cinder_nodes){
+    $all_nodes['cinder'] = $cinder_nodes
+  }
+  if !empty($base_os_nodes){
+    $all_nodes['base-os'] = $base_os_nodes
+  }
+  if !empty($osd_nodes){
+    $all_nodes['ceph-osd'] = $osd_nodes
+  }
+
+  class { 'lma_infra_alerting::nagios::hosts':
+    hosts => $all_nodes,
+    host_name_key => 'name',
+    host_address_key => 'internal_address',
+    host_display_name_keys => ['name', 'user_node_name'],
+    host_custom_vars_keys => ['internal_address', 'private_address',
+                              'public_address', 'storage_address',
+                              'fqdn', 'role'],
+    require  => Class[lma_infra_alerting],
+  }
+
+
+  # Nodes have private IPs only with GRE segmentation
+  $network_config = hiera('quantum_settings')
+  $segmentation_type = $network_config['L2']['segmentation_type']
+  if $segmentation_type == 'gre' {
+    $private_network = true
+  } else {
+    $private_network = false
+  }
+
+  # Configure SSH checks
+  lma_infra_alerting::nagios::check_ssh { 'management':
+    hostgroups => keys($all_nodes),
+    require  => Class[lma_infra_alerting],
+  }
+
+  lma_infra_alerting::nagios::check_ssh { 'storage':
+    hostgroups => keys($all_nodes),
+    custom_var_address => 'storage_address',
+    require  => Class[lma_infra_alerting],
+  }
+
+  if $private_network {
+    lma_infra_alerting::nagios::check_ssh { 'private':
+      hostgroups => keys($all_nodes),
+      custom_var_address => 'private_address',
+      require  => Class[lma_infra_alerting],
+    }
+  }
 }

deployment_scripts/puppet/modules/lma_infra_alerting/lib/puppet/parser/functions/nodes_to_nagios_hostgroups.rb

@@ -0,0 +1,28 @@
+module Puppet::Parser::Functions
+  newfunction(:nodes_to_nagios_hostgroups, :type => :rvalue,  :doc => <<-EOS
+    Return a Hash grouped by role with all attributes matching
+    Nagios_Hostgroup resource properties (for nagios::hostgroup type).
+    {
+     'controller' => {
+       'properties' => {
+         'members' => 'node-1,node-2',
+        },
+      },
+    }
+    EOS
+ ) do |arguments|
+
+    raise(Puppet::ParseError, "nodes_to_nagios_hostgroups(): Wrong number of arguments " +
+      "given (#{arguments.size} for 2") if arguments.size < 2
+
+    hash = arguments[0]
+    raise(Puppet::ParseError, "not a hash!") if ! hash.is_a?(Hash)
+    name_key = arguments[1]
+
+    result = {}
+    hash.each do |group, nodes|
+        result[group] = {'properties' => {'members' => nodes.collect{|x| x[name_key]}.sort().join(',') }}
+    end
+    return result
+  end
+end

deployment_scripts/puppet/modules/lma_infra_alerting/lib/puppet/parser/functions/nodes_to_nagios_hosts.rb

@@ -0,0 +1,46 @@
+module Puppet::Parser::Functions
+  newfunction(:nodes_to_nagios_hosts, :type => :rvalue,  :doc => <<-EOS
+    Return a Hash grouped by host_name with all attributes matching
+    Nagios_Host resource properties (for nagios::host type).
+    {
+     'node-1' => {
+       'properties' => { .. nagios_host properties .. },
+       'custom_vars' => { .. nagios_host custom variables .. },
+     },
+    }
+    EOS
+ ) do |arguments|
+
+    raise(Puppet::ParseError, "nodes_to_nagios_hosts(): Wrong number of arguments " +
+      "given (#{arguments.size} for 4") if arguments.size < 4
+
+    hash = arguments[0]
+    raise(Puppet::ParseError, "not a hash!") if ! hash.is_a?(Hash)
+
+    name_key = arguments[1]
+    ip_key = arguments[2]
+    display_name_keys = arguments[3]
+    custom_vars_keys = arguments[4] or []
+
+    result = {}
+    hash.each do |role, nodes|
+      nodes.each do |value|
+          ip = value[ip_key]
+          name = value[name_key]
+          display_name = display_name_keys.collect{ |x| value[x] }.join('_')
+          if ! result[name] then
+            result[name] = {
+                'properties' => {
+                  'address' => ip,
+                  'hostgroups' => hash.select{|k,v| v.count{|n| n[name_key] == name} > 0 }.keys(),
+                  'display_name' => display_name,
+                  'alias' => display_name,
+                },
+                'custom_vars' => Hash[*custom_vars_keys.collect{|x| ["_#{x}", value[x]]}.flatten()],
+            }
+          end
+      end
+    end
+    return result
+  end
+end

deployment_scripts/puppet/modules/lma_infra_alerting/manifests/nagios/check_ssh.pp

@@ -0,0 +1,65 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# === Resource lma_infra_alerting::nagios::check_ssh
+#
+# Configure SSH checks for per hostgroup
+#
+# === Parameters
+# hostgroup: list of hostgroup names
+# custom_var_address: (optional) the name of the custom variable used for the IP,
+#                     if not defined use the address of the host.
+#
+define lma_infra_alerting::nagios::check_ssh(
+  $hostgroups = [],
+  $custom_var_address = undef,
+){
+
+  include lma_infra_alerting::params
+
+  $prefix = $lma_infra_alerting::params::nagios_config_filename_prefix
+
+  if $custom_var_address {
+    # create custom command check_ssh with param using custom variable Host
+    $var_name = upcase($custom_var_address)
+    $check_command = "check_ssh_${name}"
+    $command = {
+      "check_ssh_${name}" => {
+        properties => {
+          command_line => "${nagios::params::nagios_plugin_dir}/check_ssh '\$_HOST${var_name}\$'",
+        }
+      }
+    }
+    create_resources(nagios::command, $command, {'prefix' => $prefix})
+  } else {
+    $check_command = $lma_infra_alerting::params::nagios_cmd_check_ssh
+  }
+
+  $service_check = {
+    "SSH ${name} network" => {
+      properties => {
+        hostgroup_name => $hostgroups,
+        check_command => $check_command,
+      }
+    }
+  }
+
+  $default_services = {
+    prefix => $prefix,
+    defaults => {
+      'use' => $lma_infra_alerting::params::nagios_generic_service_template,
+    },
+  }
+  create_resources(nagios::service, $service_check, $default_services)
+}

deployment_scripts/puppet/modules/lma_infra_alerting/manifests/nagios/hosts.pp

@@ -0,0 +1,66 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# == Class lma_infra_alerting::nagios::hosts
+#
+# Configure a Nagios hosts and Nagios hostgroups objects.
+#
+# == Parameters
+# hosts: Hash of hosts grouped by hostgroup: { group1 => [{host_hash}, ..]}
+# host_<property>_key: the key of the host hash to use for nagios <property>
+# host_<property>_keys: list of keys used to build <property> by concatenation
+# host_custom_vars_keys: list of keys to define as Custom Variables for the Host
+#
+class lma_infra_alerting::nagios::hosts (
+  $ensure = present,
+  $hosts = [],
+  $host_name_key = undef,
+  $host_address_key = undef,
+  $host_display_name_keys = [],
+  $host_custom_vars_keys = [],
+){
+
+  include lma_infra_alerting::params
+
+  validate_hash($hosts)
+  validate_string($host_name_key, $host_address_key)
+  validate_array($host_display_name_keys, $host_custom_vars_keys)
+
+  $nagios_hosts = nodes_to_nagios_hosts(
+                    $hosts,
+                    $host_name_key,
+                    $host_address_key,
+                    $host_display_name_keys,
+                    $host_custom_vars_keys)
+  $nagios_hostgroups = nodes_to_nagios_hostgroups($hosts, $host_name_key)
+
+  $default = {
+    ensure => $ensure,
+    prefix => $lma_infra_alerting::params::nagios_config_filename_prefix,
+    defaults => {
+      contact_groups => $lma_infra_alerting::params::nagios_contactgroup,
+      active_checks_enabled => 1,
+      passive_checks_enabled => 0,
+      use => $lma_infra_alerting::params::nagios_generic_host_template,
+    }
+  }
+
+  $hg_default = {
+    prefix => $lma_infra_alerting::params::nagios_config_filename_prefix,
+    ensure => $ensure,
+  }
+
+  create_resources(nagios::hostgroup, $nagios_hostgroups, $hg_default)
+  create_resources(nagios::host, $nagios_hosts, $default)
+}

deployment_scripts/puppet/modules/nagios/manifests/host.pp

@@ -29,6 +29,7 @@ define nagios::host (
   $properties = {},
   $defaults = {},
   $ensure = present,
+  $custom_vars = {},
 ){
 
   validate_hash($properties, $defaults)
@@ -59,8 +60,28 @@ define nagios::host (
     $opts['display_name'] = $name
   }
 
+  $final_params = merge($properties, $opts)
+  if ! empty($custom_vars){
+    # overide inheritence
+    $new_use = "custom-vars-${host_name}"
+    if $final_params['use']{
+      $original_use = $final_params['use']
+    }elsif $defaults['use']{
+      $original_use = $defaults['use']
+    }else{
+      $original_use = undef
+    }
+    $final_params['use'] = $new_use
+    nagios::object_custom_vars{ $host_name:
+      object_name => 'host',
+      variables => $custom_vars,
+      use => $original_use,
+      prefix => $prefix,
+    }
+  }
+
   $params = {
-    "${host_name}" => merge($properties, $opts),
+    "${host_name}" => $final_params,
   }
 
   create_resources(nagios_host, $params, $defaults)

deployment_scripts/puppet/modules/nagios/manifests/hostgroup.pp

@@ -0,0 +1,79 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# == Resource: nagios::hostgroup
+#
+# Manage a Nagios hostgroup object
+#
+# == Parameters
+# path: the directory conf.d of nagios
+# prefix: an optional prefix of the filename(s)
+# onefile: all objects are defined in one file if true else one file per service and cmd.
+# properties: properties of the nagios_hostgroup resource.
+
+define nagios::hostgroup (
+  $path = $nagios::params::config_dir,
+  $prefix = '',
+  $onefile = true,
+  $properties = {},
+  $defaults = {},
+  $ensure = present,
+){
+
+  validate_hash($properties, $defaults)
+  $opts = {}
+
+  if is_array($properties['members']){
+    $opts['members'] = join($properties['members'], ',')
+  } elsif $properties['members']{
+    $opts['members'] = $properties['members']
+  }
+
+  if is_array($properties['hostgroup_members']){
+    $opts['hostgroup_members'] = join($properties['hostgroup_members'], ',')
+  }elsif $properties['hostgroup_members']{
+    $opts['hostgroup_members'] = $properties['hostgroup_members']
+  }
+
+  if $onefile {
+    $target = "${path}/${prefix}hostgroups.cfg"
+  }else{
+    $target = "${path}/${prefix}hostgroup_${name}.cfg"
+  }
+  $opts['target'] = $target
+  $opts['notify'] = Class['nagios::server_service']
+  $opts['ensure'] = $ensure
+
+
+  if $properties['hostgroup_name'] == undef {
+    $opts['hostgroup_name'] = $name
+    $hostgroup_name = $name
+  }else{
+    $hostgroup_name = $properties['hostgroup_name']
+  }
+
+  $params = {
+    "${hostgroup_name}" => merge($properties, $opts),
+  }
+  create_resources(nagios_hostgroup, $params, $defaults)
+
+  if ! defined(File[$target]){
+    file { $target:
+      ensure => $ensure,
+      mode => '0644',
+      require => Nagios_Hostgroup[$hostgroup_name],
+      notify => Class['nagios::server_service'],
+    }
+  }
+}

deployment_scripts/puppet/modules/nagios/manifests/object_custom_vars.pp

@@ -0,0 +1,49 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# == Resource: nagios::object_custom_vars
+#
+# Manage a Nagios Object template to set custom variables
+#
+# This is a workaround for the puppet issue
+# https://tickets.puppetlabs.com/browse/PUP-1067
+#
+# == Parameters
+# path: the directory conf.d of nagios
+# prefix: an optional prefix of the filename(s)
+# object_name: one of nagios object: 'host', 'service', ..
+# variable: a Hash of variables (keys must start wih '_')
+# use: (optional) a nagios template object name already existing
+#
+define nagios::object_custom_vars(
+  $path = $nagios::params::config_dir,
+  $prefix = '',
+  $object_name = undef,
+  $variables = {},
+  $use = false,
+  $ensure = present,
+){
+
+  validate_hash($variables)
+  validate_string($object_name)
+  if $object_name == '' {
+    fail('object_name parameter must be set')
+  }
+
+  file { "${path}/${prefix}tpl_${object_name}_${name}_custom_vars.cfg":
+    ensure => $ensure,
+    content => template('nagios/object_custom_vars.erb'),
+    mode => '0644',
+  }
+}

deployment_scripts/puppet/modules/nagios/manifests/params.pp

@@ -19,7 +19,7 @@ class nagios::params {
 
   # plugins
   $nagios_plugin_package = 'nagios-plugins'
-  $nagios_pluigin_dir = '/usr/lib/nagios/plugins'
+  $nagios_plugin_dir = '/usr/lib/nagios/plugins'
 
   # CGI
   $nagios_cgi_package = 'nagios3-cgi'

deployment_scripts/puppet/modules/nagios/templates/object_custom_vars.erb

@@ -0,0 +1,11 @@
+define <%= @object_name %> {
+  register 0
+  name custom-vars-<%= @title %>
+  <% @variables.keys().sort().each do |k| -%>
+  <%- next if @variables[k].nil? or @variables[k] == :undef -%>
+  <%= k %> <%= @variables[k] %>
+  <% end -%>
+  <% if @use -%>
+  use <%= @use %>
+  <% end -%>
+}

pre_build_hook

@@ -54,11 +54,11 @@ download_packages \
     http://mirrors.kernel.org/ubuntu/pool/main/w/whois/whois_5.1.1_amd64.deb
 
 
-rm -rf "${MODULES_DIR:?}"/{stdlib,htpasswd}
-mkdir -p "${MODULES_DIR}"/{stdlib,htpasswd}
+rm -rf "${MODULES_DIR:?}"/{openstack,stdlib,htpasswd}
+mkdir -p "${MODULES_DIR}"/{openstack,stdlib,htpasswd}
 
 wget -qO- "${FUEL_LIB_TARBALL_URL}" | \
     tar -C "${MODULES_DIR}" --strip-components=3 -zxvf - \
-    fuel-library-${FUEL_LIB_VERSION}/deployment/puppet/stdlib
+    fuel-library-${FUEL_LIB_VERSION}/deployment/puppet/{openstack,stdlib}
 
 wget -qO- "${HTPASSWD_TARBALL_URL}" | tar -C "${MODULES_DIR}/htpasswd" --strip-components=1 -xz