Improve snmpd element.

The purpose of this patch is to improve the ability to monitor hosts,
correct an error in the current snmpd.conf, and align variable names
with their function.

Renames the Heat variable name prefix from "snmp" to "snmpd" in order to
align the Heat variable name with the element name.

Changes the Heat variable MIB to export_MIB to better represent what
the variable is used for.

Removes a hardcoded user name in snmpd.conf for the rouser to fix
an authentication issue for user names other then "authOnlyUser".

The OoOEnabled command will allow elements like Nagios to poll
hosts with this element via SNMP, and receive a consistent
response.

The CheckProcOpenFiles command is a simple Nagios style check of
open files per process that will be monitored so that an alert can
be sent if any one process exceeds its open file limit.

Change-Id: I92afd1f81dfc47c3bb0f4bb3a5aa72bee8870f48
Authored-by: Chris Krelle <nobodycam@gmail.com>

elements/snmpd/README.md

@@ -1,13 +1,22 @@
 Simple Network Management Protocol (SNMP) daemon.
 
-Installs packaged snmpd, snmp-mibs-downloader, and lm-sensors. Creates a
-read only user with password for snmp v3 authentication.
+Installs packaged snmpd, snmp-mibs-downloader, lm-sensors and
+nagios-plugins-basic.
 
-Enabling the SNMP daemon allows more advanced monitoring of the deployed nodes.
+nagios-plugins-basic is installed to provide a basic frame work for the initial
+check.
 
+Nagios style check for open file limits, per process is provided.
+
+Creates a read only user with password for snmp v3 authentication.
+exposes OoOEnabled and CheckProcOpenFiles via the NET-SNMP-EXTEND-MIB OID
+block.
+
+Grants snmp user password-less sudo access to lsof, so that the per process
+check works correctly.
 
 Options should be provided via heat. For example:
-    snmp:
-        MIB: UCD-SNMP-MIB
+    snmpd:
+        export_MIB: UCD-SNMP-MIB
         readonly_user_name: RoUser
         readonly_user_password: password

elements/snmpd/files/check_proc_open_files.py

@@ -0,0 +1,93 @@
+#! /usr/bin/env python
+# Copyright 2013 Hewlett-Packard Development Company, L.P.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import shlex
+import subprocess
+import sys
+import itertools
+
+warn_limit = 90
+crit_limit = 95
+crit_msg = ''
+warn_msg = ''
+output_msg = ''
+status = {'OK': 0, 'WARNING': 1, 'CRITICAL': 2, 'UNKNOWN': 3}
+
+
+def _get_open_file_limits(file_handle):
+    limit = 0
+    for line in file_handle:
+        if line.startswith('Max open files'):
+            limit = line.split()[3]
+    return limit
+
+
+# Define an in-line generator, no temporary lists
+def _split_lines(lines):
+    for line in lines:
+        yield line[0:19].strip().split()
+
+
+def _get_lsof_pid_list():
+    # get open files limit per PID exclude any line with a TID.
+    lsof_cmd = shlex.split("sudo lsof -n")
+    lsof_proc = subprocess.Popen(lsof_cmd, stdout=subprocess.PIPE)
+    filtered_lines = itertools.ifilterfalse(lambda line: line[1] == 'PID',
+                                            _split_lines(lsof_proc.stdout))
+    filtered_by_len = itertools.ifilter(lambda line: len(line) == 2,
+                                        filtered_lines)
+    for cmd, pid in itertools.groupby(filtered_by_len, lambda line: line[:2]):
+        yield cmd, pid
+
+
+for (pid_cmd, pid_number), count in _get_lsof_pid_list():
+    pid_count = len(list(count))
+    pid_limit_file = "/proc/%s/limits" % pid_number
+    try:
+        with open(pid_limit_file, 'rb') as limit_file_handle:
+            pid_limit = _get_open_file_limits(limit_file_handle)
+    except IOError:
+        # no limit file skip
+        continue
+
+    pid_usage_pct = pid_count * 100 / int(pid_limit)
+    if pid_usage_pct >= crit_limit:
+        crit_msg += ": %s/%s/%s %s" % (pid_number,
+                                       pid_count,
+                                       pid_limit,
+                                       pid_cmd)
+    elif pid_usage_pct >= warn_limit and pid_usage_pct < crit_limit:
+        warn_msg += ": %s/%s/%s %s" % (pid_number,
+                                       pid_count,
+                                       pid_limit,
+                                       pid_cmd)
+
+# if any critical services then exit critical (include warnings)
+if crit_msg:
+    output_msg = "Critical: pid/files/limit Proc%s\n%s" % (crit_msg,
+                                                           warn_msg)
+    exit_code = status['CRITICAL']
+# if not critical and warn not empty exit warning
+if warn_msg:
+    output_msg = "Warning: pid/files/limit Proc%s" % warn_msg
+    exit_code = status['WARNING']
+# All ok here
+if not output_msg:
+    output_msg = "OK - All processes open files with in limits."
+    exit_code = status['OK']
+
+print output_msg
+sys.exit(exit_code)

elements/snmpd/files/snmp

@@ -0,0 +1 @@
+snmp ALL=(root) NOPASSWD: /usr/bin/lsof

elements/snmpd/install.d/67-snmpd

@@ -1,4 +1,7 @@
 #!/bin/bash
 set -eux
 
-install-packages snmpd snmp-mibs-downloader lm-sensors
+install-packages snmpd snmp-mibs-downloader lm-sensors nagios-plugins-basic
+# install our snmp check script
+install -m 0755 -o root -g root $(dirname $0)/../files/check_proc_open_files.py /usr/lib/nagios/plugins/check_proc_open_files.py
+install -m 0440 -o root -g root $(dirname $0)/../files/snmp /etc/sudoers.d/snmp

elements/snmpd/os-apply-config/etc/default/snmpd

@@ -1,6 +1,6 @@
 # This file controls the activity of snmpd and snmptrapd
 # enable a default MIB
-export MIBS={{snmp.MIB}}
+export MIBS={{snmpd.export_MIB}}
 
 # snmpd control (yes means start daemon).
 SNMPDRUN=yes

elements/snmpd/os-apply-config/etc/snmp/snmpd.conf

@@ -1,9 +1,9 @@
 #  Listen for connections on all interfaces (both IPv4 *and* IPv6)
 agentAddress udp:161,udp6:[::1]:161
-createUser {{snmp.readonly_user_name}}  MD5 "{{snmp.readonly_user_password}}"
+createUser {{snmpd.readonly_user_name}}  MD5 "{{snmpd.readonly_user_password}}"
 view   systemonly  included   .1.3.6.1.2.1.1
 view   systemonly  included   .1.3.6.1.2.1.25.1
-rouser   authOnlyUser
+rouser   {{snmpd.readonly_user_name}}
 sysLocation    Sitting on top of the cloud
 sysContact     Me <me@example.org>
 
@@ -27,9 +27,8 @@ defaultMonitors          yes
 linkUpDownNotifications  yes
 
 #  Arbitrary extension commands
-
-#extend    test1   /bin/echo  Hello, cloud!
-#extend-sh test2   echo Hello, cloud ; echo Hi there ; exit 35
+extend  OoOEnabled /bin/echo  Yes
+extend  CheckProcOpenFiles /bin/bash /usr/lib/nagios/plugins/check_proc_open_files.py
 
 #  "Pass-through" MIB extension command
 #pass .1.3.6.1.4.1.8072.2.255  /bin/sh       PREFIX/local/passtest