From 719425d9f217798bbf52b2e3dcb5529101c6c0b8 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Fri, 1 Jun 2018 14:25:22 +1000 Subject: [PATCH] Add statsd reporter and test --- .stestr.conf | 2 +- afsmon/__init__.py | 18 +++--- afsmon/cmd/main.py | 123 +++++++++++++++++++++++++++++------- afsmon/tests/base.py | 102 +++++++++++++++++++++++++++++- afsmon/tests/test_afsmon.py | 53 ++++++++++++++-- requirements.txt | 1 + 6 files changed, 262 insertions(+), 37 deletions(-) diff --git a/.stestr.conf b/.stestr.conf index 3f11fb0..d3d12d0 100644 --- a/.stestr.conf +++ b/.stestr.conf @@ -1,2 +1,2 @@ [DEFAULT] -test_path=./pyafsmon/tests \ No newline at end of file +test_path=./afsmon/tests \ No newline at end of file diff --git a/afsmon/__init__.py b/afsmon/__init__.py index 21fdb5b..924b53f 100644 --- a/afsmon/__init__.py +++ b/afsmon/__init__.py @@ -22,6 +22,8 @@ from datetime import datetime from enum import Enum from prettytable import PrettyTable +logger = logging.getLogger("afsmon") + # # Fileserver # @@ -66,7 +68,7 @@ class FileServerStats: def _get_volumes(self): cmd = ["vos", "listvol", "-long", "-server", self.hostname] - logging.debug("Running: %s" % cmd) + logger.debug("Running: %s" % cmd) output = subprocess.check_output( cmd, stderr=subprocess.STDOUT).decode('ascii') @@ -104,7 +106,7 @@ class FileServerStats: def _get_calls_waiting(self): cmd = ["rxdebug", self.hostname, "7000", "-rxstats", "-noconns"] - logging.debug("Running: %s" % cmd) + logger.debug("Running: %s" % cmd) output = subprocess.check_output( cmd, stderr=subprocess.STDOUT).decode('ascii') @@ -118,7 +120,7 @@ class FileServerStats: def _get_partition_stats(self): cmd = ["vos", "partinfo", self.hostname, "-noauth"] - logging.debug("Running: %s" % cmd) + logger.debug("Running: %s" % cmd) output = subprocess.check_output( cmd, stderr=subprocess.STDOUT).decode('ascii') @@ -137,12 +139,12 @@ class FileServerStats: def _get_fs_stats(self): cmd = ["bos", "status", self.hostname, "-long", "-noauth"] - logging.debug("Running: %s" % cmd) + logger.debug("Running: %s" % cmd) try: output = subprocess.check_output( cmd, stderr=subprocess.STDOUT).decode('ascii') except subprocess.CalledProcessError: - logging.debug(" ... failed!") + logger.debug(" ... failed!") self.status = FileServerStatus.NO_CONNECTION return @@ -159,7 +161,7 @@ class FileServerStats: elif re.search('disabled, currently shutdown', output): self.status = FileServerStatus.DISABLED else: - logging.debug(output) + logger.debug(output) self.status = FileServerStatus.UNKNOWN def get_stats(self): @@ -223,12 +225,12 @@ def get_fs_addresses(cell): ''' fs = [] cmd = ["vos", "listaddrs", "-noauth", "-cell", cell] - logging.debug("Running: %s" % cmd) + logger.debug("Running: %s" % cmd) try: output = subprocess.check_output( cmd, stderr=subprocess.STDOUT).decode('ascii') except subprocess.CalledProcessError: - logging.debug(" ... failed!") + logger.debug(" ... failed!") return [] for line in output.split('\n'): diff --git a/afsmon/cmd/main.py b/afsmon/cmd/main.py index cd8c91a..b13cfd6 100644 --- a/afsmon/cmd/main.py +++ b/afsmon/cmd/main.py @@ -14,40 +14,119 @@ import argparse import configparser import logging +import os import sys +import statsd import afsmon -def main(args=None): +logger = logging.getLogger("afsmon.main") - if args is None: - args = sys.argv[1:] +class AFSMonCmd: - parser = argparse.ArgumentParser( - description='An AFS monitoring tool') + def cmd_show(self): + for fs in self.fileservers: + print(fs) + return 0 - parser.add_argument("config", help="Path to config file") - parser.add_argument("-d", '--debug', action="store_true") + def cmd_statsd(self): + # note we're just being careful to let the default values fall + # through to StatsClient() + statsd_args = {} + try: + try: + statsd_args['host'] = self.config.get('statsd', 'host') + except configparser.NoOptionError: + pass + try: + statsd_args['port'] = self.config.get('statsd', 'port') + except configparser.NoOptionerror: + pass + except configparser.NoSectionError: + pass + if os.getenv('STATSD_HOST', None): + statsd_args['host'] = os.environ['STATSD_HOST'] + if os.getenv('STATSD_PORT', None): + statsd_args['port'] = os.environ['STATSD_PORT'] + logger.debug("Sending stats to %s:%s" % (statsd_args['host'], + statsd_args['port'])) + self.statsd = statsd.StatsClient(**statsd_args) - args = parser.parse_args(args) + for f in self.fileservers: + if f.status != afsmon.FileServerStatus.NORMAL: + continue - if args.debug: - logging.basicConfig(level=logging.DEBUG) - logging.debug("Debugging enabled") + hn = f.hostname.replace('.', '_') + self.statsd.gauge('afs.%s.idle_threads' % hn, f.idle_threads) + self.statsd.gauge('afs.%s.calls_waiting'% hn, f.calls_waiting) + for p in f.partitions: + self.statsd.gauge( + 'afs.%s.part.%s.used' % (hn, p.partition), p.used) + self.statsd.gauge( + 'afs.%s.part.%s.free' % (hn, p.partition), p.free) + self.statsd.gauge( + 'afs.%s.part.%s.total' % (hn, p.partition), p.total) + for v in f.volumes: + if v.perms != 'RW': + continue + vn = v.volume.replace('.', '_') + self.statsd.gauge( + 'afs.%s.vol.%s.used' % (hn, vn), v.used) + self.statsd.gauge( + 'afs.%s.vol.%s.quota' % (hn, vn), v.quota) - config = configparser.RawConfigParser() - config.read(args.config) - cell = config.get('main', 'cell').strip() + def main(self, args=None): + if args is None: + args = sys.argv[1:] - fileservers = afsmon.get_fs_addresses(cell) - logging.debug("Found fileservers: %s" % ", ".join(fileservers)) + self.fileservers = [] - for fileserver in fileservers: - logging.debug("Finding stats for: %s" % fileserver) + parser = argparse.ArgumentParser( + description='An AFS monitoring tool') - fs = afsmon.FileServerStats(fileserver) - fs.get_stats() - print(fs) + parser.add_argument("-c", "--config", action='store', + default="/etc/afsmon.cfg", + help="Path to config file") + parser.add_argument("-d", '--debug', action="store_true") - sys.exit(0) + subparsers = parser.add_subparsers(title='commands', + description='valid commands', + dest='command') + + cmd_show = subparsers.add_parser('show', help='show table of results') + cmd_show.set_defaults(func=self.cmd_show) + + cmd_statsd = subparsers.add_parser('statsd', help='report to statsd') + cmd_statsd.set_defaults(func=self.cmd_statsd) + + self.args = parser.parse_args(args) + + if self.args.debug: + logging.basicConfig(level=logging.DEBUG) + logger.debug("Debugging enabled") + + if not os.path.exists(self.args.config): + raise ValueError("Config file %s does not exist" % self.args.config) + + self.config = configparser.RawConfigParser() + self.config.read(self.args.config) + + cell = self.config.get('main', 'cell').strip() + + fs_addrs = afsmon.get_fs_addresses(cell) + logger.debug("Found fileservers: %s" % ", ".join(fs_addrs)) + + for addr in fs_addrs: + logger.debug("Finding stats for: %s" % addr) + fs = afsmon.FileServerStats(addr) + fs.get_stats() + self.fileservers.append(fs) + + # run the subcommand + return self.args.func() + + +def main(): + cmd = AFSMonCmd() + return cmd.main() diff --git a/afsmon/tests/base.py b/afsmon/tests/base.py index 8b4427d..3cf15c3 100644 --- a/afsmon/tests/base.py +++ b/afsmon/tests/base.py @@ -15,11 +15,46 @@ import os +import logging import fixtures +import select +import socket import testtools +import threading +import time _TRUE_VALUES = ('True', 'true', '1', 'yes') +logger = logging.getLogger("afsmon.tests.base") + +class FakeStatsd(threading.Thread): + def __init__(self): + threading.Thread.__init__(self) + self.daemon = True + self.sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + self.sock.bind(('', 0)) + self.port = self.sock.getsockname()[1] + self.wake_read, self.wake_write = os.pipe() + self.stats = [] + + def run(self): + while True: + poll = select.poll() + poll.register(self.sock, select.POLLIN) + poll.register(self.wake_read, select.POLLIN) + ret = poll.poll() + for (fd, event) in ret: + if fd == self.sock.fileno(): + data = self.sock.recvfrom(1024) + if not data: + return + self.stats.append(data[0]) + if fd == self.wake_read: + return + + def stop(self): + os.write(self.wake_write, b'1\n') + class TestCase(testtools.TestCase): """Test case base class for all unit tests.""" @@ -47,4 +82,69 @@ class TestCase(testtools.TestCase): stderr = self.useFixture(fixtures.StringStream('stderr')).stream self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr)) - self.log_fixture = self.useFixture(fixtures.FakeLogger()) + self.log_fixture = self.useFixture( + fixtures.FakeLogger(level=logging.DEBUG)) + + self.statsd = FakeStatsd() + os.environ['STATSD_HOST'] = '127.0.0.1' + os.environ['STATSD_PORT'] = str(self.statsd.port) + self.statsd.start() + + def shutdown(self): + self.statsd.stop() + self.statsd.join() + + def assertReportedStat(self, key, value=None, kind=None): + """Check statsd output + + Check statsd return values. A ``value`` should specify a + ``kind``, however a ``kind`` may be specified without a + ``value`` for a generic match. Leave both empy to just check + for key presence. + + :arg str key: The statsd key + :arg str value: The expected value of the metric ``key`` + :arg str kind: The expected type of the metric ``key`` For example + + - ``c`` counter + - ``g`` gauge + - ``ms`` timing + - ``s`` set + """ + if value: + self.assertNotEqual(kind, None) + + start = time.time() + while time.time() < (start + 5): + # Note our fake statsd just queues up results in a queue. + # We just keep going through them until we find one that + # matches, or fail out. + for stat in self.statsd.stats: + k, v = stat.decode('utf-8').split(':') + if key == k: + if kind is None: + # key with no qualifiers is found + return True + + s_value, s_kind = v.split('|') + # if no kind match, look for other keys + if kind != s_kind: + continue + + if value: + # special-case value|ms because statsd can turn + # timing results into float of indeterminate + # length, hence foiling string matching. + if kind == 'ms': + if float(value) == float(s_value): + return True + if value == s_value: + return True + # otherwise keep looking for other matches + continue + + # this key matches + return True + time.sleep(0.1) + + raise Exception("Key %s not found in reported stats" % key) diff --git a/afsmon/tests/test_afsmon.py b/afsmon/tests/test_afsmon.py index 8388d35..753fa5e 100644 --- a/afsmon/tests/test_afsmon.py +++ b/afsmon/tests/test_afsmon.py @@ -9,14 +9,17 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import afsmon +import configparser -from pyafsmon.tests import base +from afsmon.tests import base +from afsmon.cmd.main import AFSMonCmd """ -test_pyafsmon +test_afsmon ---------------------------------- -Tests for `pyafsmon` module. +Tests for `afsmon` module. """ class TestPyAFSMon(base.TestCase): @@ -24,5 +27,45 @@ class TestPyAFSMon(base.TestCase): def setUp(self): super(TestPyAFSMon, self).setUp() - def test_blank(self): - self.assertEqual(0, 0) + def test_statsd(self): + cmd = AFSMonCmd() + cmd.config = configparser.ConfigParser() + + a = afsmon.FileServerStats('afs01.dfw.openstack.org') + a.status = afsmon.FileServerStatus.NORMAL + a.idle_threads = 250 + a.calls_waiting = 0 + a.partitions = [afsmon.Partition('vicepa', 512, 512, 1024, 50.00)] + a.volumes = [ + afsmon.Volume('mirror.foo', 12345678, 'RW', 512, 1024, 50.00), + afsmon.Volume('mirror.moo', 87654321, 'RW', 1024, 2048, 50.00), + ] + + b = afsmon.FileServerStats('afs02.ord.openstack.org') + b.status = afsmon.FileServerStatus.NORMAL + b.idle_threads = 100 + b.calls_waiting = 2 + b.partitions = [afsmon.Partition('vicepa', 512, 512, 1024, 50.00)] + b.volumes = [] + + cmd.fileservers = [a, b] + + cmd.cmd_statsd() + + self.assertReportedStat( + 'afs.afs01_dfw_openstack_org.idle_threads', value='250', kind='g') + self.assertReportedStat( + 'afs.afs02_ord_openstack_org.calls_waiting', value='2', kind='g') + self.assertReportedStat( + 'afs.afs01_dfw_openstack_org.part.vicepa.used', + value='512', kind='g') + self.assertReportedStat( + 'afs.afs01_dfw_openstack_org.part.vicepa.total', + value='1024', kind='g') + self.assertReportedStat( + 'afs.afs01_dfw_openstack_org.vol.mirror_moo.used', + value='1024', kind='g') + self.assertReportedStat( + 'afs.afs01_dfw_openstack_org.vol.mirror_moo.quota', + value='2048', kind='g') + diff --git a/requirements.txt b/requirements.txt index 2d41b89..d7dccb8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ pbr!=2.1.0,>=2.0.0 # Apache-2.0 Babel!=2.4.0,>=2.3.4 # BSD PrettyTable<0.8 # BSD +statsd>=3.2.1 # MIT