diff --git a/oslo_middleware/stats.py b/oslo_middleware/stats.py new file mode 100644 index 0000000..9839fd5 --- /dev/null +++ b/oslo_middleware/stats.py @@ -0,0 +1,131 @@ +# Copyright (c) 2016 Cisco Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import re + +import statsd +import webob.dec + +from oslo_middleware import base + +LOG = logging.getLogger(__name__) +VERSION_REGEX = re.compile("/(v[0-9]{1}\.[0-9]{1})") +UUID_REGEX = re.compile( + '.*(\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}).*a', + re.IGNORECASE) +# UUIDs without the - char, used in some places in Nova URLs. +SHORT_UUID_REGEX = re.compile('.*(\.[0-9a-fA-F]{32}).*') + + +class StatsMiddleware(base.ConfigurableMiddleware): + """Send stats to statsd based on API requests. + + Examines the URL path and request method, and sends a stat count and timer + to a statsd host based on the path/method. + + If your statsd is configured to send stats to Graphite, you'll end up with + stat names of the form:: + + timer..... + + Note that URLs with versions in them (pretty much all of Openstack) + are always processed to replace the dot with _, so for example v2.0 + becomes v2_0, and v1.1 becomes v1_1, since a dot '.' has special + meaning in Graphite. + + The original StatsD is written in nodejs. If you want a Python + implementation, install Bucky instead as it's a drop-in replacement + (and much nicer IMO). + + The Paste config must contain some parameters. Configure a filter like + this:: + + [filter:stats] + paste.filter_factory = oslo_middleware.stats:StatsMiddleware.factory + name = my_application_name # e.g. 'glance' + stats_host = my_statsd_host.example.com + # Optional args to further process the stat name that's generated: + remove_uuid = True + remove_short_uuid = True + # The above uuid processing is required in, e.g. Nova, if you want to + # collect generic stats rather than one per server instance. + """ + + def __init__(self, application, conf): + super(StatsMiddleware, self).__init__(application, conf) + self.application = application + self.stat_name = conf.get('name') + if self.stat_name is None: + raise AttributeError('name must be specified') + self.stats_host = conf.get('stats_host') + if self.stats_host is None: + raise AttributeError('stats_host must be specified') + self.remove_uuid = conf.get('remove_uuid', False) + self.remove_short_uuid = conf.get('remove_short_uuid', False) + self.statsd = statsd.StatsClient(self.stats_host) + + @staticmethod + def strip_short_uuid(path): + """Remove short-form UUID from supplied path. + + Only call after replacing slashes with dots in path. + """ + match = SHORT_UUID_REGEX.match(path) + if match is None: + return path + return path.replace(match.group(1), '') + + @staticmethod + def strip_uuid(path): + """Remove normal-form UUID from supplied path. + + Only call after replacing slashes with dots in path. + """ + match = UUID_REGEX.match(path) + if match is None: + return path + return path.replace(match.group(1), '') + + @staticmethod + def strip_dot_from_version(path): + # Replace vN.N with vNN. + match = VERSION_REGEX.match(path) + if match is None: + return path + return path.replace(match.group(1), match.group(1).replace('.', '')) + + @webob.dec.wsgify + def __call__(self, request): + path = request.path + path = self.strip_dot_from_version(path) + + # Remove leading slash, if any, so we can be sure of the number + # of dots just below. + path = path.lstrip('/') + + stat = "{name}.{method}".format( + name=self.stat_name, method=request.method) + if path != '': + stat += '.' + path.replace('/', '.') + + if self.remove_short_uuid: + stat = self.strip_short_uuid(stat) + + if self.remove_uuid: + stat = self.strip_uuid(stat) + + LOG.debug("Incrementing stat count %s", stat) + with self.statsd.timer(stat): + return request.get_response(self.application) diff --git a/oslo_middleware/tests/test_stats.py b/oslo_middleware/tests/test_stats.py new file mode 100644 index 0000000..1e2e412 --- /dev/null +++ b/oslo_middleware/tests/test_stats.py @@ -0,0 +1,157 @@ +# Copyright (c) 2016 Cisco Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import uuid + +import mock +from oslotest import base as test_base +import statsd +import webob.dec +import webob.exc + +from oslo_middleware import stats + + +class TestStaticMethods(test_base.BaseTestCase): + + def test_removes_uuid(self): + # Generate a long-format UUID (standard form). + id = str(uuid.uuid4()) + path = "foo.{uuid}.bar".format(uuid=id) + stat = stats.StatsMiddleware.strip_uuid(path) + self.assertEqual("foo.bar", stat) + + def test_removes_short_uuid(self): + id = uuid.uuid4().hex + path = "foo.{uuid}.bar".format(uuid=id) + stat = stats.StatsMiddleware.strip_short_uuid(path) + self.assertEqual("foo.bar", stat) + + def test_strips_dots_from_version(self): + # NOTE(bigjools): Good testing practice says to randomise inputs + # that have no meaning to the test. However my reviewer has said + # not to do this, so the versions are static. + path = "/v1.2/foo.bar/bar.foo" + stat = stats.StatsMiddleware.strip_dot_from_version(path) + self.assertEqual("/v12/foo.bar/bar.foo", stat) + + +class TestStatsMiddleware(test_base.BaseTestCase): + + def setUp(self): + super(TestStatsMiddleware, self).setUp() + self.patch(statsd, 'StatsClient', mock.MagicMock()) + + def make_stats_middleware(self, stat_name=None, stats_host=None, + remove_uuid=False, remove_short_uuid=False): + if stat_name is None: + stat_name = uuid.uuid4().hex + if stats_host is None: + stats_host = uuid.uuid4().hex + + conf = dict( + name=stat_name, + stats_host=stats_host, + remove_uuid=remove_uuid, + remove_short_uuid=remove_short_uuid, + ) + + @webob.dec.wsgify + def fake_application(req): + return 'Hello, World' + + return stats.StatsMiddleware(fake_application, conf) + + def get_random_method(self): + # NOTE(bigjools): Good testing practice says to randomise inputs + # that have no meaning to the test. However my reviewer has said + # not to do this, so the methods are static. + return "methodXVNMapyr" + + def perform_request(self, app, path, method): + req = webob.Request.blank(path, method=method) + return req.get_response(app) + + def test_sends_counter_to_statsd(self): + app = self.make_stats_middleware() + random_method = self.get_random_method() + path = '/test/foo/bar' + + self.perform_request(app, path, random_method) + + expected_stat = "{name}.{method}.{path}".format( + name=app.stat_name, method=random_method, + path=path.lstrip('/').replace('/', '.')) + app.statsd.timer.assert_called_once_with(expected_stat) + + def test_strips_uuid_if_configured(self): + app = self.make_stats_middleware(remove_uuid=True) + random_method = self.get_random_method() + random_uuid = str(uuid.uuid4()) + path = '/foo/{uuid}/bar'.format(uuid=random_uuid) + + self.perform_request(app, path, random_method) + + expected_stat = "{name}.{method}.foo.bar".format( + name=app.stat_name, method=random_method) + app.statsd.timer.assert_called_once_with(expected_stat) + + def test_strips_short_uuid_if_configured(self): + app = self.make_stats_middleware(remove_short_uuid=True) + random_method = self.get_random_method() + random_uuid = uuid.uuid4().hex + path = '/foo/{uuid}/bar'.format(uuid=random_uuid) + + self.perform_request(app, path, random_method) + + expected_stat = "{name}.{method}.foo.bar".format( + name=app.stat_name, method=random_method) + app.statsd.timer.assert_called_once_with(expected_stat) + + def test_strips_both_uuid_types_if_configured(self): + app = self.make_stats_middleware( + remove_uuid=True, remove_short_uuid=True) + random_method = self.get_random_method() + random_short_uuid = uuid.uuid4().hex + random_uuid = str(uuid.uuid4()) + path = '/foo/{uuid}/bar/{short_uuid}'.format( + uuid=random_uuid, short_uuid=random_short_uuid) + + self.perform_request(app, path, random_method) + + expected_stat = "{name}.{method}.foo.bar".format( + name=app.stat_name, method=random_method) + app.statsd.timer.assert_called_once_with(expected_stat) + + def test_always_mutates_version_id(self): + app = self.make_stats_middleware() + random_method = self.get_random_method() + path = '/v2.1/foo/bar' + + self.perform_request(app, path, random_method) + + expected_stat = "{name}.{method}.v21.foo.bar".format( + name=app.stat_name, method=random_method) + app.statsd.timer.assert_called_once_with(expected_stat) + + def test_empty_path_has_sane_stat_name(self): + app = self.make_stats_middleware() + random_method = self.get_random_method() + path = '/' + + self.perform_request(app, path, random_method) + + expected_stat = "{name}.{method}".format( + name=app.stat_name, method=random_method) + app.statsd.timer.assert_called_once_with(expected_stat) diff --git a/requirements.txt b/requirements.txt index 06deacb..11fc3e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ six>=1.9.0 # MIT stevedore>=1.17.1 # Apache-2.0 WebOb>=1.6.0 # MIT debtcollector>=1.2.0 # Apache-2.0 +statsd>=3.2.1 # MIT