Add new middleware to send API data to statsd
The new StatsMiddleware is a Paste filter that examines the URL path and request method, and sends a stat count and a timer to a statsd host whose name is based on the path/method. If your statsd is configured to send stats to Graphite, you'll end up with stat names of the form: timer.<appname>.<METHOD>.<path>.<from>.<url> Because a dot has special meaning in Graphite, dots in API versions that appear in the path will be replaced with _, so for example v2.1 becomes v2_1, and v1.0 becomes v1_0. Change-Id: Ieaffeded1bf81c0782d88f49b6f5209f11744899
This commit is contained in:
Julian Edwards
parent
5b5acf6596
commit
032addde67
|
|
@ -0,0 +1,131 @@
|
|||
# Copyright (c) 2016 Cisco Systems
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
import statsd
|
||||
import webob.dec
|
||||
|
||||
from oslo_middleware import base
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
VERSION_REGEX = re.compile("/(v[0-9]{1}\.[0-9]{1})")
|
||||
UUID_REGEX = re.compile(
|
||||
'.*(\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}).*a',
|
||||
re.IGNORECASE)
|
||||
# UUIDs without the - char, used in some places in Nova URLs.
|
||||
SHORT_UUID_REGEX = re.compile('.*(\.[0-9a-fA-F]{32}).*')
|
||||
|
||||
|
||||
class StatsMiddleware(base.ConfigurableMiddleware):
|
||||
"""Send stats to statsd based on API requests.
|
||||
|
||||
Examines the URL path and request method, and sends a stat count and timer
|
||||
to a statsd host based on the path/method.
|
||||
|
||||
If your statsd is configured to send stats to Graphite, you'll end up with
|
||||
stat names of the form::
|
||||
|
||||
timer.<appname>.<METHOD>.<path>.<from>.<url>
|
||||
|
||||
Note that URLs with versions in them (pretty much all of Openstack)
|
||||
are always processed to replace the dot with _, so for example v2.0
|
||||
becomes v2_0, and v1.1 becomes v1_1, since a dot '.' has special
|
||||
meaning in Graphite.
|
||||
|
||||
The original StatsD is written in nodejs. If you want a Python
|
||||
implementation, install Bucky instead as it's a drop-in replacement
|
||||
(and much nicer IMO).
|
||||
|
||||
The Paste config must contain some parameters. Configure a filter like
|
||||
this::
|
||||
|
||||
[filter:stats]
|
||||
paste.filter_factory = oslo_middleware.stats:StatsMiddleware.factory
|
||||
name = my_application_name # e.g. 'glance'
|
||||
stats_host = my_statsd_host.example.com
|
||||
# Optional args to further process the stat name that's generated:
|
||||
remove_uuid = True
|
||||
remove_short_uuid = True
|
||||
# The above uuid processing is required in, e.g. Nova, if you want to
|
||||
# collect generic stats rather than one per server instance.
|
||||
"""
|
||||
|
||||
def __init__(self, application, conf):
|
||||
super(StatsMiddleware, self).__init__(application, conf)
|
||||
self.application = application
|
||||
self.stat_name = conf.get('name')
|
||||
if self.stat_name is None:
|
||||
raise AttributeError('name must be specified')
|
||||
self.stats_host = conf.get('stats_host')
|
||||
if self.stats_host is None:
|
||||
raise AttributeError('stats_host must be specified')
|
||||
self.remove_uuid = conf.get('remove_uuid', False)
|
||||
self.remove_short_uuid = conf.get('remove_short_uuid', False)
|
||||
self.statsd = statsd.StatsClient(self.stats_host)
|
||||
|
||||
@staticmethod
|
||||
def strip_short_uuid(path):
|
||||
"""Remove short-form UUID from supplied path.
|
||||
|
||||
Only call after replacing slashes with dots in path.
|
||||
"""
|
||||
match = SHORT_UUID_REGEX.match(path)
|
||||
if match is None:
|
||||
return path
|
||||
return path.replace(match.group(1), '')
|
||||
|
||||
@staticmethod
|
||||
def strip_uuid(path):
|
||||
"""Remove normal-form UUID from supplied path.
|
||||
|
||||
Only call after replacing slashes with dots in path.
|
||||
"""
|
||||
match = UUID_REGEX.match(path)
|
||||
if match is None:
|
||||
return path
|
||||
return path.replace(match.group(1), '')
|
||||
|
||||
@staticmethod
|
||||
def strip_dot_from_version(path):
|
||||
# Replace vN.N with vNN.
|
||||
match = VERSION_REGEX.match(path)
|
||||
if match is None:
|
||||
return path
|
||||
return path.replace(match.group(1), match.group(1).replace('.', ''))
|
||||
|
||||
@webob.dec.wsgify
|
||||
def __call__(self, request):
|
||||
path = request.path
|
||||
path = self.strip_dot_from_version(path)
|
||||
|
||||
# Remove leading slash, if any, so we can be sure of the number
|
||||
# of dots just below.
|
||||
path = path.lstrip('/')
|
||||
|
||||
stat = "{name}.{method}".format(
|
||||
name=self.stat_name, method=request.method)
|
||||
if path != '':
|
||||
stat += '.' + path.replace('/', '.')
|
||||
|
||||
if self.remove_short_uuid:
|
||||
stat = self.strip_short_uuid(stat)
|
||||
|
||||
if self.remove_uuid:
|
||||
stat = self.strip_uuid(stat)
|
||||
|
||||
LOG.debug("Incrementing stat count %s", stat)
|
||||
with self.statsd.timer(stat):
|
||||
return request.get_response(self.application)
|
||||
|
|
@ -0,0 +1,157 @@
|
|||
# Copyright (c) 2016 Cisco Systems
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import uuid
|
||||
|
||||
import mock
|
||||
from oslotest import base as test_base
|
||||
import statsd
|
||||
import webob.dec
|
||||
import webob.exc
|
||||
|
||||
from oslo_middleware import stats
|
||||
|
||||
|
||||
class TestStaticMethods(test_base.BaseTestCase):
|
||||
|
||||
def test_removes_uuid(self):
|
||||
# Generate a long-format UUID (standard form).
|
||||
id = str(uuid.uuid4())
|
||||
path = "foo.{uuid}.bar".format(uuid=id)
|
||||
stat = stats.StatsMiddleware.strip_uuid(path)
|
||||
self.assertEqual("foo.bar", stat)
|
||||
|
||||
def test_removes_short_uuid(self):
|
||||
id = uuid.uuid4().hex
|
||||
path = "foo.{uuid}.bar".format(uuid=id)
|
||||
stat = stats.StatsMiddleware.strip_short_uuid(path)
|
||||
self.assertEqual("foo.bar", stat)
|
||||
|
||||
def test_strips_dots_from_version(self):
|
||||
# NOTE(bigjools): Good testing practice says to randomise inputs
|
||||
# that have no meaning to the test. However my reviewer has said
|
||||
# not to do this, so the versions are static.
|
||||
path = "/v1.2/foo.bar/bar.foo"
|
||||
stat = stats.StatsMiddleware.strip_dot_from_version(path)
|
||||
self.assertEqual("/v12/foo.bar/bar.foo", stat)
|
||||
|
||||
|
||||
class TestStatsMiddleware(test_base.BaseTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestStatsMiddleware, self).setUp()
|
||||
self.patch(statsd, 'StatsClient', mock.MagicMock())
|
||||
|
||||
def make_stats_middleware(self, stat_name=None, stats_host=None,
|
||||
remove_uuid=False, remove_short_uuid=False):
|
||||
if stat_name is None:
|
||||
stat_name = uuid.uuid4().hex
|
||||
if stats_host is None:
|
||||
stats_host = uuid.uuid4().hex
|
||||
|
||||
conf = dict(
|
||||
name=stat_name,
|
||||
stats_host=stats_host,
|
||||
remove_uuid=remove_uuid,
|
||||
remove_short_uuid=remove_short_uuid,
|
||||
)
|
||||
|
||||
@webob.dec.wsgify
|
||||
def fake_application(req):
|
||||
return 'Hello, World'
|
||||
|
||||
return stats.StatsMiddleware(fake_application, conf)
|
||||
|
||||
def get_random_method(self):
|
||||
# NOTE(bigjools): Good testing practice says to randomise inputs
|
||||
# that have no meaning to the test. However my reviewer has said
|
||||
# not to do this, so the methods are static.
|
||||
return "methodXVNMapyr"
|
||||
|
||||
def perform_request(self, app, path, method):
|
||||
req = webob.Request.blank(path, method=method)
|
||||
return req.get_response(app)
|
||||
|
||||
def test_sends_counter_to_statsd(self):
|
||||
app = self.make_stats_middleware()
|
||||
random_method = self.get_random_method()
|
||||
path = '/test/foo/bar'
|
||||
|
||||
self.perform_request(app, path, random_method)
|
||||
|
||||
expected_stat = "{name}.{method}.{path}".format(
|
||||
name=app.stat_name, method=random_method,
|
||||
path=path.lstrip('/').replace('/', '.'))
|
||||
app.statsd.timer.assert_called_once_with(expected_stat)
|
||||
|
||||
def test_strips_uuid_if_configured(self):
|
||||
app = self.make_stats_middleware(remove_uuid=True)
|
||||
random_method = self.get_random_method()
|
||||
random_uuid = str(uuid.uuid4())
|
||||
path = '/foo/{uuid}/bar'.format(uuid=random_uuid)
|
||||
|
||||
self.perform_request(app, path, random_method)
|
||||
|
||||
expected_stat = "{name}.{method}.foo.bar".format(
|
||||
name=app.stat_name, method=random_method)
|
||||
app.statsd.timer.assert_called_once_with(expected_stat)
|
||||
|
||||
def test_strips_short_uuid_if_configured(self):
|
||||
app = self.make_stats_middleware(remove_short_uuid=True)
|
||||
random_method = self.get_random_method()
|
||||
random_uuid = uuid.uuid4().hex
|
||||
path = '/foo/{uuid}/bar'.format(uuid=random_uuid)
|
||||
|
||||
self.perform_request(app, path, random_method)
|
||||
|
||||
expected_stat = "{name}.{method}.foo.bar".format(
|
||||
name=app.stat_name, method=random_method)
|
||||
app.statsd.timer.assert_called_once_with(expected_stat)
|
||||
|
||||
def test_strips_both_uuid_types_if_configured(self):
|
||||
app = self.make_stats_middleware(
|
||||
remove_uuid=True, remove_short_uuid=True)
|
||||
random_method = self.get_random_method()
|
||||
random_short_uuid = uuid.uuid4().hex
|
||||
random_uuid = str(uuid.uuid4())
|
||||
path = '/foo/{uuid}/bar/{short_uuid}'.format(
|
||||
uuid=random_uuid, short_uuid=random_short_uuid)
|
||||
|
||||
self.perform_request(app, path, random_method)
|
||||
|
||||
expected_stat = "{name}.{method}.foo.bar".format(
|
||||
name=app.stat_name, method=random_method)
|
||||
app.statsd.timer.assert_called_once_with(expected_stat)
|
||||
|
||||
def test_always_mutates_version_id(self):
|
||||
app = self.make_stats_middleware()
|
||||
random_method = self.get_random_method()
|
||||
path = '/v2.1/foo/bar'
|
||||
|
||||
self.perform_request(app, path, random_method)
|
||||
|
||||
expected_stat = "{name}.{method}.v21.foo.bar".format(
|
||||
name=app.stat_name, method=random_method)
|
||||
app.statsd.timer.assert_called_once_with(expected_stat)
|
||||
|
||||
def test_empty_path_has_sane_stat_name(self):
|
||||
app = self.make_stats_middleware()
|
||||
random_method = self.get_random_method()
|
||||
path = '/'
|
||||
|
||||
self.perform_request(app, path, random_method)
|
||||
|
||||
expected_stat = "{name}.{method}".format(
|
||||
name=app.stat_name, method=random_method)
|
||||
app.statsd.timer.assert_called_once_with(expected_stat)
|
||||
|
|
@ -12,3 +12,4 @@ six>=1.9.0 # MIT
|
|||
stevedore>=1.16.0 # Apache-2.0
|
||||
WebOb>=1.2.3 # MIT
|
||||
debtcollector>=1.2.0 # Apache-2.0
|
||||
statsd>=3.2.1 # MIT
|
||||
|
|
|
|||
Loading…
Reference in New Issue