From 9541a29761db75c4b7b24c216a311214696a0582 Mon Sep 17 00:00:00 2001 From: kpdev Date: Tue, 2 Feb 2021 07:29:25 +0100 Subject: [PATCH] Integrate OSprofiler and Designate *) Add osprofiler wsgi middleware This middleware is used for 2 things: 1) It checks that person who want to trace is trusted and knows secret HMAC key. 2) It start tracing in case of proper trace headers and add first wsgi trace point, with info about HTTP request *) Add initialization of osprofiler at start of serivce. You should use python-designateclient with this patch: https://review.opendev.org/#/c/773575 Run any command with --os-profile SECRET_KEY $ openstack zone create --email \ --os-profile SECRET_KEY # it will print Get pretty HTML with traces: $ osprofiler trace show --html --connection-string \ --out e.g. --connection-string can be redis://localhost:6379 Note that osprofiler should be run from admin user name & tenant. Implements: blueprint designate-os-profiler Change-Id: I2a3787b6428d679555a9add3a57ffe8c2112b6d3 --- designate/api/wsgi.py | 2 + designate/central/rpcapi.py | 2 + designate/common/profiler.py | 88 +++++++++++++++++++ designate/mdns/rpcapi.py | 2 + designate/rpc.py | 19 +++- designate/service.py | 4 + designate/sqlalchemy/session.py | 28 ++++++ designate/tests/__init__.py | 19 ++++ designate/worker/rpcapi.py | 2 + etc/designate/api-paste.ini | 15 ++-- lower-constraints.txt | 1 + ...esignate-os-profiler-3f507d5e1e319f3d.yaml | 13 +++ requirements.txt | 1 + 13 files changed, 189 insertions(+), 7 deletions(-) create mode 100644 designate/common/profiler.py create mode 100644 releasenotes/notes/bp-designate-os-profiler-3f507d5e1e319f3d.yaml diff --git a/designate/api/wsgi.py b/designate/api/wsgi.py index 35b8ae19a..6f7ec6cab 100644 --- a/designate/api/wsgi.py +++ b/designate/api/wsgi.py @@ -18,6 +18,7 @@ from oslo_log import log as logging from paste import deploy from designate.common import config +from designate.common import profiler from designate import conf from designate import heartbeat_emitter from designate import policy @@ -47,6 +48,7 @@ def init_application(): if not rpc.initialized(): rpc.init(CONF) + profiler.setup_profiler("designate-api", CONF.host) heartbeat = heartbeat_emitter.get_heartbeat_emitter('api') heartbeat.start() diff --git a/designate/central/rpcapi.py b/designate/central/rpcapi.py index ccd92d76f..62b5a9a7a 100644 --- a/designate/central/rpcapi.py +++ b/designate/central/rpcapi.py @@ -18,6 +18,7 @@ from oslo_config import cfg from oslo_log import log as logging import oslo_messaging as messaging +from designate.common import profiler from designate.loggingutils import rpc_logging from designate import rpc @@ -31,6 +32,7 @@ def reset(): CENTRAL_API = None +@profiler.trace_cls("rpc") @rpc_logging(LOG, 'central') class CentralAPI(object): """ diff --git a/designate/common/profiler.py b/designate/common/profiler.py new file mode 100644 index 000000000..f045a79b1 --- /dev/null +++ b/designate/common/profiler.py @@ -0,0 +1,88 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging +from oslo_utils import importutils + +import designate.conf +from designate.context import DesignateContext +import webob.dec + +profiler = importutils.try_import("osprofiler.profiler") +profiler_initializer = importutils.try_import("osprofiler.initializer") +profiler_opts = importutils.try_import('osprofiler.opts') +profiler_web = importutils.try_import("osprofiler.web") + +CONF = designate.conf.CONF +LOG = logging.getLogger(__name__) + +if profiler_opts: + profiler_opts.set_defaults(CONF) + + +class WsgiMiddleware(object): + + def __init__(self, application, **kwargs): + self.application = application + + @classmethod + def factory(cls, global_conf, **local_conf): + if profiler_web: + return profiler_web.WsgiMiddleware.factory(global_conf, + **local_conf) + + def filter_(app): + return cls(app, **local_conf) + + return filter_ + + @webob.dec.wsgify + def __call__(self, request): + return request.get_response(self.application) + + +def setup_profiler(binary, host): + if hasattr(CONF, 'profiler') and not CONF.profiler.enabled: + return + + if (profiler_initializer is None or profiler is None or + profiler_opts is None): + LOG.debug('osprofiler is not present') + return + + profiler_initializer.init_from_conf( + conf=CONF, + context=DesignateContext.get_admin_context().to_dict(), + project="designate", + service=binary, + host=host) + LOG.info("osprofiler is enabled") + + +def trace_cls(name, **kwargs): + """Wrap the OSprofiler trace_cls. + + Wrap the OSprofiler trace_cls decorator so that it will not try to + patch the class unless OSprofiler is present. + + :param name: The name of action. For example, wsgi, rpc, db, ... + :param kwargs: Any other keyword args used by profiler.trace_cls + """ + + def decorator(cls): + if profiler and hasattr(CONF, 'profiler') and CONF.profiler.enabled: + trace_decorator = profiler.trace_cls(name, kwargs) + return trace_decorator(cls) + return cls + + return decorator diff --git a/designate/mdns/rpcapi.py b/designate/mdns/rpcapi.py index fc6cc8698..1af843736 100644 --- a/designate/mdns/rpcapi.py +++ b/designate/mdns/rpcapi.py @@ -16,6 +16,7 @@ from oslo_config import cfg from oslo_log import log as logging import oslo_messaging as messaging +from designate.common import profiler from designate.loggingutils import rpc_logging from designate import rpc @@ -30,6 +31,7 @@ def reset(): MDNS_API = None +@profiler.trace_cls("rpc") @rpc_logging(LOG, 'mdns') class MdnsAPI(object): diff --git a/designate/rpc.py b/designate/rpc.py index 4a832f6d1..51efeb714 100644 --- a/designate/rpc.py +++ b/designate/rpc.py @@ -18,11 +18,14 @@ from oslo_config import cfg import oslo_messaging as messaging from oslo_messaging.rpc import dispatcher as rpc_dispatcher from oslo_serialization import jsonutils +from oslo_utils import importutils import designate.context import designate.exceptions from designate import objects +profiler = importutils.try_import('osprofiler.profiler') + __all__ = [ 'init', 'cleanup', @@ -151,9 +154,23 @@ class RequestContextSerializer(messaging.Serializer): return self._base.deserialize_entity(context, entity) def serialize_context(self, context): - return context.to_dict() + _context = context.to_dict() + if profiler is not None: + prof = profiler.get() + if prof is not None: + trace_info = { + "hmac_key": prof.hmac_key, + "base_id": prof.get_base_id(), + "parent_id": prof.get_id() + } + _context.update({"trace_info": trace_info}) + return _context def deserialize_context(self, context): + trace_info = context.pop("trace_info", None) + if trace_info is not None: + if profiler is not None: + profiler.init(**trace_info) return designate.context.DesignateContext.from_dict(context) diff --git a/designate/service.py b/designate/service.py index 0f020c49e..c7f7550e8 100644 --- a/designate/service.py +++ b/designate/service.py @@ -30,6 +30,7 @@ from oslo_service import sslutils from oslo_service import wsgi from oslo_utils import netutils +from designate.common import profiler import designate.conf from designate.i18n import _ from designate.metrics import metrics @@ -54,6 +55,9 @@ class Service(service.Service): if not rpc.initialized(): rpc.init(CONF) + profiler.setup_profiler((''.join(('designate-', self.name))), + self.host) + def start(self): LOG.info('Starting %(name)s service (version: %(version)s)', { diff --git a/designate/sqlalchemy/session.py b/designate/sqlalchemy/session.py index 2c000b4ba..452b85a55 100644 --- a/designate/sqlalchemy/session.py +++ b/designate/sqlalchemy/session.py @@ -16,17 +16,44 @@ """Session Handling for SQLAlchemy backend.""" +import sqlalchemy +import threading + from oslo_config import cfg from oslo_db.sqlalchemy import session from oslo_log import log as logging +from oslo_utils import importutils +osprofiler_sqlalchemy = importutils.try_import('osprofiler.sqlalchemy') LOG = logging.getLogger(__name__) CONF = cfg.CONF +try: + CONF.import_group("profiler", "designate.service") +except cfg.NoSuchGroupError: + pass _FACADES = {} +_LOCK = threading.Lock() + + +def add_db_tracing(cache_name): + global _LOCK + + if not osprofiler_sqlalchemy: + return + if not hasattr(CONF, 'profiler'): + return + if not CONF.profiler.enabled or not CONF.profiler.trace_sqlalchemy: + return + with _LOCK: + osprofiler_sqlalchemy.add_tracing( + sqlalchemy, + _FACADES[cache_name].get_engine(), + "db" + ) def _create_facade_lazily(cfg_group, connection=None, discriminator=None): @@ -39,6 +66,7 @@ def _create_facade_lazily(cfg_group, connection=None, discriminator=None): connection, **conf ) + add_db_tracing(cache_name) return _FACADES[cache_name] diff --git a/designate/tests/__init__.py b/designate/tests/__init__.py index 25258df15..6bbe1bae2 100644 --- a/designate/tests/__init__.py +++ b/designate/tests/__init__.py @@ -19,6 +19,7 @@ import functools import inspect import os import time +from unittest import mock import eventlet from oslo_config import cfg @@ -355,6 +356,8 @@ class TestCase(base.BaseTestCase): group='service:api' ) + self._disable_osprofiler() + # The database fixture needs to be set up here (as opposed to isolated # in a storage test case) because many tests end up using storage. REPOSITORY = os.path.abspath(os.path.join(os.path.dirname(__file__), @@ -399,6 +402,22 @@ class TestCase(base.BaseTestCase): # Setup the Default Pool with some useful settings self._setup_default_pool() + def _disable_osprofiler(self): + """Disable osprofiler. + + osprofiler should not run for unit tests. + """ + + def side_effect(value): + return value + mock_decorator = mock.MagicMock(side_effect=side_effect) + try: + p = mock.patch("osprofiler.profiler.trace_cls", + return_value=mock_decorator) + p.start() + except ModuleNotFoundError: + pass + def _setup_default_pool(self): # Fetch the default pool pool = self.storage.get_pool(self.admin_context, default_pool_id) diff --git a/designate/worker/rpcapi.py b/designate/worker/rpcapi.py index 9d7dd58da..03249677f 100644 --- a/designate/worker/rpcapi.py +++ b/designate/worker/rpcapi.py @@ -17,6 +17,7 @@ from oslo_config import cfg from oslo_log import log as logging import oslo_messaging as messaging +from designate.common import profiler from designate.loggingutils import rpc_logging from designate import rpc @@ -25,6 +26,7 @@ LOG = logging.getLogger(__name__) WORKER_API = None +@profiler.trace_cls("rpc") @rpc_logging(LOG, 'worker') class WorkerAPI(object): """ diff --git a/etc/designate/api-paste.ini b/etc/designate/api-paste.ini index 816ae34c3..59049d0d5 100644 --- a/etc/designate/api-paste.ini +++ b/etc/designate/api-paste.ini @@ -7,8 +7,8 @@ use = egg:Paste#urlmap [composite:osapi_dns_versions] use = call:designate.api.middleware:auth_pipeline_factory -noauth = http_proxy_to_wsgi cors maintenance faultwrapper osapi_dns_app_versions -keystone = http_proxy_to_wsgi cors maintenance faultwrapper osapi_dns_app_versions +noauth = http_proxy_to_wsgi cors maintenance faultwrapper osprofiler osapi_dns_app_versions +keystone = http_proxy_to_wsgi cors maintenance faultwrapper osprofiler osapi_dns_app_versions [app:osapi_dns_app_versions] paste.app_factory = designate.api.versions:factory @@ -16,16 +16,16 @@ paste.app_factory = designate.api.versions:factory [composite:osapi_dns_v2] use = call:designate.api.middleware:auth_pipeline_factory -noauth = http_proxy_to_wsgi cors request_id faultwrapper validation_API_v2 noauthcontext maintenance normalizeuri osapi_dns_app_v2 -keystone = http_proxy_to_wsgi cors request_id faultwrapper validation_API_v2 authtoken keystonecontext maintenance normalizeuri osapi_dns_app_v2 +noauth = http_proxy_to_wsgi cors request_id faultwrapper validation_API_v2 osprofiler noauthcontext maintenance normalizeuri osapi_dns_app_v2 +keystone = http_proxy_to_wsgi cors request_id faultwrapper validation_API_v2 osprofiler authtoken keystonecontext maintenance normalizeuri osapi_dns_app_v2 [app:osapi_dns_app_v2] paste.app_factory = designate.api.v2:factory [composite:osapi_dns_admin] use = call:designate.api.middleware:auth_pipeline_factory -noauth = http_proxy_to_wsgi cors request_id faultwrapper noauthcontext maintenance normalizeuri osapi_dns_app_admin -keystone = http_proxy_to_wsgi cors request_id faultwrapper authtoken keystonecontext maintenance normalizeuri osapi_dns_app_admin +noauth = http_proxy_to_wsgi cors request_id faultwrapper osprofiler noauthcontext maintenance normalizeuri osapi_dns_app_admin +keystone = http_proxy_to_wsgi cors request_id faultwrapper osprofiler authtoken keystonecontext maintenance normalizeuri osapi_dns_app_admin [app:osapi_dns_app_admin] paste.app_factory = designate.api.admin:factory @@ -40,6 +40,9 @@ paste.filter_factory = oslo_middleware:RequestId.factory [filter:http_proxy_to_wsgi] paste.filter_factory = oslo_middleware:HTTPProxyToWSGI.factory +[filter:osprofiler] +paste.filter_factory = designate.common.profiler:WsgiMiddleware.factory + [filter:noauthcontext] paste.filter_factory = designate.api.middleware:NoAuthContextMiddleware.factory diff --git a/lower-constraints.txt b/lower-constraints.txt index 8bf607d50..f961b6d18 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -86,6 +86,7 @@ oslo.upgradecheck==1.3.0 oslo.utils==4.7.0 oslo.versionedobjects==1.31.2 oslotest==3.2.0 +osprofiler==3.4.0 packaging==20.4 paramiko==2.7.1 Paste==2.0.2 diff --git a/releasenotes/notes/bp-designate-os-profiler-3f507d5e1e319f3d.yaml b/releasenotes/notes/bp-designate-os-profiler-3f507d5e1e319f3d.yaml new file mode 100644 index 000000000..58f29d3a8 --- /dev/null +++ b/releasenotes/notes/bp-designate-os-profiler-3f507d5e1e319f3d.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + OSprofiler support was introduced. To allow its usage, the api-paste.ini + file needs to be modified to contain osprofiler middleware. Also + `[profiler]` section needs to be added to the designate.conf file with + `enabled`, `hmac_keys` and `trace_sqlalchemy` flags defined. +security: + - OSprofiler support requires passing of trace information + between various OpenStack services. This information is + securely signed by one of HMAC keys, defined in designate.conf + configuration file. To allow cross-project tracing user should use the key, + that is common among all OpenStack services they want to trace. diff --git a/requirements.txt b/requirements.txt index 0e5b8d9d8..d995a9c83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,7 @@ oslo.service>=1.31.0 # Apache-2.0 oslo.upgradecheck>=1.3.0 oslo.utils>=4.7.0 # Apache-2.0 oslo.versionedobjects>=1.31.2 # Apache-2.0 +osprofiler>=3.4.0 # Apache-2.0 Paste>=2.0.2 # MIT PasteDeploy>=1.5.0 # MIT pbr>=3.1.1 # Apache-2.0