diff --git a/designate/api/wsgi.py b/designate/api/wsgi.py index 35b8ae19a..6f7ec6cab 100644 --- a/designate/api/wsgi.py +++ b/designate/api/wsgi.py @@ -18,6 +18,7 @@ from oslo_log import log as logging from paste import deploy from designate.common import config +from designate.common import profiler from designate import conf from designate import heartbeat_emitter from designate import policy @@ -47,6 +48,7 @@ def init_application(): if not rpc.initialized(): rpc.init(CONF) + profiler.setup_profiler("designate-api", CONF.host) heartbeat = heartbeat_emitter.get_heartbeat_emitter('api') heartbeat.start() diff --git a/designate/central/rpcapi.py b/designate/central/rpcapi.py index ccd92d76f..62b5a9a7a 100644 --- a/designate/central/rpcapi.py +++ b/designate/central/rpcapi.py @@ -18,6 +18,7 @@ from oslo_config import cfg from oslo_log import log as logging import oslo_messaging as messaging +from designate.common import profiler from designate.loggingutils import rpc_logging from designate import rpc @@ -31,6 +32,7 @@ def reset(): CENTRAL_API = None +@profiler.trace_cls("rpc") @rpc_logging(LOG, 'central') class CentralAPI(object): """ diff --git a/designate/common/profiler.py b/designate/common/profiler.py new file mode 100644 index 000000000..f045a79b1 --- /dev/null +++ b/designate/common/profiler.py @@ -0,0 +1,88 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging +from oslo_utils import importutils + +import designate.conf +from designate.context import DesignateContext +import webob.dec + +profiler = importutils.try_import("osprofiler.profiler") +profiler_initializer = importutils.try_import("osprofiler.initializer") +profiler_opts = importutils.try_import('osprofiler.opts') +profiler_web = importutils.try_import("osprofiler.web") + +CONF = designate.conf.CONF +LOG = logging.getLogger(__name__) + +if profiler_opts: + profiler_opts.set_defaults(CONF) + + +class WsgiMiddleware(object): + + def __init__(self, application, **kwargs): + self.application = application + + @classmethod + def factory(cls, global_conf, **local_conf): + if profiler_web: + return profiler_web.WsgiMiddleware.factory(global_conf, + **local_conf) + + def filter_(app): + return cls(app, **local_conf) + + return filter_ + + @webob.dec.wsgify + def __call__(self, request): + return request.get_response(self.application) + + +def setup_profiler(binary, host): + if hasattr(CONF, 'profiler') and not CONF.profiler.enabled: + return + + if (profiler_initializer is None or profiler is None or + profiler_opts is None): + LOG.debug('osprofiler is not present') + return + + profiler_initializer.init_from_conf( + conf=CONF, + context=DesignateContext.get_admin_context().to_dict(), + project="designate", + service=binary, + host=host) + LOG.info("osprofiler is enabled") + + +def trace_cls(name, **kwargs): + """Wrap the OSprofiler trace_cls. + + Wrap the OSprofiler trace_cls decorator so that it will not try to + patch the class unless OSprofiler is present. + + :param name: The name of action. For example, wsgi, rpc, db, ... + :param kwargs: Any other keyword args used by profiler.trace_cls + """ + + def decorator(cls): + if profiler and hasattr(CONF, 'profiler') and CONF.profiler.enabled: + trace_decorator = profiler.trace_cls(name, kwargs) + return trace_decorator(cls) + return cls + + return decorator diff --git a/designate/mdns/rpcapi.py b/designate/mdns/rpcapi.py index fc6cc8698..1af843736 100644 --- a/designate/mdns/rpcapi.py +++ b/designate/mdns/rpcapi.py @@ -16,6 +16,7 @@ from oslo_config import cfg from oslo_log import log as logging import oslo_messaging as messaging +from designate.common import profiler from designate.loggingutils import rpc_logging from designate import rpc @@ -30,6 +31,7 @@ def reset(): MDNS_API = None +@profiler.trace_cls("rpc") @rpc_logging(LOG, 'mdns') class MdnsAPI(object): diff --git a/designate/rpc.py b/designate/rpc.py index 4a832f6d1..51efeb714 100644 --- a/designate/rpc.py +++ b/designate/rpc.py @@ -18,11 +18,14 @@ from oslo_config import cfg import oslo_messaging as messaging from oslo_messaging.rpc import dispatcher as rpc_dispatcher from oslo_serialization import jsonutils +from oslo_utils import importutils import designate.context import designate.exceptions from designate import objects +profiler = importutils.try_import('osprofiler.profiler') + __all__ = [ 'init', 'cleanup', @@ -151,9 +154,23 @@ class RequestContextSerializer(messaging.Serializer): return self._base.deserialize_entity(context, entity) def serialize_context(self, context): - return context.to_dict() + _context = context.to_dict() + if profiler is not None: + prof = profiler.get() + if prof is not None: + trace_info = { + "hmac_key": prof.hmac_key, + "base_id": prof.get_base_id(), + "parent_id": prof.get_id() + } + _context.update({"trace_info": trace_info}) + return _context def deserialize_context(self, context): + trace_info = context.pop("trace_info", None) + if trace_info is not None: + if profiler is not None: + profiler.init(**trace_info) return designate.context.DesignateContext.from_dict(context) diff --git a/designate/service.py b/designate/service.py index 0f020c49e..c7f7550e8 100644 --- a/designate/service.py +++ b/designate/service.py @@ -30,6 +30,7 @@ from oslo_service import sslutils from oslo_service import wsgi from oslo_utils import netutils +from designate.common import profiler import designate.conf from designate.i18n import _ from designate.metrics import metrics @@ -54,6 +55,9 @@ class Service(service.Service): if not rpc.initialized(): rpc.init(CONF) + profiler.setup_profiler((''.join(('designate-', self.name))), + self.host) + def start(self): LOG.info('Starting %(name)s service (version: %(version)s)', { diff --git a/designate/sqlalchemy/session.py b/designate/sqlalchemy/session.py index 2c000b4ba..452b85a55 100644 --- a/designate/sqlalchemy/session.py +++ b/designate/sqlalchemy/session.py @@ -16,17 +16,44 @@ """Session Handling for SQLAlchemy backend.""" +import sqlalchemy +import threading + from oslo_config import cfg from oslo_db.sqlalchemy import session from oslo_log import log as logging +from oslo_utils import importutils +osprofiler_sqlalchemy = importutils.try_import('osprofiler.sqlalchemy') LOG = logging.getLogger(__name__) CONF = cfg.CONF +try: + CONF.import_group("profiler", "designate.service") +except cfg.NoSuchGroupError: + pass _FACADES = {} +_LOCK = threading.Lock() + + +def add_db_tracing(cache_name): + global _LOCK + + if not osprofiler_sqlalchemy: + return + if not hasattr(CONF, 'profiler'): + return + if not CONF.profiler.enabled or not CONF.profiler.trace_sqlalchemy: + return + with _LOCK: + osprofiler_sqlalchemy.add_tracing( + sqlalchemy, + _FACADES[cache_name].get_engine(), + "db" + ) def _create_facade_lazily(cfg_group, connection=None, discriminator=None): @@ -39,6 +66,7 @@ def _create_facade_lazily(cfg_group, connection=None, discriminator=None): connection, **conf ) + add_db_tracing(cache_name) return _FACADES[cache_name] diff --git a/designate/tests/__init__.py b/designate/tests/__init__.py index 25258df15..6bbe1bae2 100644 --- a/designate/tests/__init__.py +++ b/designate/tests/__init__.py @@ -19,6 +19,7 @@ import functools import inspect import os import time +from unittest import mock import eventlet from oslo_config import cfg @@ -355,6 +356,8 @@ class TestCase(base.BaseTestCase): group='service:api' ) + self._disable_osprofiler() + # The database fixture needs to be set up here (as opposed to isolated # in a storage test case) because many tests end up using storage. REPOSITORY = os.path.abspath(os.path.join(os.path.dirname(__file__), @@ -399,6 +402,22 @@ class TestCase(base.BaseTestCase): # Setup the Default Pool with some useful settings self._setup_default_pool() + def _disable_osprofiler(self): + """Disable osprofiler. + + osprofiler should not run for unit tests. + """ + + def side_effect(value): + return value + mock_decorator = mock.MagicMock(side_effect=side_effect) + try: + p = mock.patch("osprofiler.profiler.trace_cls", + return_value=mock_decorator) + p.start() + except ModuleNotFoundError: + pass + def _setup_default_pool(self): # Fetch the default pool pool = self.storage.get_pool(self.admin_context, default_pool_id) diff --git a/designate/worker/rpcapi.py b/designate/worker/rpcapi.py index 9d7dd58da..03249677f 100644 --- a/designate/worker/rpcapi.py +++ b/designate/worker/rpcapi.py @@ -17,6 +17,7 @@ from oslo_config import cfg from oslo_log import log as logging import oslo_messaging as messaging +from designate.common import profiler from designate.loggingutils import rpc_logging from designate import rpc @@ -25,6 +26,7 @@ LOG = logging.getLogger(__name__) WORKER_API = None +@profiler.trace_cls("rpc") @rpc_logging(LOG, 'worker') class WorkerAPI(object): """ diff --git a/etc/designate/api-paste.ini b/etc/designate/api-paste.ini index 816ae34c3..59049d0d5 100644 --- a/etc/designate/api-paste.ini +++ b/etc/designate/api-paste.ini @@ -7,8 +7,8 @@ use = egg:Paste#urlmap [composite:osapi_dns_versions] use = call:designate.api.middleware:auth_pipeline_factory -noauth = http_proxy_to_wsgi cors maintenance faultwrapper osapi_dns_app_versions -keystone = http_proxy_to_wsgi cors maintenance faultwrapper osapi_dns_app_versions +noauth = http_proxy_to_wsgi cors maintenance faultwrapper osprofiler osapi_dns_app_versions +keystone = http_proxy_to_wsgi cors maintenance faultwrapper osprofiler osapi_dns_app_versions [app:osapi_dns_app_versions] paste.app_factory = designate.api.versions:factory @@ -16,16 +16,16 @@ paste.app_factory = designate.api.versions:factory [composite:osapi_dns_v2] use = call:designate.api.middleware:auth_pipeline_factory -noauth = http_proxy_to_wsgi cors request_id faultwrapper validation_API_v2 noauthcontext maintenance normalizeuri osapi_dns_app_v2 -keystone = http_proxy_to_wsgi cors request_id faultwrapper validation_API_v2 authtoken keystonecontext maintenance normalizeuri osapi_dns_app_v2 +noauth = http_proxy_to_wsgi cors request_id faultwrapper validation_API_v2 osprofiler noauthcontext maintenance normalizeuri osapi_dns_app_v2 +keystone = http_proxy_to_wsgi cors request_id faultwrapper validation_API_v2 osprofiler authtoken keystonecontext maintenance normalizeuri osapi_dns_app_v2 [app:osapi_dns_app_v2] paste.app_factory = designate.api.v2:factory [composite:osapi_dns_admin] use = call:designate.api.middleware:auth_pipeline_factory -noauth = http_proxy_to_wsgi cors request_id faultwrapper noauthcontext maintenance normalizeuri osapi_dns_app_admin -keystone = http_proxy_to_wsgi cors request_id faultwrapper authtoken keystonecontext maintenance normalizeuri osapi_dns_app_admin +noauth = http_proxy_to_wsgi cors request_id faultwrapper osprofiler noauthcontext maintenance normalizeuri osapi_dns_app_admin +keystone = http_proxy_to_wsgi cors request_id faultwrapper osprofiler authtoken keystonecontext maintenance normalizeuri osapi_dns_app_admin [app:osapi_dns_app_admin] paste.app_factory = designate.api.admin:factory @@ -40,6 +40,9 @@ paste.filter_factory = oslo_middleware:RequestId.factory [filter:http_proxy_to_wsgi] paste.filter_factory = oslo_middleware:HTTPProxyToWSGI.factory +[filter:osprofiler] +paste.filter_factory = designate.common.profiler:WsgiMiddleware.factory + [filter:noauthcontext] paste.filter_factory = designate.api.middleware:NoAuthContextMiddleware.factory diff --git a/lower-constraints.txt b/lower-constraints.txt index 8bf607d50..f961b6d18 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -86,6 +86,7 @@ oslo.upgradecheck==1.3.0 oslo.utils==4.7.0 oslo.versionedobjects==1.31.2 oslotest==3.2.0 +osprofiler==3.4.0 packaging==20.4 paramiko==2.7.1 Paste==2.0.2 diff --git a/releasenotes/notes/bp-designate-os-profiler-3f507d5e1e319f3d.yaml b/releasenotes/notes/bp-designate-os-profiler-3f507d5e1e319f3d.yaml new file mode 100644 index 000000000..58f29d3a8 --- /dev/null +++ b/releasenotes/notes/bp-designate-os-profiler-3f507d5e1e319f3d.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + OSprofiler support was introduced. To allow its usage, the api-paste.ini + file needs to be modified to contain osprofiler middleware. Also + `[profiler]` section needs to be added to the designate.conf file with + `enabled`, `hmac_keys` and `trace_sqlalchemy` flags defined. +security: + - OSprofiler support requires passing of trace information + between various OpenStack services. This information is + securely signed by one of HMAC keys, defined in designate.conf + configuration file. To allow cross-project tracing user should use the key, + that is common among all OpenStack services they want to trace. diff --git a/requirements.txt b/requirements.txt index 0e5b8d9d8..d995a9c83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,7 @@ oslo.service>=1.31.0 # Apache-2.0 oslo.upgradecheck>=1.3.0 oslo.utils>=4.7.0 # Apache-2.0 oslo.versionedobjects>=1.31.2 # Apache-2.0 +osprofiler>=3.4.0 # Apache-2.0 Paste>=2.0.2 # MIT PasteDeploy>=1.5.0 # MIT pbr>=3.1.1 # Apache-2.0