Integrate OSprofiler and Designate

*) Add osprofiler wsgi middleware

This middleware is used for 2 things:
1) It checks that person who want to trace is trusted and knows
secret HMAC key.
2) It start tracing in case of proper trace headers and add
first wsgi trace point, with info about HTTP request
*) Add initialization of osprofiler at start of serivce.

You should use python-designateclient with this patch:

https://review.opendev.org/#/c/773575

Run any command with --os-profile SECRET_KEY

  $ openstack zone create --email <email_id> <zone_name> \
    --os-profile SECRET_KEY
  # it will print <Trace ID>

Get pretty HTML with traces:

  $ osprofiler trace show --html <Trace ID> --connection-string \
   <connection_string> --out <output.html>
  e.g. --connection-string can be redis://localhost:6379

Note that osprofiler should be run from admin user name & tenant.

Implements: blueprint designate-os-profiler
Change-Id: I2a3787b6428d679555a9add3a57ffe8c2112b6d3

designate/api/wsgi.py

@@ -18,6 +18,7 @@ from oslo_log import log as logging
 from paste import deploy
 
 from designate.common import config
+from designate.common import profiler
 from designate import conf
 from designate import heartbeat_emitter
 from designate import policy
@@ -47,6 +48,7 @@ def init_application():
     if not rpc.initialized():
         rpc.init(CONF)
 
+    profiler.setup_profiler("designate-api", CONF.host)
     heartbeat = heartbeat_emitter.get_heartbeat_emitter('api')
     heartbeat.start()
 

designate/central/rpcapi.py

@@ -18,6 +18,7 @@ from oslo_config import cfg
 from oslo_log import log as logging
 import oslo_messaging as messaging
 
+from designate.common import profiler
 from designate.loggingutils import rpc_logging
 from designate import rpc
 
@@ -31,6 +32,7 @@ def reset():
     CENTRAL_API = None
 
 
+@profiler.trace_cls("rpc")
 @rpc_logging(LOG, 'central')
 class CentralAPI(object):
     """

designate/common/profiler.py

@@ -0,0 +1,88 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_log import log as logging
+from oslo_utils import importutils
+
+import designate.conf
+from designate.context import DesignateContext
+import webob.dec
+
+profiler = importutils.try_import("osprofiler.profiler")
+profiler_initializer = importutils.try_import("osprofiler.initializer")
+profiler_opts = importutils.try_import('osprofiler.opts')
+profiler_web = importutils.try_import("osprofiler.web")
+
+CONF = designate.conf.CONF
+LOG = logging.getLogger(__name__)
+
+if profiler_opts:
+    profiler_opts.set_defaults(CONF)
+
+
+class WsgiMiddleware(object):
+
+    def __init__(self, application, **kwargs):
+        self.application = application
+
+    @classmethod
+    def factory(cls, global_conf, **local_conf):
+        if profiler_web:
+            return profiler_web.WsgiMiddleware.factory(global_conf,
+                                                       **local_conf)
+
+        def filter_(app):
+            return cls(app, **local_conf)
+
+        return filter_
+
+    @webob.dec.wsgify
+    def __call__(self, request):
+        return request.get_response(self.application)
+
+
+def setup_profiler(binary, host):
+    if hasattr(CONF, 'profiler') and not CONF.profiler.enabled:
+        return
+
+    if (profiler_initializer is None or profiler is None or
+            profiler_opts is None):
+        LOG.debug('osprofiler is not present')
+        return
+
+    profiler_initializer.init_from_conf(
+            conf=CONF,
+            context=DesignateContext.get_admin_context().to_dict(),
+            project="designate",
+            service=binary,
+            host=host)
+    LOG.info("osprofiler is enabled")
+
+
+def trace_cls(name, **kwargs):
+    """Wrap the OSprofiler trace_cls.
+
+    Wrap the OSprofiler trace_cls decorator so that it will not try to
+    patch the class unless OSprofiler is present.
+
+    :param name: The name of action. For example, wsgi, rpc, db, ...
+    :param kwargs: Any other keyword args used by profiler.trace_cls
+    """
+
+    def decorator(cls):
+        if profiler and hasattr(CONF, 'profiler') and CONF.profiler.enabled:
+            trace_decorator = profiler.trace_cls(name, kwargs)
+            return trace_decorator(cls)
+        return cls
+
+    return decorator

designate/mdns/rpcapi.py

@@ -16,6 +16,7 @@ from oslo_config import cfg
 from oslo_log import log as logging
 import oslo_messaging as messaging
 
+from designate.common import profiler
 from designate.loggingutils import rpc_logging
 from designate import rpc
 
@@ -30,6 +31,7 @@ def reset():
     MDNS_API = None
 
 
+@profiler.trace_cls("rpc")
 @rpc_logging(LOG, 'mdns')
 class MdnsAPI(object):
 

designate/rpc.py

@@ -18,11 +18,14 @@ from oslo_config import cfg
 import oslo_messaging as messaging
 from oslo_messaging.rpc import dispatcher as rpc_dispatcher
 from oslo_serialization import jsonutils
+from oslo_utils import importutils
 
 import designate.context
 import designate.exceptions
 from designate import objects
 
+profiler = importutils.try_import('osprofiler.profiler')
+
 __all__ = [
     'init',
     'cleanup',
@@ -151,9 +154,23 @@ class RequestContextSerializer(messaging.Serializer):
         return self._base.deserialize_entity(context, entity)
 
     def serialize_context(self, context):
-        return context.to_dict()
+        _context = context.to_dict()
+        if profiler is not None:
+            prof = profiler.get()
+            if prof is not None:
+                trace_info = {
+                    "hmac_key": prof.hmac_key,
+                    "base_id": prof.get_base_id(),
+                    "parent_id": prof.get_id()
+                }
+                _context.update({"trace_info": trace_info})
+        return _context
 
     def deserialize_context(self, context):
+        trace_info = context.pop("trace_info", None)
+        if trace_info is not None:
+            if profiler is not None:
+                profiler.init(**trace_info)
         return designate.context.DesignateContext.from_dict(context)
 
 

designate/service.py

@@ -30,6 +30,7 @@ from oslo_service import sslutils
 from oslo_service import wsgi
 from oslo_utils import netutils
 
+from designate.common import profiler
 import designate.conf
 from designate.i18n import _
 from designate.metrics import metrics
@@ -54,6 +55,9 @@ class Service(service.Service):
         if not rpc.initialized():
             rpc.init(CONF)
 
+        profiler.setup_profiler((''.join(('designate-', self.name))),
+                                self.host)
+
     def start(self):
         LOG.info('Starting %(name)s service (version: %(version)s)',
                  {

designate/sqlalchemy/session.py

@@ -16,17 +16,44 @@
 
 """Session Handling for SQLAlchemy backend."""
 
+import sqlalchemy
+import threading
+
 from oslo_config import cfg
 from oslo_db.sqlalchemy import session
 from oslo_log import log as logging
+from oslo_utils import importutils
 
+osprofiler_sqlalchemy = importutils.try_import('osprofiler.sqlalchemy')
 
 LOG = logging.getLogger(__name__)
 
 CONF = cfg.CONF
+try:
+    CONF.import_group("profiler", "designate.service")
+except cfg.NoSuchGroupError:
+    pass
 
 
 _FACADES = {}
+_LOCK = threading.Lock()
+
+
+def add_db_tracing(cache_name):
+    global _LOCK
+
+    if not osprofiler_sqlalchemy:
+        return
+    if not hasattr(CONF, 'profiler'):
+        return
+    if not CONF.profiler.enabled or not CONF.profiler.trace_sqlalchemy:
+        return
+    with _LOCK:
+        osprofiler_sqlalchemy.add_tracing(
+            sqlalchemy,
+            _FACADES[cache_name].get_engine(),
+            "db"
+        )
 
 
 def _create_facade_lazily(cfg_group, connection=None, discriminator=None):
@@ -39,6 +66,7 @@ def _create_facade_lazily(cfg_group, connection=None, discriminator=None):
             connection,
             **conf
         )
+        add_db_tracing(cache_name)
 
     return _FACADES[cache_name]
 

designate/tests/__init__.py

@@ -19,6 +19,7 @@ import functools
 import inspect
 import os
 import time
+from unittest import mock
 
 import eventlet
 from oslo_config import cfg
@@ -355,6 +356,8 @@ class TestCase(base.BaseTestCase):
             group='service:api'
         )
 
+        self._disable_osprofiler()
+
         # The database fixture needs to be set up here (as opposed to isolated
         # in a storage test case) because many tests end up using storage.
         REPOSITORY = os.path.abspath(os.path.join(os.path.dirname(__file__),
@@ -399,6 +402,22 @@ class TestCase(base.BaseTestCase):
         # Setup the Default Pool with some useful settings
         self._setup_default_pool()
 
+    def _disable_osprofiler(self):
+        """Disable osprofiler.
+
+        osprofiler should not run for unit tests.
+        """
+
+        def side_effect(value):
+            return value
+        mock_decorator = mock.MagicMock(side_effect=side_effect)
+        try:
+            p = mock.patch("osprofiler.profiler.trace_cls",
+                           return_value=mock_decorator)
+            p.start()
+        except ModuleNotFoundError:
+            pass
+
     def _setup_default_pool(self):
         # Fetch the default pool
         pool = self.storage.get_pool(self.admin_context, default_pool_id)

designate/worker/rpcapi.py

@@ -17,6 +17,7 @@ from oslo_config import cfg
 from oslo_log import log as logging
 import oslo_messaging as messaging
 
+from designate.common import profiler
 from designate.loggingutils import rpc_logging
 from designate import rpc
 
@@ -25,6 +26,7 @@ LOG = logging.getLogger(__name__)
 WORKER_API = None
 
 
+@profiler.trace_cls("rpc")
 @rpc_logging(LOG, 'worker')
 class WorkerAPI(object):
     """

etc/designate/api-paste.ini

@@ -7,8 +7,8 @@ use = egg:Paste#urlmap
 
 [composite:osapi_dns_versions]
 use = call:designate.api.middleware:auth_pipeline_factory
-noauth = http_proxy_to_wsgi cors maintenance faultwrapper osapi_dns_app_versions
-keystone = http_proxy_to_wsgi cors maintenance faultwrapper osapi_dns_app_versions
+noauth = http_proxy_to_wsgi cors maintenance faultwrapper osprofiler osapi_dns_app_versions
+keystone = http_proxy_to_wsgi cors maintenance faultwrapper osprofiler osapi_dns_app_versions
 
 [app:osapi_dns_app_versions]
 paste.app_factory = designate.api.versions:factory
@@ -16,16 +16,16 @@ paste.app_factory = designate.api.versions:factory
 
 [composite:osapi_dns_v2]
 use = call:designate.api.middleware:auth_pipeline_factory
-noauth = http_proxy_to_wsgi cors request_id faultwrapper validation_API_v2 noauthcontext maintenance normalizeuri osapi_dns_app_v2
-keystone = http_proxy_to_wsgi cors request_id faultwrapper validation_API_v2 authtoken keystonecontext maintenance normalizeuri osapi_dns_app_v2
+noauth = http_proxy_to_wsgi cors request_id faultwrapper validation_API_v2 osprofiler noauthcontext maintenance normalizeuri osapi_dns_app_v2
+keystone = http_proxy_to_wsgi cors request_id faultwrapper validation_API_v2 osprofiler authtoken keystonecontext maintenance normalizeuri osapi_dns_app_v2
 
 [app:osapi_dns_app_v2]
 paste.app_factory = designate.api.v2:factory
 
 [composite:osapi_dns_admin]
 use = call:designate.api.middleware:auth_pipeline_factory
-noauth = http_proxy_to_wsgi cors request_id faultwrapper noauthcontext maintenance normalizeuri osapi_dns_app_admin
-keystone = http_proxy_to_wsgi cors request_id faultwrapper authtoken keystonecontext maintenance normalizeuri osapi_dns_app_admin
+noauth = http_proxy_to_wsgi cors request_id faultwrapper osprofiler noauthcontext maintenance normalizeuri osapi_dns_app_admin
+keystone = http_proxy_to_wsgi cors request_id faultwrapper osprofiler authtoken keystonecontext maintenance normalizeuri osapi_dns_app_admin
 
 [app:osapi_dns_app_admin]
 paste.app_factory = designate.api.admin:factory
@@ -40,6 +40,9 @@ paste.filter_factory = oslo_middleware:RequestId.factory
 [filter:http_proxy_to_wsgi]
 paste.filter_factory = oslo_middleware:HTTPProxyToWSGI.factory
 
+[filter:osprofiler]
+paste.filter_factory = designate.common.profiler:WsgiMiddleware.factory
+
 [filter:noauthcontext]
 paste.filter_factory = designate.api.middleware:NoAuthContextMiddleware.factory
 

lower-constraints.txt

@@ -86,6 +86,7 @@ oslo.upgradecheck==1.3.0
 oslo.utils==4.7.0
 oslo.versionedobjects==1.31.2
 oslotest==3.2.0
+osprofiler==3.4.0
 packaging==20.4
 paramiko==2.7.1
 Paste==2.0.2

releasenotes/notes/bp-designate-os-profiler-3f507d5e1e319f3d.yaml

@@ -0,0 +1,13 @@
+---
+features:
+  - |
+    OSprofiler support was introduced. To allow its usage, the api-paste.ini
+    file needs to be modified to contain osprofiler middleware. Also
+    `[profiler]` section needs to be added to the designate.conf file with
+    `enabled`, `hmac_keys` and `trace_sqlalchemy` flags defined.
+security:
+  - OSprofiler support requires passing of trace information
+    between various OpenStack services. This information is
+    securely signed by one of HMAC keys, defined in designate.conf
+    configuration file. To allow cross-project tracing user should use the key,
+    that is common among all OpenStack services they want to trace.

requirements.txt

@@ -22,6 +22,7 @@ oslo.service>=1.31.0 # Apache-2.0
 oslo.upgradecheck>=1.3.0
 oslo.utils>=4.7.0 # Apache-2.0
 oslo.versionedobjects>=1.31.2 # Apache-2.0
+osprofiler>=3.4.0 # Apache-2.0
 Paste>=2.0.2 # MIT
 PasteDeploy>=1.5.0 # MIT
 pbr>=3.1.1 # Apache-2.0