Merge "Add support for osprofiler in dragonflow"

doc/source/osprofiler.rst

@@ -0,0 +1,68 @@
+..
+  Licensed under the Apache License, Version 2.0 (the "License"); you may
+  not use this file except in compliance with the License. You may obtain
+  a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  License for the specific language governing permissions and limitations
+  under the License.
+
+==========
+OSProfiler
+==========
+
+OSProfiler provides a tiny but powerful library that is used by
+most (soon to be all) OpenStack projects and their python clients. It
+provides functionality to be able to generate one trace per request, that goes
+through all involved services. This trace can then be extracted and used
+to build a tree of calls which can be quite handy for a variety of
+reasons (for example in isolating cross-project performance issues).
+
+More about OSProfiler:
+https://docs.openstack.org/osprofiler/latest/
+
+Senlin supports using OSProfiler to trace the performance of each
+key internal processing, including RESTful API, RPC, cluster actions,
+node actions, DB operations etc.
+
+Enabling OSProfiler
+~~~~~~~~~~~~~~~~~~~
+
+To configure DevStack to enable OSProfiler, edit the
+``${DEVSTACK_DIR}/local.conf`` file and add::
+
+    enable_plugin panko https://git.openstack.org/openstack/panko
+    enable_plugin ceilometer https://git.openstack.org/openstack/ceilometer
+    enable_plugin osprofiler https://git.openstack.org/openstack/osprofiler
+
+to the ``[[local|localrc]]`` section.
+
+.. note:: The order of the plugins enabling matters.
+
+Using OSProfiler
+~~~~~~~~~~~~~~~~
+
+After successfully deploy your development environment, following profiler
+configs will be auto added to ``dragonflow.conf``::
+
+    [profiler]
+    enabled = True
+    trace_sqlalchemy = True
+    hmac_keys = SECRET_KEY
+
+``hmac_keys`` is the secret key(s) to use for encrypting context data for
+performance profiling, default value is 'SECRET_KEY', you can modify it to
+any random string(s).
+
+Run any command with ``--os-profile SECRET_KEY``::
+
+  $ openstack --os-profile SECRET_KEY floating ip create public
+  # it will print a <Trace ID>
+
+Get pretty HTML with traces::
+
+  $ osprofiler trace show --html <Trace ID>

dragonflow/common/profiler.py

@@ -0,0 +1,88 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import contextlib
+
+from oslo_log import log as logging
+
+try:
+    import osprofiler.initializer
+    from osprofiler import opts as profiler_opts
+    from osprofiler import profiler
+except Exception:
+    # osprofiler package is not installed
+    profiler_opts = None
+    profiler = None
+
+from dragonflow import conf as cfg
+
+
+CONF = cfg.CONF
+if profiler_opts:
+    profiler_opts.set_defaults(CONF)
+LOG = logging.getLogger(__name__)
+
+
+@contextlib.contextmanager
+def profiler_context(*args, **kwargs):
+    if is_profiler_enabled():
+        with profiler.Trace(*args, **kwargs) as tracer:
+            yield tracer
+    else:
+        yield None
+
+
+def _get_profiler_instance():
+    # If profiler does not exist or not enabled
+    if profiler is None or not CONF.profiler.enabled:
+        return None
+    instance = profiler.get()
+    # Try to initialize an instance
+    if instance is None:
+        instance = profiler.init(CONF.profiler.hmac_keys)
+        LOG.debug("Initialized osprofiler, base trace ID: %s",
+                  instance.get_id())
+    return instance
+
+
+def is_profiler_enabled():
+    return _get_profiler_instance() is not None
+
+
+def setup(name, host='0.0.0.0'):
+    """Setup OSprofiler notifier and enable profiling.
+
+    :param name: name of the service, that will be profiled
+    :param host: host (either host name or host address) the service will be
+                 running on. By default host will be set to 0.0.0.0, but more
+                 specified host name / address usage is highly recommended.
+    """
+    if CONF.profiler.enabled:
+        osprofiler.initializer.init_from_conf(
+            conf=CONF,
+            context={},
+            project='dragonflow',
+            service=name,
+            host=host
+        )
+        LOG.info("OSProfiler is enabled.\n"
+                 "Traces provided from the profiler "
+                 "can only be subscribed to using the same HMAC keys that "
+                 "are configured in Dragonflow's configuration file "
+                 "under the [profiler] section.\n To disable OSprofiler "
+                 "set in /etc/neutron/dragonflow.conf:\n"
+                 "[profiler]\n"
+                 "enabled=False")
+
+
+def get():
+    return profiler

dragonflow/controller/df_bgp_service.py

@@ -12,13 +12,13 @@
 
 import sys
 
-from neutron.common import config as common_config
 from oslo_log import log as logging
 from oslo_service import loopingcall
 from oslo_service import service
 from oslo_utils import importutils
 
 from dragonflow import conf as cfg
+from dragonflow.controller import df_config
 from dragonflow.controller import service as df_service
 from dragonflow.db import api_nb
 from dragonflow.db import db_store
@@ -164,8 +164,7 @@ class BGPService(service.Service):
 
 
 def main():
-    common_config.init(sys.argv[1:])
-    common_config.setup_logging()
+    df_config.init(sys.argv)
     # BGP dynamic route is not a service that needs real time response.
     # So disable pubsub here and use period task to do BGP job.
     cfg.CONF.set_override('enable_df_pub_sub', False, group='df')

dragonflow/controller/df_config.py

@@ -0,0 +1,23 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from neutron.common import config as common_config
+
+from dragonflow.common import profiler as df_profiler
+from dragonflow import conf as cfg
+
+
+def init(argv):
+    common_config.init(argv[1:])
+    common_config.setup_logging()
+    df_profiler.setup(argv[0], cfg.CONF.host)

dragonflow/controller/df_local_controller.py

@@ -15,7 +15,6 @@
 
 import sys
 
-from neutron.common import config as common_config
 from oslo_log import log
 from oslo_service import loopingcall
 from ryu.app.ofctl import service as of_service
@@ -25,6 +24,7 @@ from ryu import cfg as ryu_cfg
 from dragonflow.common import utils as df_utils
 from dragonflow import conf as cfg
 from dragonflow.controller.common import constants as ctrl_const
+from dragonflow.controller import df_config
 from dragonflow.controller import ryu_base_app
 from dragonflow.controller import service
 from dragonflow.controller import topology
@@ -343,8 +343,8 @@ def init_ryu_config():
 # <local ip address> <southbound_db_ip_address>
 def main():
     chassis_name = cfg.CONF.host
-    common_config.init(sys.argv[1:])
-    common_config.setup_logging()
+    df_config.init(sys.argv)
+
     init_ryu_config()
     nb_api = api_nb.NbApi.get_instance(False)
     controller = DfLocalController(chassis_name, nb_api)

dragonflow/controller/df_publisher_service.py

@@ -15,12 +15,12 @@ import sys
 import time
 import traceback
 
-from neutron.common import config as common_config
 from oslo_log import log as logging
 
 from dragonflow.common import exceptions
 from dragonflow.common import utils as df_utils
 from dragonflow import conf as cfg
+from dragonflow.controller import df_config
 from dragonflow.controller import service as df_service
 from dragonflow.db import api_nb
 from dragonflow.db import db_common
@@ -146,8 +146,7 @@ class PublisherService(object):
 
 
 def main():
-    common_config.init(sys.argv[1:])
-    common_config.setup_logging()
+    df_config.init(sys.argv)
     cfg.CONF.set_override('enable_df_pub_sub', False, group='df')
     nb_api = api_nb.NbApi.get_instance(False)
     service = PublisherService(nb_api)

dragonflow/controller/ryu_base_app.py

@@ -25,6 +25,7 @@ from ryu.ofproto import ofproto_parser
 from ryu.ofproto import ofproto_v1_3
 from ryu import utils
 
+from dragonflow.common import profiler as df_profiler
 from dragonflow.controller.common import constants
 from dragonflow.controller import dispatcher
 
@@ -133,7 +134,9 @@ class RyuDFAdapter(ofp_handler.OFPHandler):
         table_id = msg.table_id
         if table_id in self.table_handlers:
             handler = self.table_handlers[table_id]
-            handler(event)
+            with df_profiler.profiler_context('packet_in',
+                                              info={"func": handler.__name__}):
+                handler(event)
         else:
             LOG.info("No handler for table id %(table)s with message "
                      "%(msg)", {'table': table_id, 'msg': msg})

dragonflow/db/model_framework.py

@@ -20,6 +20,8 @@ from oslo_serialization import jsonutils
 import six
 
 from dragonflow._i18n import _
+from dragonflow.common import profiler as df_profiler
+
 
 LOG = log.getLogger(__name__)
 
@@ -42,7 +44,7 @@ def _normalize_tuple(v):
 class _CommonBase(models.Base):
     '''Base class for extending jsonmodels' Base
 
-    Here we add common facilites needed to support:
+    Here we add common facilities needed to support:
 
      * Event registration/dispatch
      * CRUD hooks
@@ -105,7 +107,12 @@ class _CommonBase(models.Base):
                        'event': event,
                        'resource': self})
             try:
-                cb(self, *args, **kwargs)
+                with df_profiler.profiler_context(
+                        'emit',
+                        info={'func': cb.__name__,
+                              'module': cb.__module__,
+                              'event': event}):
+                    cb(self, *args, **kwargs)
             except Exception:
                 LOG.exception(
                     'Error while calling %(func)r(*%(_args)r, **%(kw)r)',
@@ -340,6 +347,12 @@ def construct_nb_db_model(cls_=None, indexes=None, events=frozenset()):
         fields = frozenset(n for n, _ in cls_.iterate_over_fields())
         cls_._field_names = fields
 
+        # Make sure profiler is properly initialized
+        # if df_profiler.is_profiler_enabled():
+        # FIXME snapiri: This SHOULD be the right place, but in our code it
+        #                creates a loop. Should fix to support more traces
+        #    cls_ = df_profiler.get().trace_cls('model')(cls_)
+
         return cls_
 
     if cls_ is None:

dragonflow/tests/unit/test_ryu_base_app.py

@@ -68,6 +68,7 @@ class TestRyuDFAdapter(tests_base.BaseTestCase):
         self.mock_app.reset_mock()
         ev = mock.Mock()
         ev.msg.table_id = 10
+        self.mock_app.packet_in_handler.__name__ = 'mock'
         self.ryu_df_adapter.register_table_handler(
                 10, self.mock_app.packet_in_handler)
         self.ryu_df_adapter.OF_packet_in_handler(ev)

releasenotes/notes/osprofiler-97302fc3440b4fd9.yaml

@@ -0,0 +1,12 @@
+---
+features:
+  - |
+    Added support for the OSProfiler.
+    OSProfiler provides a tiny but powerful library that is used by
+    most (soon to be all) OpenStack projects and their python clients. It
+    provides functionality to be able to generate one trace per request, that
+    goes through all involved services. This trace can then be extracted and
+    used to build a tree of calls which can be quite handy for a variety of
+    reasons (for example in isolating cross-project performance issues).
+    The OSProfiler is off by default, extra configuration is required to
+    enable it. Please refer to the documentation for further details.