diff --git a/doc/source/user/usage.rst b/doc/source/user/usage.rst index 60c6ac82..93c0c1d7 100644 --- a/doc/source/user/usage.rst +++ b/doc/source/user/usage.rst @@ -179,3 +179,40 @@ logging options by sending a SIGHUP. logging.setup(cfg.CONF, 'foo') +Profiling +~~~~~~~~~ + +Processes spawned through oslo_service.service can be profiled (function +calltrace) through eventlet_backdoor module. Service has to configure +backdoor_port option to enable it's workers to listen on TCP ports. +Then user can send "prof()" command to capture worker processes function +calltrace. + +1) To start profiling send "prof()" command on processes listening port + +2) To stop profiling and capture "pstat" calltrace to a file, send prof + command with filename as argument i.e "prof(filename)" + on worker processes listening port. Stats file (in pstat format) with + user provided filename by adding .prof as suffix will be generated + in temp directory. + +For example, to profile neutron server process (which is listening on +port 8002 configured through backdoor_port option), + +.. code-block:: bash + + $ echo "prof()" | nc localhost 8002 + $ neutron net-create n1; neutron port-create --name p1 n1; + $ neutron port-delete p1; neutron port-delete p1 + $ echo "prof('neutron')" | nc localhost 8002 + + +This will generate "/tmp/neutron.prof" as stats file. Later user can print +the stats from the trace file like below + +.. code-block:: python + + import pstats + + stats = pstats.Stats('/tmp/neutron.prof') + stats.print_stats() diff --git a/lower-constraints.txt b/lower-constraints.txt index 47570201..5c00875a 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -72,3 +72,4 @@ traceback2==1.4.0 unittest2==1.1.0 WebOb==1.7.1 wrapt==1.7.0 +Yappi==0.98 diff --git a/oslo_service/eventlet_backdoor.py b/oslo_service/eventlet_backdoor.py index a82d5e2b..5e3186f3 100644 --- a/oslo_service/eventlet_backdoor.py +++ b/oslo_service/eventlet_backdoor.py @@ -22,10 +22,12 @@ import logging import os import pprint import sys +import tempfile import traceback import eventlet.backdoor import greenlet +import yappi from eventlet.green import socket from oslo_service._i18n import _ @@ -89,6 +91,30 @@ def _find_objects(t): return [o for o in gc.get_objects() if isinstance(o, t)] +def _capture_profile(fname=''): + if not fname: + yappi.set_clock_type('cpu') + # We need to set context to greenlet to profile greenlets + # https://bitbucket.org/sumerc/yappi/pull-requests/3 + yappi.set_context_id_callback( + lambda: id(greenlet.getcurrent())) + yappi.set_context_name_callback( + lambda: greenlet.getcurrent().__class__.__name__) + yappi.start() + else: + yappi.stop() + stats = yappi.get_func_stats() + # User should provide filename. This file with a suffix .prof + # will be created in temp directory. + try: + stats_file = os.path.join(tempfile.gettempdir(), fname + '.prof') + stats.save(stats_file, "pstat") + except Exception as e: + print("Error while saving the trace stats ", str(e)) + finally: + yappi.clear_stats() + + def _print_greenthreads(simple=True): for i, gt in enumerate(_find_objects(greenlet.greenlet)): print(i, gt) @@ -175,6 +201,7 @@ def _initialize_if_enabled(conf): 'fo': _find_objects, 'pgt': _print_greenthreads, 'pnt': _print_nativethreads, + 'prof': _capture_profile, } if conf.backdoor_port is None and conf.backdoor_socket is None: diff --git a/releasenotes/notes/profile-worker-5d3fd0f0251d62b8.yaml b/releasenotes/notes/profile-worker-5d3fd0f0251d62b8.yaml new file mode 100644 index 00000000..97878512 --- /dev/null +++ b/releasenotes/notes/profile-worker-5d3fd0f0251d62b8.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add support for profiling (capture function calltrace) service's worker + processes. diff --git a/requirements.txt b/requirements.txt index 450a8532..ad86e4b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,4 @@ oslo.i18n>=3.15.3 # Apache-2.0 PasteDeploy>=1.5.0 # MIT Routes>=2.3.1 # MIT Paste>=2.0.2 # MIT +Yappi>=0.98 # MIT