Merge "Profile Oslo Service processes"

This commit is contained in:
Zuul 2019-02-11 21:31:19 +00:00 committed by Gerrit Code Review
commit fdcfd0fcff
5 changed files with 71 additions and 0 deletions

View File

@ -179,3 +179,40 @@ logging options by sending a SIGHUP.
logging.setup(cfg.CONF, 'foo')
Profiling
~~~~~~~~~
Processes spawned through oslo_service.service can be profiled (function
calltrace) through eventlet_backdoor module. Service has to configure
backdoor_port option to enable it's workers to listen on TCP ports.
Then user can send "prof()" command to capture worker processes function
calltrace.
1) To start profiling send "prof()" command on processes listening port
2) To stop profiling and capture "pstat" calltrace to a file, send prof
command with filename as argument i.e "prof(filename)"
on worker processes listening port. Stats file (in pstat format) with
user provided filename by adding .prof as suffix will be generated
in temp directory.
For example, to profile neutron server process (which is listening on
port 8002 configured through backdoor_port option),
.. code-block:: bash
$ echo "prof()" | nc localhost 8002
$ neutron net-create n1; neutron port-create --name p1 n1;
$ neutron port-delete p1; neutron port-delete p1
$ echo "prof('neutron')" | nc localhost 8002
This will generate "/tmp/neutron.prof" as stats file. Later user can print
the stats from the trace file like below
.. code-block:: python
import pstats
stats = pstats.Stats('/tmp/neutron.prof')
stats.print_stats()

View File

@ -72,3 +72,4 @@ traceback2==1.4.0
unittest2==1.1.0
WebOb==1.7.1
wrapt==1.7.0
Yappi==0.98

View File

@ -22,10 +22,12 @@ import logging
import os
import pprint
import sys
import tempfile
import traceback
import eventlet.backdoor
import greenlet
import yappi
from eventlet.green import socket
from oslo_service._i18n import _
@ -89,6 +91,30 @@ def _find_objects(t):
return [o for o in gc.get_objects() if isinstance(o, t)]
def _capture_profile(fname=''):
if not fname:
yappi.set_clock_type('cpu')
# We need to set context to greenlet to profile greenlets
# https://bitbucket.org/sumerc/yappi/pull-requests/3
yappi.set_context_id_callback(
lambda: id(greenlet.getcurrent()))
yappi.set_context_name_callback(
lambda: greenlet.getcurrent().__class__.__name__)
yappi.start()
else:
yappi.stop()
stats = yappi.get_func_stats()
# User should provide filename. This file with a suffix .prof
# will be created in temp directory.
try:
stats_file = os.path.join(tempfile.gettempdir(), fname + '.prof')
stats.save(stats_file, "pstat")
except Exception as e:
print("Error while saving the trace stats ", str(e))
finally:
yappi.clear_stats()
def _print_greenthreads(simple=True):
for i, gt in enumerate(_find_objects(greenlet.greenlet)):
print(i, gt)
@ -175,6 +201,7 @@ def _initialize_if_enabled(conf):
'fo': _find_objects,
'pgt': _print_greenthreads,
'pnt': _print_nativethreads,
'prof': _capture_profile,
}
if conf.backdoor_port is None and conf.backdoor_socket is None:

View File

@ -0,0 +1,5 @@
---
features:
- |
Add support for profiling (capture function calltrace) service's worker
processes.

View File

@ -17,3 +17,4 @@ oslo.i18n>=3.15.3 # Apache-2.0
PasteDeploy>=1.5.0 # MIT
Routes>=2.3.1 # MIT
Paste>=2.0.2 # MIT
Yappi>=0.98 # MIT