Merge "Profile Oslo Service processes"
This commit is contained in:
commit
fdcfd0fcff
|
@ -179,3 +179,40 @@ logging options by sending a SIGHUP.
|
||||||
logging.setup(cfg.CONF, 'foo')
|
logging.setup(cfg.CONF, 'foo')
|
||||||
|
|
||||||
|
|
||||||
|
Profiling
|
||||||
|
~~~~~~~~~
|
||||||
|
|
||||||
|
Processes spawned through oslo_service.service can be profiled (function
|
||||||
|
calltrace) through eventlet_backdoor module. Service has to configure
|
||||||
|
backdoor_port option to enable it's workers to listen on TCP ports.
|
||||||
|
Then user can send "prof()" command to capture worker processes function
|
||||||
|
calltrace.
|
||||||
|
|
||||||
|
1) To start profiling send "prof()" command on processes listening port
|
||||||
|
|
||||||
|
2) To stop profiling and capture "pstat" calltrace to a file, send prof
|
||||||
|
command with filename as argument i.e "prof(filename)"
|
||||||
|
on worker processes listening port. Stats file (in pstat format) with
|
||||||
|
user provided filename by adding .prof as suffix will be generated
|
||||||
|
in temp directory.
|
||||||
|
|
||||||
|
For example, to profile neutron server process (which is listening on
|
||||||
|
port 8002 configured through backdoor_port option),
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
$ echo "prof()" | nc localhost 8002
|
||||||
|
$ neutron net-create n1; neutron port-create --name p1 n1;
|
||||||
|
$ neutron port-delete p1; neutron port-delete p1
|
||||||
|
$ echo "prof('neutron')" | nc localhost 8002
|
||||||
|
|
||||||
|
|
||||||
|
This will generate "/tmp/neutron.prof" as stats file. Later user can print
|
||||||
|
the stats from the trace file like below
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
import pstats
|
||||||
|
|
||||||
|
stats = pstats.Stats('/tmp/neutron.prof')
|
||||||
|
stats.print_stats()
|
||||||
|
|
|
@ -72,3 +72,4 @@ traceback2==1.4.0
|
||||||
unittest2==1.1.0
|
unittest2==1.1.0
|
||||||
WebOb==1.7.1
|
WebOb==1.7.1
|
||||||
wrapt==1.7.0
|
wrapt==1.7.0
|
||||||
|
Yappi==0.98
|
||||||
|
|
|
@ -22,10 +22,12 @@ import logging
|
||||||
import os
|
import os
|
||||||
import pprint
|
import pprint
|
||||||
import sys
|
import sys
|
||||||
|
import tempfile
|
||||||
import traceback
|
import traceback
|
||||||
|
|
||||||
import eventlet.backdoor
|
import eventlet.backdoor
|
||||||
import greenlet
|
import greenlet
|
||||||
|
import yappi
|
||||||
|
|
||||||
from eventlet.green import socket
|
from eventlet.green import socket
|
||||||
from oslo_service._i18n import _
|
from oslo_service._i18n import _
|
||||||
|
@ -89,6 +91,30 @@ def _find_objects(t):
|
||||||
return [o for o in gc.get_objects() if isinstance(o, t)]
|
return [o for o in gc.get_objects() if isinstance(o, t)]
|
||||||
|
|
||||||
|
|
||||||
|
def _capture_profile(fname=''):
|
||||||
|
if not fname:
|
||||||
|
yappi.set_clock_type('cpu')
|
||||||
|
# We need to set context to greenlet to profile greenlets
|
||||||
|
# https://bitbucket.org/sumerc/yappi/pull-requests/3
|
||||||
|
yappi.set_context_id_callback(
|
||||||
|
lambda: id(greenlet.getcurrent()))
|
||||||
|
yappi.set_context_name_callback(
|
||||||
|
lambda: greenlet.getcurrent().__class__.__name__)
|
||||||
|
yappi.start()
|
||||||
|
else:
|
||||||
|
yappi.stop()
|
||||||
|
stats = yappi.get_func_stats()
|
||||||
|
# User should provide filename. This file with a suffix .prof
|
||||||
|
# will be created in temp directory.
|
||||||
|
try:
|
||||||
|
stats_file = os.path.join(tempfile.gettempdir(), fname + '.prof')
|
||||||
|
stats.save(stats_file, "pstat")
|
||||||
|
except Exception as e:
|
||||||
|
print("Error while saving the trace stats ", str(e))
|
||||||
|
finally:
|
||||||
|
yappi.clear_stats()
|
||||||
|
|
||||||
|
|
||||||
def _print_greenthreads(simple=True):
|
def _print_greenthreads(simple=True):
|
||||||
for i, gt in enumerate(_find_objects(greenlet.greenlet)):
|
for i, gt in enumerate(_find_objects(greenlet.greenlet)):
|
||||||
print(i, gt)
|
print(i, gt)
|
||||||
|
@ -175,6 +201,7 @@ def _initialize_if_enabled(conf):
|
||||||
'fo': _find_objects,
|
'fo': _find_objects,
|
||||||
'pgt': _print_greenthreads,
|
'pgt': _print_greenthreads,
|
||||||
'pnt': _print_nativethreads,
|
'pnt': _print_nativethreads,
|
||||||
|
'prof': _capture_profile,
|
||||||
}
|
}
|
||||||
|
|
||||||
if conf.backdoor_port is None and conf.backdoor_socket is None:
|
if conf.backdoor_port is None and conf.backdoor_socket is None:
|
||||||
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
---
|
||||||
|
features:
|
||||||
|
- |
|
||||||
|
Add support for profiling (capture function calltrace) service's worker
|
||||||
|
processes.
|
|
@ -17,3 +17,4 @@ oslo.i18n>=3.15.3 # Apache-2.0
|
||||||
PasteDeploy>=1.5.0 # MIT
|
PasteDeploy>=1.5.0 # MIT
|
||||||
Routes>=2.3.1 # MIT
|
Routes>=2.3.1 # MIT
|
||||||
Paste>=2.0.2 # MIT
|
Paste>=2.0.2 # MIT
|
||||||
|
Yappi>=0.98 # MIT
|
||||||
|
|
Loading…
Reference in New Issue