summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZuul <zuul@review.openstack.org>2019-02-11 21:31:19 +0000
committerGerrit Code Review <review@openstack.org>2019-02-11 21:31:19 +0000
commitfdcfd0fcff8cbbfd435d682696c5c9bb6cfdc841 (patch)
treee3959a2cdd221c62bcfc027ccfbdb0d4d6ddcf15
parent3c645a77ec04a36e999bb985dbe0116588b81ef0 (diff)
parenta04daefbb158e955dcfe7379e2b38c272ff31da2 (diff)
Merge "Profile Oslo Service processes"1.37.0
-rw-r--r--doc/source/user/usage.rst37
-rw-r--r--lower-constraints.txt1
-rw-r--r--oslo_service/eventlet_backdoor.py27
-rw-r--r--releasenotes/notes/profile-worker-5d3fd0f0251d62b8.yaml5
-rw-r--r--requirements.txt1
5 files changed, 71 insertions, 0 deletions
diff --git a/doc/source/user/usage.rst b/doc/source/user/usage.rst
index 60c6ac8..93c0c1d 100644
--- a/doc/source/user/usage.rst
+++ b/doc/source/user/usage.rst
@@ -179,3 +179,40 @@ logging options by sending a SIGHUP.
179 logging.setup(cfg.CONF, 'foo') 179 logging.setup(cfg.CONF, 'foo')
180 180
181 181
182Profiling
183~~~~~~~~~
184
185Processes spawned through oslo_service.service can be profiled (function
186calltrace) through eventlet_backdoor module. Service has to configure
187backdoor_port option to enable it's workers to listen on TCP ports.
188Then user can send "prof()" command to capture worker processes function
189calltrace.
190
1911) To start profiling send "prof()" command on processes listening port
192
1932) To stop profiling and capture "pstat" calltrace to a file, send prof
194 command with filename as argument i.e "prof(filename)"
195 on worker processes listening port. Stats file (in pstat format) with
196 user provided filename by adding .prof as suffix will be generated
197 in temp directory.
198
199For example, to profile neutron server process (which is listening on
200port 8002 configured through backdoor_port option),
201
202.. code-block:: bash
203
204 $ echo "prof()" | nc localhost 8002
205 $ neutron net-create n1; neutron port-create --name p1 n1;
206 $ neutron port-delete p1; neutron port-delete p1
207 $ echo "prof('neutron')" | nc localhost 8002
208
209
210This will generate "/tmp/neutron.prof" as stats file. Later user can print
211the stats from the trace file like below
212
213.. code-block:: python
214
215 import pstats
216
217 stats = pstats.Stats('/tmp/neutron.prof')
218 stats.print_stats()
diff --git a/lower-constraints.txt b/lower-constraints.txt
index 4757020..5c00875 100644
--- a/lower-constraints.txt
+++ b/lower-constraints.txt
@@ -72,3 +72,4 @@ traceback2==1.4.0
72unittest2==1.1.0 72unittest2==1.1.0
73WebOb==1.7.1 73WebOb==1.7.1
74wrapt==1.7.0 74wrapt==1.7.0
75Yappi==0.98
diff --git a/oslo_service/eventlet_backdoor.py b/oslo_service/eventlet_backdoor.py
index a82d5e2..5e3186f 100644
--- a/oslo_service/eventlet_backdoor.py
+++ b/oslo_service/eventlet_backdoor.py
@@ -22,10 +22,12 @@ import logging
22import os 22import os
23import pprint 23import pprint
24import sys 24import sys
25import tempfile
25import traceback 26import traceback
26 27
27import eventlet.backdoor 28import eventlet.backdoor
28import greenlet 29import greenlet
30import yappi
29 31
30from eventlet.green import socket 32from eventlet.green import socket
31from oslo_service._i18n import _ 33from oslo_service._i18n import _
@@ -89,6 +91,30 @@ def _find_objects(t):
89 return [o for o in gc.get_objects() if isinstance(o, t)] 91 return [o for o in gc.get_objects() if isinstance(o, t)]
90 92
91 93
94def _capture_profile(fname=''):
95 if not fname:
96 yappi.set_clock_type('cpu')
97 # We need to set context to greenlet to profile greenlets
98 # https://bitbucket.org/sumerc/yappi/pull-requests/3
99 yappi.set_context_id_callback(
100 lambda: id(greenlet.getcurrent()))
101 yappi.set_context_name_callback(
102 lambda: greenlet.getcurrent().__class__.__name__)
103 yappi.start()
104 else:
105 yappi.stop()
106 stats = yappi.get_func_stats()
107 # User should provide filename. This file with a suffix .prof
108 # will be created in temp directory.
109 try:
110 stats_file = os.path.join(tempfile.gettempdir(), fname + '.prof')
111 stats.save(stats_file, "pstat")
112 except Exception as e:
113 print("Error while saving the trace stats ", str(e))
114 finally:
115 yappi.clear_stats()
116
117
92def _print_greenthreads(simple=True): 118def _print_greenthreads(simple=True):
93 for i, gt in enumerate(_find_objects(greenlet.greenlet)): 119 for i, gt in enumerate(_find_objects(greenlet.greenlet)):
94 print(i, gt) 120 print(i, gt)
@@ -175,6 +201,7 @@ def _initialize_if_enabled(conf):
175 'fo': _find_objects, 201 'fo': _find_objects,
176 'pgt': _print_greenthreads, 202 'pgt': _print_greenthreads,
177 'pnt': _print_nativethreads, 203 'pnt': _print_nativethreads,
204 'prof': _capture_profile,
178 } 205 }
179 206
180 if conf.backdoor_port is None and conf.backdoor_socket is None: 207 if conf.backdoor_port is None and conf.backdoor_socket is None:
diff --git a/releasenotes/notes/profile-worker-5d3fd0f0251d62b8.yaml b/releasenotes/notes/profile-worker-5d3fd0f0251d62b8.yaml
new file mode 100644
index 0000000..9787851
--- /dev/null
+++ b/releasenotes/notes/profile-worker-5d3fd0f0251d62b8.yaml
@@ -0,0 +1,5 @@
1---
2features:
3 - |
4 Add support for profiling (capture function calltrace) service's worker
5 processes.
diff --git a/requirements.txt b/requirements.txt
index 450a853..ad86e4b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,3 +17,4 @@ oslo.i18n>=3.15.3 # Apache-2.0
17PasteDeploy>=1.5.0 # MIT 17PasteDeploy>=1.5.0 # MIT
18Routes>=2.3.1 # MIT 18Routes>=2.3.1 # MIT
19Paste>=2.0.2 # MIT 19Paste>=2.0.2 # MIT
20Yappi>=0.98 # MIT