summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorvenkata anil <anilvenkata@redhat.com>2018-12-26 20:27:33 +0530
committervenkata anil <anilvenkata@redhat.com>2019-01-16 07:38:13 -0500
commita04daefbb158e955dcfe7379e2b38c272ff31da2 (patch)
treea6358d605c45674428ff09c52cdb06f767731780
parentd987a4a84c40ac905a1295341c0e05a73c7b6b1c (diff)
Profile Oslo Service processes
This patch enables profiling (capturing function call trace like cProfile [1]) worker processes on the fly while service is running. User requests the oslo service process to start profiling by writing "prof()" command to backdoor socket, once the service (like neutron-server) finishes expected processing (example finishing API call), user again writes "prof()" command with file name as argument to dump the function calltrace stats. Stats file (in pstat format with user provided filename by adding .prof) will be generated in temp directory. For example, to profile neutron server process, 1) echo "prof()" | nc localhost 8002 2) Issue neutron command (or run rally scenarios tests) neutron net-create n1 neutron port-create --name p1 n1 neutron port-delete p1 neutron net-delete n1 3) echo "prof('neutron')" | nc localhost 8002 where 8002 is the port which we set like below in neutron.conf backdoor_port=8002 We can later print the stats from the trace file like below stats = pstats.Stats('/tmp/neutron.prof') stats.print_stats() The trace file will look like in (for above neutron API calls) [2]. We use Yappi with context set to greenlet [3] to profile greenlets. We can't use GreenletProfiler [4], which does the same [5] 1) as it is no more maintained 2) Also compiling yappi source inside GreenletProfiler is failing for python3. [1] https://docs.python.org/2/library/profile.html [2] https://gist.github.com/venkataanil/64d5e672bf0206dc151e73fc1058a983 [3] https://bitbucket.org/sumerc/yappi/pull-requests/3 [4] https://pypi.org/project/GreenletProfiler/ [5] https://emptysqua.re/blog/greenletprofiler/ Depends-On: Ibea0cdb732923f1b53d5cb6aeeb4041fb5973494 Change-Id: Id2418093494f1e233a653f6c73bd6894e4a40184
Notes
Notes (review): Code-Review+1: Joe Talerico <jtaleric@redhat.com> Code-Review+1: Hervé Beraud <hberaud@redhat.com> Code-Review+1: Daniel Alvarez <dalvarez@redhat.com> Code-Review+2: Zane Bitter <zbitter@redhat.com> Code-Review+1: Numan Siddique <nusiddiq@redhat.com> Code-Review+1: Sean McGinnis <sean.mcginnis@gmail.com> Code-Review+1: Slawek Kaplonski <skaplons@redhat.com> Code-Review+1: Nate Johnston <nate.johnston@redhat.com> Workflow+1: Moisés Guimarães <moguimar@redhat.com> Code-Review+2: Moisés Guimarães <moguimar@redhat.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Mon, 11 Feb 2019 21:31:19 +0000 Reviewed-on: https://review.openstack.org/627414 Project: openstack/oslo.service Branch: refs/heads/master
-rw-r--r--doc/source/user/usage.rst37
-rw-r--r--lower-constraints.txt1
-rw-r--r--oslo_service/eventlet_backdoor.py27
-rw-r--r--releasenotes/notes/profile-worker-5d3fd0f0251d62b8.yaml5
-rw-r--r--requirements.txt1
5 files changed, 71 insertions, 0 deletions
diff --git a/doc/source/user/usage.rst b/doc/source/user/usage.rst
index 60c6ac8..93c0c1d 100644
--- a/doc/source/user/usage.rst
+++ b/doc/source/user/usage.rst
@@ -179,3 +179,40 @@ logging options by sending a SIGHUP.
179 logging.setup(cfg.CONF, 'foo') 179 logging.setup(cfg.CONF, 'foo')
180 180
181 181
182Profiling
183~~~~~~~~~
184
185Processes spawned through oslo_service.service can be profiled (function
186calltrace) through eventlet_backdoor module. Service has to configure
187backdoor_port option to enable it's workers to listen on TCP ports.
188Then user can send "prof()" command to capture worker processes function
189calltrace.
190
1911) To start profiling send "prof()" command on processes listening port
192
1932) To stop profiling and capture "pstat" calltrace to a file, send prof
194 command with filename as argument i.e "prof(filename)"
195 on worker processes listening port. Stats file (in pstat format) with
196 user provided filename by adding .prof as suffix will be generated
197 in temp directory.
198
199For example, to profile neutron server process (which is listening on
200port 8002 configured through backdoor_port option),
201
202.. code-block:: bash
203
204 $ echo "prof()" | nc localhost 8002
205 $ neutron net-create n1; neutron port-create --name p1 n1;
206 $ neutron port-delete p1; neutron port-delete p1
207 $ echo "prof('neutron')" | nc localhost 8002
208
209
210This will generate "/tmp/neutron.prof" as stats file. Later user can print
211the stats from the trace file like below
212
213.. code-block:: python
214
215 import pstats
216
217 stats = pstats.Stats('/tmp/neutron.prof')
218 stats.print_stats()
diff --git a/lower-constraints.txt b/lower-constraints.txt
index 9f72e90..710a330 100644
--- a/lower-constraints.txt
+++ b/lower-constraints.txt
@@ -72,3 +72,4 @@ traceback2==1.4.0
72unittest2==1.1.0 72unittest2==1.1.0
73WebOb==1.7.1 73WebOb==1.7.1
74wrapt==1.7.0 74wrapt==1.7.0
75Yappi==0.98
diff --git a/oslo_service/eventlet_backdoor.py b/oslo_service/eventlet_backdoor.py
index f4ecbf4..e14694c 100644
--- a/oslo_service/eventlet_backdoor.py
+++ b/oslo_service/eventlet_backdoor.py
@@ -23,10 +23,12 @@ import os
23import pprint 23import pprint
24import socket 24import socket
25import sys 25import sys
26import tempfile
26import traceback 27import traceback
27 28
28import eventlet.backdoor 29import eventlet.backdoor
29import greenlet 30import greenlet
31import yappi
30 32
31from oslo_service._i18n import _ 33from oslo_service._i18n import _
32from oslo_service import _options 34from oslo_service import _options
@@ -89,6 +91,30 @@ def _find_objects(t):
89 return [o for o in gc.get_objects() if isinstance(o, t)] 91 return [o for o in gc.get_objects() if isinstance(o, t)]
90 92
91 93
94def _capture_profile(fname=''):
95 if not fname:
96 yappi.set_clock_type('cpu')
97 # We need to set context to greenlet to profile greenlets
98 # https://bitbucket.org/sumerc/yappi/pull-requests/3
99 yappi.set_context_id_callback(
100 lambda: id(greenlet.getcurrent()))
101 yappi.set_context_name_callback(
102 lambda: greenlet.getcurrent().__class__.__name__)
103 yappi.start()
104 else:
105 yappi.stop()
106 stats = yappi.get_func_stats()
107 # User should provide filename. This file with a suffix .prof
108 # will be created in temp directory.
109 try:
110 stats_file = os.path.join(tempfile.gettempdir(), fname + '.prof')
111 stats.save(stats_file, "pstat")
112 except Exception as e:
113 print("Error while saving the trace stats ", str(e))
114 finally:
115 yappi.clear_stats()
116
117
92def _print_greenthreads(simple=True): 118def _print_greenthreads(simple=True):
93 for i, gt in enumerate(_find_objects(greenlet.greenlet)): 119 for i, gt in enumerate(_find_objects(greenlet.greenlet)):
94 print(i, gt) 120 print(i, gt)
@@ -162,6 +188,7 @@ def _initialize_if_enabled(conf):
162 'fo': _find_objects, 188 'fo': _find_objects,
163 'pgt': _print_greenthreads, 189 'pgt': _print_greenthreads,
164 'pnt': _print_nativethreads, 190 'pnt': _print_nativethreads,
191 'prof': _capture_profile,
165 } 192 }
166 193
167 if conf.backdoor_port is None and conf.backdoor_socket is None: 194 if conf.backdoor_port is None and conf.backdoor_socket is None:
diff --git a/releasenotes/notes/profile-worker-5d3fd0f0251d62b8.yaml b/releasenotes/notes/profile-worker-5d3fd0f0251d62b8.yaml
new file mode 100644
index 0000000..9787851
--- /dev/null
+++ b/releasenotes/notes/profile-worker-5d3fd0f0251d62b8.yaml
@@ -0,0 +1,5 @@
1---
2features:
3 - |
4 Add support for profiling (capture function calltrace) service's worker
5 processes.
diff --git a/requirements.txt b/requirements.txt
index ecca359..a72850e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,3 +17,4 @@ oslo.i18n>=3.15.3 # Apache-2.0
17PasteDeploy>=1.5.0 # MIT 17PasteDeploy>=1.5.0 # MIT
18Routes>=2.3.1 # MIT 18Routes>=2.3.1 # MIT
19Paste>=2.0.2 # MIT 19Paste>=2.0.2 # MIT
20Yappi>=0.98 # MIT