profiler: add python requests profile

Signed-off-by: Sahid Orentino Ferdjaoui <sahid.ferdjaoui@industrialdiscipline.com>
Change-Id: I00e9f5661a3bd54acc846e8c326896d21e70be36

doc/source/user/integration.rst

@@ -87,6 +87,10 @@ In case of OpenStack there are 2 kinds of interaction between 2 services:
   the list and rolling out that change and then removing the older key at
   some time in the future).
 
+  * Optionally you can enable client tracing using `requests`_,
+    Currently only supported by OTLP driver, this will add client call
+    tracing. see `profiler/trace_requests`'s option.
+
 * RPC API
 
   RPC calls are used for interaction between services of one project.
@@ -132,3 +136,4 @@ I think that for all projects we should include by default 5 kinds of points:
 .. _Ceilometer: https://wiki.openstack.org/wiki/Ceilometer
 .. _oslo.messaging: https://pypi.org/project/oslo.messaging
 .. _OSprofiler WSGI middleware: https://github.com/openstack/osprofiler/blob/master/osprofiler/web.py
+.. _requests: https://docs.python-requests.org/en/latest/index.html

osprofiler/drivers/otlp.py

@@ -81,18 +81,21 @@ class OTLP(base.Driver):
     def _kind(self, name):
         if "wsgi" in name:
             return self.trace_api.SpanKind.SERVER
-        elif ("db" in name or "http_client" in name or "api" in name):
+        elif ("db" in name or "http" in name or "api" in name):
             return self.trace_api.SpanKind.CLIENT
         return self.trace_api.SpanKind.INTERNAL
 
     def _name(self, payload):
         info = payload["info"]
         if info.get("request"):
-            return "{}_{}".format(
+            return "WSGI_{}_{}".format(
                 info["request"]["method"], info["request"]["path"])
         elif info.get("db"):
             return "SQL_{}".format(
                 info["db"]["statement"].split(' ', 1)[0].upper())
+        elif info.get("requests"):
+            return "REQUESTS_{}_{}".format(
+                info["requests"]["method"], info["requests"]["hostname"])
         return payload["name"].rstrip("-start")
 
     def notify(self, payload):
@@ -130,6 +133,10 @@ class OTLP(base.Driver):
                 if payload.get("info", {}).get(call):
                     span.set_attribute(
                         "result", payload["info"][call]["result"])
+            # Store result of requests
+            if payload.get("info", {}).get("requests"):
+                span.set_attribute(
+                    "status_code", payload["info"]["requests"]["status_code"])
             # Span error tag and log
             if payload["info"].get("etype"):
                 span.set_attribute("error", True)
@@ -168,6 +175,14 @@ class OTLP(base.Driver):
             tags["http.query"] = info["request"]["query"]
             tags["http.method"] = info["request"]["method"]
             tags["http.scheme"] = info["request"]["scheme"]
+        elif info.get("requests"):
+            # requests call
+            tags["http.path"] = info["requests"]["path"]
+            tags["http.query"] = info["requests"]["query"]
+            tags["http.method"] = info["requests"]["method"]
+            tags["http.scheme"] = info["requests"]["scheme"]
+            tags["http.hostname"] = info["requests"]["hostname"]
+            tags["http.port"] = info["requests"]["port"]
         elif info.get("function"):
             # RPC, function calls
             if "args" in info["function"]:

osprofiler/initializer.py

@@ -14,6 +14,7 @@
 #    under the License.
 
 from osprofiler import notifier
+from osprofiler import requests
 from osprofiler import web
 
 
@@ -39,3 +40,5 @@ def init_from_conf(conf, context, project, service, host, **kwargs):
         **kwargs)
     notifier.set(_notifier)
     web.enable(conf.profiler.hmac_keys)
+    if conf.profiler.trace_requests:
+        requests.enable()

osprofiler/opts.py

@@ -64,6 +64,22 @@ Possible values:
   higher level of operations. Single SQL queries cannot be analyzed this way.
 """)
 
+_trace_requests_opt = cfg.BoolOpt(
+    "trace_requests",
+    default=False,
+    help="""
+Enable python requests package profiling.
+
+Supported drivers: jaeger+otlp
+
+Default value is False.
+
+Possible values:
+
+* True: Enables requests profiling.
+* False: Disables requests profiling.
+""")
+
 _hmac_keys_opt = cfg.StrOpt(
     "hmac_keys",
     default="SECRET_KEY",
@@ -159,6 +175,7 @@ Possible values:
 _PROFILER_OPTS = [
     _enabled_opt,
     _trace_sqlalchemy_opt,
+    _trace_requests_opt,
     _hmac_keys_opt,
     _connection_string_opt,
     _es_doc_type_opt,

osprofiler/requests.py

@@ -0,0 +1,73 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging as log
+from urllib import parse as parser
+
+from osprofiler import profiler
+from osprofiler import web
+
+
+# Register an OSProfiler HTTP Adapter that will profile any call made with
+# requests.
+
+LOG = log.getLogger(__name__)
+
+_FUNC = None
+
+try:
+    from requests.adapters import HTTPAdapter
+except ImportError:
+    pass
+else:
+    def send(self, request, *args, **kwargs):
+        parsed_url = parser.urlparse(request.url)
+
+        # Best effort guessing port if needed
+        port = parsed_url.port or ""
+        if not port and parsed_url.scheme == "http":
+            port = 80
+        elif not port and parsed_url.scheme == "https":
+            port = 443
+
+        profiler.start(parsed_url.scheme, info={"requests": {
+            "method": request.method,
+            "query": parsed_url.query,
+            "path": parsed_url.path,
+            "hostname": parsed_url.hostname,
+            "port": port,
+            "scheme": parsed_url.scheme}})
+
+        # Profiling headers are overrident to take in account this new
+        # context/span.
+        request.headers.update(
+            web.get_trace_id_headers())
+
+        response = _FUNC(self, request, *args, **kwargs)
+
+        profiler.stop(info={"requests": {
+            "status_code": response.status_code}})
+
+        return response
+
+    _FUNC = HTTPAdapter.send
+
+
+def enable():
+    if _FUNC:
+        HTTPAdapter.send = send
+        LOG.debug("profiling requests enabled")
+    else:
+        LOG.warning("unable to activate profiling for requests, "
+                    "please ensure that python requests is installed.")

releasenotes/notes/add-requests-profiling-761e09f243d36966.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    New profiler for python requests. Currently only OTLP driver is
+    supporting it, see `profiler/trace_requests`'s option.