Add an ability to hide sensitive data from http action logs

Opportunity to hide sensitive data from http action logs, such as:
* Request headers
* Request body
* Response body

Change-Id: I6d1b1844898343b8fa30f704761096e3d2936c4d
Implements: blueprint mistral-hide-sensitive-data-from-http-actions-logs
Signed-off-by: Oleg Ovcharuk <vgvoleg@gmail.com>

doc/source/admin/configuration/config-guide.rst

@@ -168,6 +168,32 @@ directory.
 
     The grace period for the first heartbeat (in seconds).
 
+#. By default Mistral logs information about requests in HTTP action.
+   To hide request headers and endpoint response in logs apply
+   configuration like following::
+
+      [action_logging]
+      hide_response_body = True
+      hide_request_body = True
+      sensitive_headers = Header1, Header2
+
+   Example above will make Mistral hide all response's bodies and hide
+   Header1 and Header2 from requests in Mistral executor logs.
+
+   - **hide_response_body**
+
+    If this value is set to *True* then HTTP action response
+    body will be hidden in logs. Default is *False*
+
+   - **hide_request_body**
+
+    If this value is set to *True* then HTTP action request
+    body will be hidden in logs. Default is *False*
+
+   - **sensitive_headers**
+
+    List of sensitive headers that should be hidden in logs. Default is empty.
+
 #. Configure event publishers. Event publishers are plugins that are
    optionally installed in the same virtual environment as Mistral.
    Event notification can be configured for all workflow execution for one or

mistral/actions/std_actions.py

@@ -1,5 +1,6 @@
 # Copyright 2014 - Mirantis, Inc.
 # Copyright 2014 - StackStorm, Inc.
+# Copyright 2022 - NetCracker Technology Corp.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -26,6 +27,7 @@ import requests
 from mistral import exceptions as exc
 from mistral import utils
 from mistral.utils import javascript
+from mistral.utils import rest_utils
 from mistral.utils import ssh_utils
 from mistral_lib import actions
 
@@ -213,9 +215,9 @@ class HTTPAction(actions.Action):
             self.url,
             self.method,
             self.params,
-            self.body,
+            rest_utils.prepare_request_body_log(self.body),
             self.json,
-            self.headers,
+            rest_utils.clear_sensitive_headers(self.headers),
             self.cookies,
             self.auth,
             self.timeout,
@@ -255,7 +257,7 @@ class HTTPAction(actions.Action):
         LOG.info(
             "HTTP action response:\n%s\n%s",
             resp.status_code,
-            resp.content
+            rest_utils.prepare_response_body_log(resp.content)
         )
 
         # Represent important resp data as a dictionary.

mistral/config.py

@@ -2,6 +2,7 @@
 # Copyright 2016 - Brocade Communications Systems, Inc.
 # Copyright 2018 - Extreme Networks, Inc.
 # Copyright 2019 - Nokia Networks
+# Copyright 2022 - NetCracker Technology Corp.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -526,6 +527,30 @@ action_heartbeat_opts = [
     )
 ]
 
+action_logging_opts = [
+    cfg.BoolOpt(
+        'hide_response_body',
+        default=False,
+        help=(
+            'If this value is set to True then HTTP action response '
+            'body will be hidden in logs.'
+        )
+    ),
+    cfg.BoolOpt(
+        'hide_request_body',
+        default=False,
+        help=(
+            'If this value is set to True then HTTP action request '
+            'body will be hidden in logs.'
+        )
+    ),
+    cfg.ListOpt(
+        'sensitive_headers',
+        default=[],
+        help='List of sensitive headers that should be hidden in logs.'
+    )
+]
+
 coordination_opts = [
     cfg.StrOpt(
         'backend_url',
@@ -687,6 +712,7 @@ PECAN_GROUP = 'pecan'
 COORDINATION_GROUP = 'coordination'
 EXECUTION_EXPIRATION_POLICY_GROUP = 'execution_expiration_policy'
 ACTION_HEARTBEAT_GROUP = 'action_heartbeat'
+ACTION_LOGGING_GROUP = 'action_logging'
 PROFILER_GROUP = profiler.list_opts()[0][0]
 KEYCLOAK_OIDC_GROUP = "keycloak_oidc"
 YAQL_GROUP = "yaql"
@@ -719,6 +745,7 @@ CONF.register_opts(
     action_heartbeat_opts,
     group=ACTION_HEARTBEAT_GROUP
 )
+CONF.register_opts(action_logging_opts, group=ACTION_LOGGING_GROUP)
 CONF.register_opts(event_engine_opts, group=EVENT_ENGINE_GROUP)
 CONF.register_opts(notifier_opts, group=NOTIFIER_GROUP)
 CONF.register_opts(pecan_opts, group=PECAN_GROUP)
@@ -774,6 +801,7 @@ def list_opts():
         (KEYCLOAK_OIDC_GROUP, keycloak_oidc_opts),
         (YAQL_GROUP, yaql_opts),
         (ACTION_HEARTBEAT_GROUP, action_heartbeat_opts),
+        (ACTION_LOGGING_GROUP, action_logging_opts),
         (None, default_group_opts)
     ]
 

mistral/tests/unit/actions/test_std_http_action.py

@@ -33,6 +33,8 @@ DATA = {
     }
 }
 
+ACTION_LOGGER = 'mistral.actions.std_actions'
+
 
 def get_fake_response(content, code, **kwargs):
     return base.FakeHTTPResponse(
@@ -187,3 +189,86 @@ class HTTPActionTest(base.BaseTest):
         result = action.run(mock_ctx)
 
         self.assertIsNone(result['encoding'])
+
+    @mock.patch.object(requests, 'request')
+    def test_http_action_hides_request_body_if_needed(self, mocked_method):
+        self.override_config(
+            'hide_request_body',
+            True,
+            group='action_logging'
+        )
+
+        sensitive_data = 'I actually love anime.'
+
+        action = std.HTTPAction(url=URL, method='POST', body=sensitive_data)
+
+        mocked_method.return_value = get_fake_response(
+            content='', code=201
+        )
+        mock_ctx = mock.Mock()
+
+        with self.assertLogs(logger=ACTION_LOGGER, level='INFO') as logs:
+            action.run(mock_ctx)
+
+        self.assertEqual(2, len(logs.output))  # Request and response loglines
+
+        log = logs.output[0]  # Request log
+        msg = "Request body hidden due to action_logging configuration."
+        self.assertNotIn(sensitive_data, log)
+        self.assertIn(msg, log)
+
+    @mock.patch.object(requests, 'request')
+    def test_http_action_hides_request_headers_if_needed(self, mocked_method):
+        sensitive_header = 'Authorization'
+        self.override_config(
+            'sensitive_headers',
+            [sensitive_header],
+            group='action_logging'
+        )
+
+        headers = {
+            sensitive_header: 'Bearer 13e7aa3fc23e50bc1529dc136791d34d'
+        }
+
+        action = std.HTTPAction(url=URL, method='GET', headers=headers)
+
+        mocked_method.return_value = get_fake_response(
+            content='', code=200
+        )
+        mock_ctx = mock.Mock()
+
+        with self.assertLogs(logger=ACTION_LOGGER, level='INFO') as logs:
+            action.run(mock_ctx)
+
+        self.assertEqual(2, len(logs.output))  # Request and response loglines
+
+        log = logs.output[0]  # Request log
+        self.assertNotIn(headers[sensitive_header], log)
+        self.assertIn('headers={}', log)
+
+    @mock.patch.object(requests, 'request')
+    def test_http_action_hides_response_body_if_needed(self, mocked_method):
+        self.override_config(
+            'hide_response_body',
+            True,
+            group='action_logging'
+        )
+
+        action = std.HTTPAction(url=URL, method='GET')
+
+        sensitive_data = 'I actually love anime.'
+
+        mocked_method.return_value = get_fake_response(
+            content=sensitive_data, code=200
+        )
+        mock_ctx = mock.Mock()
+
+        with self.assertLogs(logger=ACTION_LOGGER, level='INFO') as logs:
+            action.run(mock_ctx)
+
+        self.assertEqual(2, len(logs.output))  # Request and response loglines
+
+        log = logs.output[1]  # Response log
+        msg = "Response body hidden due to action_logging configuration."
+        self.assertNotIn(sensitive_data, log)
+        self.assertIn(msg, log)

mistral/utils/rest_utils.py

@@ -1,6 +1,7 @@
 # Copyright 2014 - Mirantis, Inc.
 # Copyright 2016 - Brocade Communications Systems, Inc.
 # Copyright 2018 - Nokia, Inc.
+# Copyright 2022 - NetCracker Technology Corp.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -17,6 +18,7 @@
 import functools
 import json
 
+from oslo_config import cfg
 from oslo_db import exception as db_exc
 from oslo_log import log as logging
 import pecan
@@ -294,3 +296,25 @@ def load_deferred_fields(ex, fields):
         hasattr(ex, f)
 
     return ex
+
+
+def clear_sensitive_headers(dirty_data):
+    if not dirty_data:
+        return dirty_data
+    clean_data = dirty_data.copy()
+    for sensitive in cfg.CONF.action_logging.sensitive_headers:
+        if sensitive in clean_data:
+            del clean_data[sensitive]
+    return clean_data
+
+
+def prepare_request_body_log(body):
+    if not cfg.CONF.action_logging.hide_request_body:
+        return body
+    return 'Request body hidden due to action_logging configuration.'
+
+
+def prepare_response_body_log(body):
+    if not cfg.CONF.action_logging.hide_response_body:
+        return body
+    return 'Response body hidden due to action_logging configuration.'