Implemented middleware to augment POSTed metrics with keystone metadata

Change-Id: I4bcb351141200eb113d1b6e990a5ef257aaa4748

kiloeyes/middleware/keystone_augmenter.py

@@ -0,0 +1,78 @@
+# Copyright 2016 Cornell University
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import StringIO
+try:
+    import ujson as json
+except ImportError:
+    import json
+
+
+class KeystoneAugmenter(object):
+    """middleware that adds keystone data to POST-ed metrics.
+
+       This middleware must be placed in the server pipeline immediately
+       following the keystone middleware. If the request coming to the server
+       is a POST request on the /v2.0/metrics endpoint, the middleware extracts
+       keystone fields from the request,and adds them to the body of the
+       metrics JSON objects.
+    """
+    def __init__(self, app, conf):
+        self.app = app
+        self.conf = conf
+
+    def add_keystone_to_metrics(self, env):
+        body = env['wsgi.input'].read()
+        metrics = json.loads(body)
+
+        # Add keystone data to metrics
+        if isinstance(metrics, list):
+            for metric in metrics:
+                metric['tenant'] = env['HTTP_X_TENANT']
+                metric['tenant_id'] = env['HTTP_X_TENANT_ID']
+                metric['user'] = env['HTTP_X_USER']
+                metric['user_agent'] = env['HTTP_USER_AGENT']
+                metric['project_id'] = env['HTTP_X_PROJECT_ID']
+                metric['user_id'] = env['HTTP_X_USER_ID']
+
+        env['wsgi.input'] = StringIO.StringIO(json.dumps(metrics))
+        return env
+
+    def __call__(self, env, start_response):
+        if (env.get('PATH_INFO', '').startswith('/v2.0/metrics') and
+                env.get('REQUEST_METHOD', '') == 'POST'):
+            # We only check the requests which are posting against metrics
+            # endpoint
+            try:
+                env = self.add_keystone_to_metrics(env)
+
+                return self.app(env, start_response)
+            except Exception:
+                pass
+            # It is either invalid or exceptioned out while parsing json
+            # we will send the request back with 400.
+            start_response("400 Bad Request", [], '')
+            return []
+        else:
+            # not a metric post request, move on.
+            return self.app(env, start_response)
+
+
+def filter_factory(global_conf, **local_conf):
+
+    def augmenter_filter(app):
+        return KeystoneAugmenter(app, local_conf)
+
+    return augmenter_filter

kiloeyes/tests/middleware/test_augmenter.py

@@ -0,0 +1,87 @@
+# Copyright 2016 Cornell University
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from kiloeyes.middleware import keystone_augmenter
+from oslotest import base
+
+import StringIO
+
+try:
+        import ujson as json
+except ImportError:
+        import json
+
+
+class TestKeystoneAugmenter(base.BaseTestCase):
+
+        def setUp(self):
+                super(TestKeystoneAugmenter, self).setUp()
+                self.augmenter = keystone_augmenter.KeystoneAugmenter({}, {})
+
+        def test_call(self):
+                test_input = [
+                    {
+                        'name': 'metric1',
+                        'timestamp': 125213525352,
+                        'dimensions': {'service': 'test_service'}
+                    },
+                    {
+                        'name': 'metric2',
+                        'timestamp': 135098109530,
+                        'dimensions': {'service': 'test_service'}
+                    }
+                ]
+
+                input_json = StringIO.StringIO(json.dumps(test_input))
+
+                env = {
+                    'wsgi.input': input_json,
+                    'HTTP_X_TENANT': 'test',
+                    'HTTP_X_TENANT_ID': 'testid1',
+                    'HTTP_X_USER': 'test_user',
+                    'HTTP_USER_AGENT': 'kiloeyes-tester',
+                    'HTTP_X_PROJECT_ID': 'projidtest2',
+                    'HTTP_X_USER_ID': 'testuid'
+                }
+
+                metrics_expected = [
+                    {
+                        'name': 'metric1',
+                        'timestamp': 125213525352,
+                        'dimensions': {'service': 'test_service'},
+                        'tenant': 'test',
+                        'tenant_id': 'testid1',
+                        'user': 'test_user',
+                        'user_agent': 'kiloeyes-tester',
+                        'project_id': 'projidtest2',
+                        'user_id': 'testuid'
+                    },
+                    {
+                        'name': 'metric2',
+                        'timestamp': 135098109530,
+                        'dimensions': {'service': 'test_service'},
+                        'tenant': 'test',
+                        'tenant_id': 'testid1',
+                        'user': 'test_user',
+                        'user_agent': 'kiloeyes-tester',
+                        'project_id': 'projidtest2',
+                        'user_id': 'testuid'
+                    }
+                ]
+
+                augmented_env = self.augmenter.add_keystone_to_metrics(env)
+
+                metrics_res = json.loads(augmented_env['wsgi.input'].read())
+
+                self.assertEqual(metrics_expected, metrics_res)

setup.cfg

@@ -66,6 +66,7 @@ paste.filter_factory =
     inspector = kiloeyes.middleware.inspector:filter_factory
     metric_validator = kiloeyes.middleware.metric_validator:filter_factory
     meter_validator = kiloeyes.middleware.meter_validator:filter_factory
+    keystone_augmenter = kiloeyes.middleware.keystone_augmenter:filter_factory
 
 [pbr]
 warnerrors = True