openstack
/
pycadf
Code Issues Proposed changes

audit middleware in pycadf 

move audit middleware to pyCADF and allow it to support
oslo.messaging instead of openstack.common.notifier

Partial-Bug: #1280327
Change-Id: I7f0b706a91a61111147bc2b3c682dfdac01c0b7d
This commit is contained in:
Gordon Chung 2014-02-13 19:17:33 -05:00
parent 4499c2953c
commit fa802a753d
12 changed files with 443 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
pycadf/audit/api.py
View File

@ -18,9 +18,9 @@
import ast
import collections
import os
from oslo.config import cfg
import re
from oslo.config import cfg
from six.moves import configparser
from six.moves.urllib import parse as urlparse
@ -37,16 +37,14 @@ from pycadf import resource
from pycadf import tag
from pycadf import timestamp
#NOTE(gordc): remove cfg once we move over to this middleware version
CONF = cfg.CONF
opts = [
cfg.StrOpt('api_audit_map',
default='api_audit_map.conf',
help='File containing mapping for api paths and '
'service endpoints'),
]
opts = [cfg.StrOpt('api_audit_map',
default='api_audit_map.conf',
help='File containing mapping for api paths and '
'service endpoints')]
CONF.register_opts(opts, group='audit')
AuditMap = collections.namedtuple('AuditMap',
['path_kw',
'custom_actions',
@ -54,7 +52,7 @@ AuditMap = collections.namedtuple('AuditMap',
'default_target_endpoint_type'])
def _configure_audit_map():
def _configure_audit_map(cfg_file):
"""Configure to recognize and map known api paths."""
path_kw = {}
@ -62,10 +60,6 @@ def _configure_audit_map():
service_endpoints = {}
default_target_endpoint_type = None
cfg_file = CONF.audit.api_audit_map
if not os.path.exists(CONF.audit.api_audit_map):
cfg_file = cfg.CONF.find_file(CONF.audit.api_audit_map)
if cfg_file:
try:
map_conf = configparser.SafeConfigParser()
@ -119,12 +113,17 @@ class PycadfAuditApiConfigError(Exception):
class OpenStackAuditApi(object):
_MAP = None
Service = collections.namedtuple('Service',
['id', 'name', 'type', 'admin_endp',
'public_endp', 'private_endp'])
def __init__(self, map_file=None):
if map_file is None:
map_file = CONF.audit.api_audit_map
if not os.path.exists(CONF.audit.api_audit_map):
map_file = cfg.CONF.find_file(CONF.audit.api_audit_map)
self._MAP = _configure_audit_map(map_file)
def _get_action(self, req):
"""Take a given Request, parse url path to calculate action type.
@ -207,9 +206,6 @@ class OpenStackAuditApi(object):
return service_type + type_uri
def create_event(self, req, correlation_id):
if not self._MAP:
self._MAP = _configure_audit_map()
action = self._get_action(req)
initiator_host = host.Host(address=req.client_addr,
agent=req.user_agent)

0
pycadf/middleware/__init__.py Normal file
View File

45
pycadf/middleware/audit.py Normal file
View File

@ -0,0 +1,45 @@
# Copyright (c) 2013 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Attach open standard audit information to request.environ
AuditMiddleware filter should be place after Keystone's auth_token middleware
in the pipeline so that it can utilise the information Keystone provides.
"""
from pycadf.audit import api as cadf_api
from pycadf.middleware import notifier
class AuditMiddleware(notifier.RequestNotifier):
def __init__(self, app, **conf):
super(AuditMiddleware, self).__init__(app, **conf)
map_file = conf.get('audit_map_file', None)
self.cadf_audit = cadf_api.OpenStackAuditApi(map_file)
@notifier.log_and_ignore_error
def process_request(self, request):
self.cadf_audit.append_audit_event(request)
super(AuditMiddleware, self).process_request(request)
@notifier.log_and_ignore_error
def process_response(self, request, response,
exception=None, traceback=None):
self.cadf_audit.mod_audit_event(request, response)
super(AuditMiddleware, self).process_response(request, response,
exception, traceback)

56
pycadf/middleware/base.py Normal file
View File

@ -0,0 +1,56 @@
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Base class(es) for WSGI Middleware."""
import webob.dec
class Middleware(object):
"""Base WSGI middleware wrapper.
These classes require an application to be initialized that will be called
next. By default the middleware will simply call its wrapped app, or you
can override __call__ to customize its behavior.
"""
@classmethod
def factory(cls, global_conf, **local_conf):
"""Factory method for paste.deploy."""
return cls
def __init__(self, application):
self.application = application
def process_request(self, req):
"""Called on each request.
If this returns None, the next application down the stack will be
executed. If it returns a response then that response will be returned
and execution will stop here.
"""
return None
def process_response(self, response):
"""Do whatever you'd like to the response."""
return response
@webob.dec.wsgify
def __call__(self, req):
response = self.process_request(req)
if response:
return response
response = req.get_response(self.application)
return self.process_response(response)

140
pycadf/middleware/notifier.py Normal file
View File

@ -0,0 +1,140 @@
# Copyright (c) 2013 eNovance
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Send notifications on request
"""
import os.path
import sys
import traceback as tb
from oslo.config import cfg
import oslo.messaging
import six
import webob.dec
from pycadf.middleware import base
from pycadf.openstack.common import context
from pycadf.openstack.common.gettextutils import _ # noqa
LOG = None
def log_and_ignore_error(fn):
def wrapped(*args, **kwargs):
try:
return fn(*args, **kwargs)
except Exception as e:
if LOG:
LOG.exception(_('An exception occurred processing '
'the API call: %s ') % e)
return wrapped
class RequestNotifier(base.Middleware):
"""Send notification on request."""
@classmethod
def factory(cls, global_conf, **local_conf):
"""Factory method for paste.deploy."""
conf = global_conf.copy()
conf.update(local_conf)
def _factory(app):
return cls(app, **conf)
return _factory
def __init__(self, app, **conf):
global LOG
proj = cfg.CONF.project
TRANSPORT_ALIASES = {}
if proj:
log_mod = '%s.openstack.common.log' % proj
if log_mod in sys.modules:
LOG = sys.modules[log_mod].getLogger(__name__)
# Aliases to support backward compatibility
TRANSPORT_ALIASES = {
'%s.openstack.common.rpc.impl_kombu' % proj: 'rabbit',
'%s.openstack.common.rpc.impl_qpid' % proj: 'qpid',
'%s.openstack.common.rpc.impl_zmq' % proj: 'zmq',
'%s.rpc.impl_kombu' % proj: 'rabbit',
'%s.rpc.impl_qpid' % proj: 'qpid',
'%s.rpc.impl_zmq' % proj: 'zmq',
}
self.service_name = conf.get('service_name')
self.ignore_req_list = [x.upper().strip() for x in
conf.get('ignore_req_list', '').split(',')]
self.notifier = oslo.messaging.Notifier(
oslo.messaging.get_transport(cfg.CONF, aliases=TRANSPORT_ALIASES),
os.path.basename(sys.argv[0]))
super(RequestNotifier, self).__init__(app)
@staticmethod
def environ_to_dict(environ):
"""Following PEP 333, server variables are lower case, so don't
include them.
"""
return dict((k, v) for k, v in six.iteritems(environ)
if k.isupper())
@log_and_ignore_error
def process_request(self, request):
request.environ['HTTP_X_SERVICE_NAME'] = \
self.service_name or request.host
payload = {
'request': self.environ_to_dict(request.environ),
}
self.notifier.info(context.get_admin_context().to_dict(),
'http.request', payload)
@log_and_ignore_error
def process_response(self, request, response,
exception=None, traceback=None):
payload = {
'request': self.environ_to_dict(request.environ),
}
if response:
payload['response'] = {
'status': response.status,
'headers': response.headers,
}
if exception:
payload['exception'] = {
'value': repr(exception),
'traceback': tb.format_tb(traceback)
}
self.notifier.info(context.get_admin_context().to_dict(),
'http.response', payload)
@webob.dec.wsgify
def __call__(self, req):
if req.method in self.ignore_req_list:
return req.get_response(self.application)
else:
self.process_request(req)
try:
response = req.get_response(self.application)
except Exception:
exc_type, value, traceback = sys.exc_info()
self.process_response(req, None, value, traceback)
raise
else:
self.process_response(req, response)
return response

31
pycadf/tests/audit/test_api.py
View File

@ -14,8 +14,9 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo.config import cfg
import uuid
from oslo.config import cfg
import webob
from pycadf.audit import api
@ -43,14 +44,8 @@ class TestAuditApi(base.TestCase):
def setUp(self):
super(TestAuditApi, self).setUp()
# set nova CONF.host value
# Set a default location for the api_audit_map config file
cfg.CONF.set_override(
'api_audit_map',
self.path_get('etc/pycadf/api_audit_map.conf'),
group='audit'
)
self.audit_api = api.OpenStackAuditApi()
self.audit_api = api.OpenStackAuditApi(
'etc/pycadf/api_audit_map.conf')
def api_request(self, method, url):
self.ENV_HEADERS['REQUEST_METHOD'] = method
@ -60,6 +55,18 @@ class TestAuditApi(base.TestCase):
self.assertIn('CADF_EVENT_CORRELATION_ID', req.environ)
return req
def test_get_list_with_cfg(self):
cfg.CONF.set_override(
'api_audit_map',
self.path_get('etc/pycadf/api_audit_map.conf'),
group='audit')
self.audit_api = api.OpenStackAuditApi()
req = self.api_request('GET',
'http://admin_host:8774/v2/'
+ str(uuid.uuid4()) + '/servers/')
payload = req.environ['CADF_EVENT']
self.assertEqual(payload['action'], 'read/list')
def test_get_list(self):
req = self.api_request('GET', 'http://admin_host:8774/v2/'
+ str(uuid.uuid4()) + '/servers')
@ -125,8 +132,7 @@ class TestAuditApi(base.TestCase):
f.write("servers = server\n\n")
f.write("[service_endpoints]\n")
f.write("compute = service/compute")
cfg.CONF.set_override('api_audit_map', tmpfile, group='audit')
self.audit_api = api.OpenStackAuditApi()
self.audit_api = api.OpenStackAuditApi(tmpfile)
req = self.api_request('GET',
'http://unknown:8774/v2/'
@ -283,5 +289,4 @@ class TestAuditApiConf(base.TestCase):
f.write("api_paths = servers\n\n")
f.write("[service_endpoints]\n")
f.write("compute = service/compute")
cfg.CONF.set_override('api_audit_map', tmpfile, group='audit')
self.audit_api = api.OpenStackAuditApi()
self.audit_api = api.OpenStackAuditApi(tmpfile)

8
pycadf/tests/base.py
View File

@ -18,15 +18,21 @@
"""
import fixtures
import os.path
from oslo.config import cfg
import testtools
from oslo.config import cfg
from pycadf.openstack.common.fixture import moxstubout
class TestCase(testtools.TestCase):
def setUp(self):
super(TestCase, self).setUp()
self.tempdir = self.useFixture(fixtures.TempDir())
moxfixture = self.useFixture(moxstubout.MoxStubout())
self.mox = moxfixture.mox
self.stubs = moxfixture.stubs
cfg.CONF([], project='pycadf')
def path_get(self, project_file=None):

0
pycadf/tests/middleware/__init__.py Normal file
View File

159
pycadf/tests/middleware/test_audit.py Normal file
View File

@ -0,0 +1,159 @@
# Copyright (c) 2014 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
import webob
from pycadf.audit import api as cadf_api
from pycadf.middleware import audit
from pycadf.tests import base
class FakeApp(object):
def __call__(self, env, start_response):
body = 'Some response'
start_response('200 OK', [
('Content-Type', 'text/plain'),
('Content-Length', str(sum(map(len, body))))
])
return [body]
class FakeFailingApp(object):
def __call__(self, env, start_response):
raise Exception("It happens!")
@mock.patch('oslo.messaging.get_transport', mock.MagicMock())
class AuditMiddlewareTest(base.TestCase):
ENV_HEADERS = {'HTTP_X_SERVICE_CATALOG':
'''[{"endpoints_links": [],
"endpoints": [{"adminURL":
"http://host:8774/v2/admin",
"region": "RegionOne",
"publicURL":
"http://host:8774/v2/public",
"internalURL":
"http://host:8774/v2/internal",
"id": "resource_id"}],
"type": "compute",
"name": "nova"},]''',
'HTTP_X_USER_ID': 'user_id',
'HTTP_X_USER_NAME': 'user_name',
'HTTP_X_AUTH_TOKEN': 'token',
'HTTP_X_PROJECT_ID': 'tenant_id',
'HTTP_X_IDENTITY_STATUS': 'Confirmed'}
def setUp(self):
super(AuditMiddlewareTest, self).setUp()
self.map_file = 'etc/pycadf/api_audit_map.conf'
def test_api_request(self):
middleware = audit.AuditMiddleware(FakeApp(),
audit_map_file=
'etc/pycadf/api_audit_map.conf',
service_name='pycadf')
self.ENV_HEADERS['REQUEST_METHOD'] = 'GET'
req = webob.Request.blank('/foo/bar',
environ=self.ENV_HEADERS)
with mock.patch('oslo.messaging.Notifier.info') as notify:
middleware(req)
# Check first notification with only 'request'
call_args = notify.call_args_list[0][0]
self.assertEqual(call_args[1], 'http.request')
self.assertEqual(set(call_args[2].keys()),
set(['request']))
request = call_args[2]['request']
self.assertEqual(request['PATH_INFO'], '/foo/bar')
self.assertEqual(request['REQUEST_METHOD'], 'GET')
self.assertIn('CADF_EVENT', request)
self.assertEqual(request['CADF_EVENT']['outcome'], 'pending')
# Check second notification with request + response
call_args = notify.call_args_list[1][0]
self.assertEqual(call_args[1], 'http.response')
self.assertEqual(set(call_args[2].keys()),
set(['request', 'response']))
request = call_args[2]['request']
self.assertEqual(request['PATH_INFO'], '/foo/bar')
self.assertEqual(request['REQUEST_METHOD'], 'GET')
self.assertIn('CADF_EVENT', request)
self.assertEqual(request['CADF_EVENT']['outcome'], 'success')
def test_api_request_failure(self):
middleware = audit.AuditMiddleware(FakeFailingApp(),
audit_map_file=
'etc/pycadf/api_audit_map.conf',
service_name='pycadf')
self.ENV_HEADERS['REQUEST_METHOD'] = 'GET'
req = webob.Request.blank('/foo/bar',
environ=self.ENV_HEADERS)
with mock.patch('oslo.messaging.Notifier.info') as notify:
try:
middleware(req)
self.fail("Application exception has not been re-raised")
except Exception:
pass
# Check first notification with only 'request'
call_args = notify.call_args_list[0][0]
self.assertEqual(call_args[1], 'http.request')
self.assertEqual(set(call_args[2].keys()),
set(['request']))
request = call_args[2]['request']
self.assertEqual(request['PATH_INFO'], '/foo/bar')
self.assertEqual(request['REQUEST_METHOD'], 'GET')
self.assertIn('CADF_EVENT', request)
self.assertEqual(request['CADF_EVENT']['outcome'], 'pending')
# Check second notification with request + response
call_args = notify.call_args_list[1][0]
self.assertEqual(call_args[1], 'http.response')
self.assertEqual(set(call_args[2].keys()),
set(['request', 'exception']))
request = call_args[2]['request']
self.assertEqual(request['PATH_INFO'], '/foo/bar')
self.assertEqual(request['REQUEST_METHOD'], 'GET')
self.assertIn('CADF_EVENT', request)
self.assertEqual(request['CADF_EVENT']['outcome'], 'unknown')
def test_process_request_fail(self):
def func_error(self, req):
raise Exception('error')
self.stubs.Set(cadf_api.OpenStackAuditApi, 'append_audit_event',
func_error)
middleware = audit.AuditMiddleware(FakeApp(),
audit_map_file=
'etc/pycadf/api_audit_map.conf',
service_name='pycadf')
req = webob.Request.blank('/foo/bar',
environ={'REQUEST_METHOD': 'GET'})
middleware.process_request(req)
def test_process_response_fail(self):
def func_error(self, req, res):
raise Exception('error')
self.stubs.Set(cadf_api.OpenStackAuditApi, 'mod_audit_event',
func_error)
middleware = audit.AuditMiddleware(FakeApp(),
audit_map_file=
'etc/pycadf/api_audit_map.conf',
service_name='pycadf')
req = webob.Request.blank('/foo/bar',
environ={'REQUEST_METHOD': 'GET'})
middleware.process_response(req, webob.response.Response())

3
requirements.txt
View File

@ -1,7 +1,8 @@
Babel>=1.3
iso8601>=0.1.8
oslo.config>=1.2.0
netaddr>=0.7.6
oslo.config>=1.2.0
oslo.messaging>=1.3.0a4
pytz>=2010h
six>=1.4.1
WebOb>=1.2.3

1
setup.cfg
View File

@ -16,7 +16,6 @@ classifier =
Programming Language :: Python
Programming Language :: Python :: 2.6
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3.3
[files]
packages =

2
test-requirements.txt
View File

@ -4,6 +4,8 @@ hacking>=0.8.0,<0.9
coverage>=3.6
discover
fixtures>=0.3.14
mock>=1.0
mox>=0.5.3
python-subunit>=0.0.18
testrepository>=0.0.17
testscenarios>=0.4