move add event creation logic to keystonemiddleware
currently, the logic to create audit events is contained in pyCADF. this makes it extremely difficult to modify as changes often require edits to two different libraries, and subsequent releases of two different libraries. additionally, it makes it impossible to test the full path. this patch attempts to minimise the chaos. Change-Id: Ic0ca193139acaa751cd96e7fadad2223a84088c8
This commit is contained in:
parent
1c16d18ff6
commit
f3a98370fe
|
@ -19,9 +19,12 @@ in the pipeline so that it can utilise the information the Identity server
|
||||||
provides.
|
provides.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import ast
|
||||||
|
import collections
|
||||||
import functools
|
import functools
|
||||||
import logging
|
import logging
|
||||||
import os.path
|
import os.path
|
||||||
|
import re
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
|
@ -30,8 +33,20 @@ try:
|
||||||
messaging = True
|
messaging = True
|
||||||
except ImportError:
|
except ImportError:
|
||||||
messaging = False
|
messaging = False
|
||||||
import pycadf
|
from pycadf import cadftaxonomy as taxonomy
|
||||||
from pycadf.audit import api
|
from pycadf import cadftype
|
||||||
|
from pycadf import credential
|
||||||
|
from pycadf import endpoint
|
||||||
|
from pycadf import eventfactory as factory
|
||||||
|
from pycadf import host
|
||||||
|
from pycadf import identifier
|
||||||
|
from pycadf import reason
|
||||||
|
from pycadf import reporterstep
|
||||||
|
from pycadf import resource
|
||||||
|
from pycadf import tag
|
||||||
|
from pycadf import timestamp
|
||||||
|
from six.moves import configparser
|
||||||
|
from six.moves.urllib import parse as urlparse
|
||||||
import webob.dec
|
import webob.dec
|
||||||
|
|
||||||
from keystonemiddleware.i18n import _LE, _LI
|
from keystonemiddleware.i18n import _LE, _LI
|
||||||
|
@ -52,6 +67,226 @@ def _log_and_ignore_error(fn):
|
||||||
return wrapper
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
|
Service = collections.namedtuple('Service',
|
||||||
|
['id', 'name', 'type', 'admin_endp',
|
||||||
|
'public_endp', 'private_endp'])
|
||||||
|
|
||||||
|
|
||||||
|
AuditMap = collections.namedtuple('AuditMap',
|
||||||
|
['path_kw',
|
||||||
|
'custom_actions',
|
||||||
|
'service_endpoints',
|
||||||
|
'default_target_endpoint_type'])
|
||||||
|
|
||||||
|
|
||||||
|
class OpenStackAuditApi(object):
|
||||||
|
|
||||||
|
def __init__(self, cfg_file):
|
||||||
|
"""Configure to recognize and map known api paths."""
|
||||||
|
path_kw = {}
|
||||||
|
custom_actions = {}
|
||||||
|
endpoints = {}
|
||||||
|
default_target_endpoint_type = None
|
||||||
|
|
||||||
|
if cfg_file:
|
||||||
|
try:
|
||||||
|
map_conf = configparser.SafeConfigParser()
|
||||||
|
map_conf.readfp(open(cfg_file))
|
||||||
|
|
||||||
|
try:
|
||||||
|
default_target_endpoint_type = map_conf.get(
|
||||||
|
'DEFAULT', 'target_endpoint_type')
|
||||||
|
except configparser.NoOptionError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
custom_actions = dict(map_conf.items('custom_actions'))
|
||||||
|
except configparser.Error:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
path_kw = dict(map_conf.items('path_keywords'))
|
||||||
|
except configparser.Error:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
endpoints = dict(map_conf.items('service_endpoints'))
|
||||||
|
except configparser.Error:
|
||||||
|
pass
|
||||||
|
except configparser.ParsingError as err:
|
||||||
|
raise PycadfAuditApiConfigError(
|
||||||
|
'Error parsing audit map file: %s' % err)
|
||||||
|
self._MAP = AuditMap(
|
||||||
|
path_kw=path_kw, custom_actions=custom_actions,
|
||||||
|
service_endpoints=endpoints,
|
||||||
|
default_target_endpoint_type=default_target_endpoint_type)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _clean_path(value):
|
||||||
|
"""Clean path if path has json suffix."""
|
||||||
|
return value[:-5] if value.endswith('.json') else value
|
||||||
|
|
||||||
|
def get_action(self, req):
|
||||||
|
"""Take a given Request, parse url path to calculate action type.
|
||||||
|
|
||||||
|
Depending on req.method:
|
||||||
|
if POST: path ends with 'action', read the body and use as action;
|
||||||
|
path ends with known custom_action, take action from config;
|
||||||
|
request ends with known path, assume is create action;
|
||||||
|
request ends with unknown path, assume is update action.
|
||||||
|
if GET: request ends with known path, assume is list action;
|
||||||
|
request ends with unknown path, assume is read action.
|
||||||
|
if PUT, assume update action.
|
||||||
|
if DELETE, assume delete action.
|
||||||
|
if HEAD, assume read action.
|
||||||
|
|
||||||
|
"""
|
||||||
|
path = req.path[:-1] if req.path.endswith('/') else req.path
|
||||||
|
url_ending = self._clean_path(path[path.rfind('/') + 1:])
|
||||||
|
method = req.method
|
||||||
|
|
||||||
|
if url_ending + '/' + method.lower() in self._MAP.custom_actions:
|
||||||
|
action = self._MAP.custom_actions[url_ending + '/' +
|
||||||
|
method.lower()]
|
||||||
|
elif url_ending in self._MAP.custom_actions:
|
||||||
|
action = self._MAP.custom_actions[url_ending]
|
||||||
|
elif method == 'POST':
|
||||||
|
if url_ending == 'action':
|
||||||
|
try:
|
||||||
|
if req.json:
|
||||||
|
body_action = list(req.json.keys())[0]
|
||||||
|
action = taxonomy.ACTION_UPDATE + '/' + body_action
|
||||||
|
else:
|
||||||
|
action = taxonomy.ACTION_CREATE
|
||||||
|
except ValueError:
|
||||||
|
action = taxonomy.ACTION_CREATE
|
||||||
|
elif url_ending not in self._MAP.path_kw:
|
||||||
|
action = taxonomy.ACTION_UPDATE
|
||||||
|
else:
|
||||||
|
action = taxonomy.ACTION_CREATE
|
||||||
|
elif method == 'GET':
|
||||||
|
if url_ending in self._MAP.path_kw:
|
||||||
|
action = taxonomy.ACTION_LIST
|
||||||
|
else:
|
||||||
|
action = taxonomy.ACTION_READ
|
||||||
|
elif method == 'PUT' or method == 'PATCH':
|
||||||
|
action = taxonomy.ACTION_UPDATE
|
||||||
|
elif method == 'DELETE':
|
||||||
|
action = taxonomy.ACTION_DELETE
|
||||||
|
elif method == 'HEAD':
|
||||||
|
action = taxonomy.ACTION_READ
|
||||||
|
else:
|
||||||
|
action = taxonomy.UNKNOWN
|
||||||
|
|
||||||
|
return action
|
||||||
|
|
||||||
|
def _get_service_info(self, endp):
|
||||||
|
service = Service(
|
||||||
|
type=self._MAP.service_endpoints.get(
|
||||||
|
endp['type'],
|
||||||
|
taxonomy.UNKNOWN),
|
||||||
|
name=endp['name'],
|
||||||
|
id=identifier.norm_ns(endp['endpoints'][0].get('id',
|
||||||
|
endp['name'])),
|
||||||
|
admin_endp=endpoint.Endpoint(
|
||||||
|
name='admin',
|
||||||
|
url=endp['endpoints'][0]['adminURL']),
|
||||||
|
private_endp=endpoint.Endpoint(
|
||||||
|
name='private',
|
||||||
|
url=endp['endpoints'][0]['internalURL']),
|
||||||
|
public_endp=endpoint.Endpoint(
|
||||||
|
name='public',
|
||||||
|
url=endp['endpoints'][0]['publicURL']))
|
||||||
|
|
||||||
|
return service
|
||||||
|
|
||||||
|
def _build_typeURI(self, req, service_type):
|
||||||
|
"""Build typeURI of target
|
||||||
|
|
||||||
|
Combines service type and corresponding path for greater detail.
|
||||||
|
"""
|
||||||
|
type_uri = ''
|
||||||
|
prev_key = None
|
||||||
|
for key in re.split('/', req.path):
|
||||||
|
key = self._clean_path(key)
|
||||||
|
if key in self._MAP.path_kw:
|
||||||
|
type_uri += '/' + key
|
||||||
|
elif prev_key in self._MAP.path_kw:
|
||||||
|
type_uri += '/' + self._MAP.path_kw[prev_key]
|
||||||
|
prev_key = key
|
||||||
|
return service_type + type_uri
|
||||||
|
|
||||||
|
def _build_target(self, req, service):
|
||||||
|
"""Build target resource."""
|
||||||
|
target_typeURI = (
|
||||||
|
self._build_typeURI(req, service.type)
|
||||||
|
if service.type != taxonomy.UNKNOWN else service.type)
|
||||||
|
target = resource.Resource(typeURI=target_typeURI,
|
||||||
|
id=service.id, name=service.name)
|
||||||
|
if service.admin_endp:
|
||||||
|
target.add_address(service.admin_endp)
|
||||||
|
if service.private_endp:
|
||||||
|
target.add_address(service.private_endp)
|
||||||
|
if service.public_endp:
|
||||||
|
target.add_address(service.public_endp)
|
||||||
|
return target
|
||||||
|
|
||||||
|
def get_target_resource(self, req):
|
||||||
|
"""Retrieve target information
|
||||||
|
|
||||||
|
If discovery is enabled, target will attempt to retrieve information
|
||||||
|
from service catalog. If not, the information will be taken from
|
||||||
|
given config file.
|
||||||
|
"""
|
||||||
|
service_info = Service(type=taxonomy.UNKNOWN, name=taxonomy.UNKNOWN,
|
||||||
|
id=taxonomy.UNKNOWN, admin_endp=None,
|
||||||
|
private_endp=None, public_endp=None)
|
||||||
|
try:
|
||||||
|
catalog = ast.literal_eval(
|
||||||
|
req.environ['HTTP_X_SERVICE_CATALOG'])
|
||||||
|
except KeyError:
|
||||||
|
raise PycadfAuditApiConfigError(
|
||||||
|
'Service catalog is missing. '
|
||||||
|
'Cannot discover target information')
|
||||||
|
|
||||||
|
default_endpoint = None
|
||||||
|
for endp in catalog:
|
||||||
|
admin_urlparse = urlparse.urlparse(
|
||||||
|
endp['endpoints'][0]['adminURL'])
|
||||||
|
public_urlparse = urlparse.urlparse(
|
||||||
|
endp['endpoints'][0]['publicURL'])
|
||||||
|
req_url = urlparse.urlparse(req.host_url)
|
||||||
|
if (req_url.netloc == admin_urlparse.netloc
|
||||||
|
or req_url.netloc == public_urlparse.netloc):
|
||||||
|
service_info = self._get_service_info(endp)
|
||||||
|
break
|
||||||
|
elif (self._MAP.default_target_endpoint_type and
|
||||||
|
endp['type'] == self._MAP.default_target_endpoint_type):
|
||||||
|
default_endpoint = endp
|
||||||
|
else:
|
||||||
|
if default_endpoint:
|
||||||
|
service_info = self._get_service_info(default_endpoint)
|
||||||
|
return self._build_target(req, service_info)
|
||||||
|
|
||||||
|
|
||||||
|
class ClientResource(resource.Resource):
|
||||||
|
def __init__(self, project_id=None, **kwargs):
|
||||||
|
super(ClientResource, self).__init__(**kwargs)
|
||||||
|
if project_id is not None:
|
||||||
|
self.project_id = project_id
|
||||||
|
|
||||||
|
|
||||||
|
class KeystoneCredential(credential.Credential):
|
||||||
|
def __init__(self, identity_status=None, **kwargs):
|
||||||
|
super(KeystoneCredential, self).__init__(**kwargs)
|
||||||
|
if identity_status is not None:
|
||||||
|
self.identity_status = identity_status
|
||||||
|
|
||||||
|
|
||||||
|
class PycadfAuditApiConfigError(Exception):
|
||||||
|
"""Error raised when pyCADF fails to configure correctly."""
|
||||||
|
|
||||||
|
|
||||||
class AuditMiddleware(object):
|
class AuditMiddleware(object):
|
||||||
"""Create an audit event based on request/response.
|
"""Create an audit event based on request/response.
|
||||||
|
|
||||||
|
@ -83,8 +318,7 @@ class AuditMiddleware(object):
|
||||||
self._service_name = conf.get('service_name')
|
self._service_name = conf.get('service_name')
|
||||||
self._ignore_req_list = [x.upper().strip() for x in
|
self._ignore_req_list = [x.upper().strip() for x in
|
||||||
conf.get('ignore_req_list', '').split(',')]
|
conf.get('ignore_req_list', '').split(',')]
|
||||||
self._cadf_audit = api.OpenStackAuditApi(
|
self._cadf_audit = OpenStackAuditApi(conf.get('audit_map_file'))
|
||||||
conf.get('audit_map_file'))
|
|
||||||
|
|
||||||
transport_aliases = self._get_aliases(cfg.CONF.project)
|
transport_aliases = self._get_aliases(cfg.CONF.project)
|
||||||
if messaging:
|
if messaging:
|
||||||
|
@ -107,40 +341,65 @@ class AuditMiddleware(object):
|
||||||
'event_type': event_type,
|
'event_type': event_type,
|
||||||
'payload': payload})
|
'payload': payload})
|
||||||
|
|
||||||
|
def _create_event(self, req):
|
||||||
|
correlation_id = identifier.generate_uuid()
|
||||||
|
action = self._cadf_audit.get_action(req)
|
||||||
|
|
||||||
|
initiator = ClientResource(
|
||||||
|
typeURI=taxonomy.ACCOUNT_USER,
|
||||||
|
id=identifier.norm_ns(str(req.environ['HTTP_X_USER_ID'])),
|
||||||
|
name=req.environ['HTTP_X_USER_NAME'],
|
||||||
|
host=host.Host(address=req.client_addr, agent=req.user_agent),
|
||||||
|
credential=KeystoneCredential(
|
||||||
|
token=req.environ['HTTP_X_AUTH_TOKEN'],
|
||||||
|
identity_status=req.environ['HTTP_X_IDENTITY_STATUS']),
|
||||||
|
project_id=identifier.norm_ns(req.environ['HTTP_X_PROJECT_ID']))
|
||||||
|
target = self._cadf_audit.get_target_resource(req)
|
||||||
|
|
||||||
|
event = factory.EventFactory().new_event(
|
||||||
|
eventType=cadftype.EVENTTYPE_ACTIVITY,
|
||||||
|
outcome=taxonomy.OUTCOME_PENDING,
|
||||||
|
action=action,
|
||||||
|
initiator=initiator,
|
||||||
|
target=target,
|
||||||
|
observer=resource.Resource(id='target'))
|
||||||
|
event.requestPath = req.path_qs
|
||||||
|
event.add_tag(tag.generate_name_value_tag('correlation_id',
|
||||||
|
correlation_id))
|
||||||
|
# cache model in request to allow tracking of transistive steps.
|
||||||
|
req.environ['cadf_event'] = event
|
||||||
|
return event
|
||||||
|
|
||||||
@_log_and_ignore_error
|
@_log_and_ignore_error
|
||||||
def _process_request(self, request):
|
def _process_request(self, request):
|
||||||
correlation_id = pycadf.identifier.generate_uuid()
|
event = self._create_event(request)
|
||||||
event = self._cadf_audit.create_event(request, correlation_id)
|
|
||||||
request.environ['cadf_event'] = event
|
|
||||||
|
|
||||||
self._emit_audit(context.get_admin_context().to_dict(),
|
self._emit_audit(context.get_admin_context().to_dict(),
|
||||||
'audit.http.request', event.as_dict())
|
'audit.http.request', event.as_dict())
|
||||||
|
|
||||||
@_log_and_ignore_error
|
@_log_and_ignore_error
|
||||||
def _process_response(self, request, response=None):
|
def _process_response(self, request, response=None):
|
||||||
|
# NOTE(gordc): handle case where error processing request
|
||||||
if 'cadf_event' not in request.environ:
|
if 'cadf_event' not in request.environ:
|
||||||
# NOTE(gordc): handle case where error processing request
|
self._create_event(request)
|
||||||
correlation_id = pycadf.identifier.generate_uuid()
|
event = request.environ['cadf_event']
|
||||||
event = self._cadf_audit.create_event(request, correlation_id)
|
|
||||||
else:
|
|
||||||
event = request.environ['cadf_event']
|
|
||||||
|
|
||||||
if response:
|
if response:
|
||||||
if response.status_int >= 200 and response.status_int < 400:
|
if response.status_int >= 200 and response.status_int < 400:
|
||||||
result = pycadf.cadftaxonomy.OUTCOME_SUCCESS
|
result = taxonomy.OUTCOME_SUCCESS
|
||||||
else:
|
else:
|
||||||
result = pycadf.cadftaxonomy.OUTCOME_FAILURE
|
result = taxonomy.OUTCOME_FAILURE
|
||||||
event.reason = pycadf.reason.Reason(
|
event.reason = reason.Reason(
|
||||||
reasonType='HTTP', reasonCode=str(response.status_int))
|
reasonType='HTTP', reasonCode=str(response.status_int))
|
||||||
else:
|
else:
|
||||||
result = pycadf.cadftaxonomy.UNKNOWN
|
result = taxonomy.UNKNOWN
|
||||||
|
|
||||||
event.outcome = result
|
event.outcome = result
|
||||||
event.add_reporterstep(
|
event.add_reporterstep(
|
||||||
pycadf.reporterstep.Reporterstep(
|
reporterstep.Reporterstep(
|
||||||
role=pycadf.cadftype.REPORTER_ROLE_MODIFIER,
|
role=cadftype.REPORTER_ROLE_MODIFIER,
|
||||||
reporter=pycadf.resource.Resource(id='target'),
|
reporter=resource.Resource(id='target'),
|
||||||
reporterTime=pycadf.timestamp.get_utc_now()))
|
reporterTime=timestamp.get_utc_now()))
|
||||||
|
|
||||||
self._emit_audit(context.get_admin_context().to_dict(),
|
self._emit_audit(context.get_admin_context().to_dict(),
|
||||||
'audit.http.response', event.as_dict())
|
'audit.http.response', event.as_dict())
|
||||||
|
|
|
@ -13,9 +13,11 @@
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import uuid
|
||||||
|
|
||||||
import mock
|
import mock
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
|
from pycadf import identifier
|
||||||
import testtools
|
import testtools
|
||||||
from testtools import matchers
|
from testtools import matchers
|
||||||
import webob
|
import webob
|
||||||
|
@ -38,28 +40,44 @@ class FakeFailingApp(object):
|
||||||
raise Exception('It happens!')
|
raise Exception('It happens!')
|
||||||
|
|
||||||
|
|
||||||
@mock.patch('oslo.messaging.get_transport', mock.MagicMock())
|
class BaseAuditMiddlewareTest(testtools.TestCase):
|
||||||
class AuditMiddlewareTest(testtools.TestCase):
|
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
super(AuditMiddlewareTest, self).setUp()
|
super(BaseAuditMiddlewareTest, self).setUp()
|
||||||
(self.fd, self.audit_map) = tempfile.mkstemp()
|
self.fd, self.audit_map = tempfile.mkstemp()
|
||||||
|
|
||||||
|
with open(self.audit_map, "w") as f:
|
||||||
|
f.write("[custom_actions]\n")
|
||||||
|
f.write("reboot = start/reboot\n")
|
||||||
|
f.write("os-migrations/get = read\n\n")
|
||||||
|
f.write("[path_keywords]\n")
|
||||||
|
f.write("action = None\n")
|
||||||
|
f.write("os-hosts = host\n")
|
||||||
|
f.write("os-migrations = None\n")
|
||||||
|
f.write("reboot = None\n")
|
||||||
|
f.write("servers = server\n\n")
|
||||||
|
f.write("[service_endpoints]\n")
|
||||||
|
f.write("compute = service/compute")
|
||||||
|
|
||||||
cfg.CONF([], project='keystonemiddleware')
|
cfg.CONF([], project='keystonemiddleware')
|
||||||
|
|
||||||
|
self.middleware = audit.AuditMiddleware(
|
||||||
|
FakeApp(), audit_map_file=self.audit_map,
|
||||||
|
service_name='pycadf')
|
||||||
|
|
||||||
self.addCleanup(lambda: os.close(self.fd))
|
self.addCleanup(lambda: os.close(self.fd))
|
||||||
self.addCleanup(cfg.CONF.reset)
|
self.addCleanup(cfg.CONF.reset)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _get_environ_header(req_type):
|
def get_environ_header(req_type):
|
||||||
env_headers = {'HTTP_X_SERVICE_CATALOG':
|
env_headers = {'HTTP_X_SERVICE_CATALOG':
|
||||||
'''[{"endpoints_links": [],
|
'''[{"endpoints_links": [],
|
||||||
"endpoints": [{"adminURL":
|
"endpoints": [{"adminURL":
|
||||||
"http://host:8774/v2/admin",
|
"http://admin_host:8774",
|
||||||
"region": "RegionOne",
|
"region": "RegionOne",
|
||||||
"publicURL":
|
"publicURL":
|
||||||
"http://host:8774/v2/public",
|
"http://public_host:8774",
|
||||||
"internalURL":
|
"internalURL":
|
||||||
"http://host:8774/v2/internal",
|
"http://internal_host:8774",
|
||||||
"id": "resource_id"}],
|
"id": "resource_id"}],
|
||||||
"type": "compute",
|
"type": "compute",
|
||||||
"name": "nova"},]''',
|
"name": "nova"},]''',
|
||||||
|
@ -71,15 +89,15 @@ class AuditMiddlewareTest(testtools.TestCase):
|
||||||
env_headers['REQUEST_METHOD'] = req_type
|
env_headers['REQUEST_METHOD'] = req_type
|
||||||
return env_headers
|
return env_headers
|
||||||
|
|
||||||
|
|
||||||
|
@mock.patch('oslo.messaging.get_transport', mock.MagicMock())
|
||||||
|
class AuditMiddlewareTest(BaseAuditMiddlewareTest):
|
||||||
|
|
||||||
def test_api_request(self):
|
def test_api_request(self):
|
||||||
middleware = audit.AuditMiddleware(
|
|
||||||
FakeApp(),
|
|
||||||
audit_map_file=self.audit_map,
|
|
||||||
service_name='pycadf')
|
|
||||||
req = webob.Request.blank('/foo/bar',
|
req = webob.Request.blank('/foo/bar',
|
||||||
environ=self._get_environ_header('GET'))
|
environ=self.get_environ_header('GET'))
|
||||||
with mock.patch('oslo.messaging.Notifier.info') as notify:
|
with mock.patch('oslo.messaging.Notifier.info') as notify:
|
||||||
middleware(req)
|
self.middleware(req)
|
||||||
# Check first notification with only 'request'
|
# Check first notification with only 'request'
|
||||||
call_args = notify.call_args_list[0][0]
|
call_args = notify.call_args_list[0][0]
|
||||||
self.assertEqual('audit.http.request', call_args[1])
|
self.assertEqual('audit.http.request', call_args[1])
|
||||||
|
@ -97,15 +115,15 @@ class AuditMiddlewareTest(testtools.TestCase):
|
||||||
self.assertIn('reporterchain', call_args[2])
|
self.assertIn('reporterchain', call_args[2])
|
||||||
|
|
||||||
def test_api_request_failure(self):
|
def test_api_request_failure(self):
|
||||||
middleware = audit.AuditMiddleware(
|
self.middleware = audit.AuditMiddleware(
|
||||||
FakeFailingApp(),
|
FakeFailingApp(),
|
||||||
audit_map_file=self.audit_map,
|
audit_map_file=self.audit_map,
|
||||||
service_name='pycadf')
|
service_name='pycadf')
|
||||||
req = webob.Request.blank('/foo/bar',
|
req = webob.Request.blank('/foo/bar',
|
||||||
environ=self._get_environ_header('GET'))
|
environ=self.get_environ_header('GET'))
|
||||||
with mock.patch('oslo.messaging.Notifier.info') as notify:
|
with mock.patch('oslo.messaging.Notifier.info') as notify:
|
||||||
try:
|
try:
|
||||||
middleware(req)
|
self.middleware(req)
|
||||||
self.fail('Application exception has not been re-raised')
|
self.fail('Application exception has not been re-raised')
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
@ -124,47 +142,39 @@ class AuditMiddlewareTest(testtools.TestCase):
|
||||||
self.assertIn('reporterchain', call_args[2])
|
self.assertIn('reporterchain', call_args[2])
|
||||||
|
|
||||||
def test_process_request_fail(self):
|
def test_process_request_fail(self):
|
||||||
middleware = audit.AuditMiddleware(
|
|
||||||
FakeApp(),
|
|
||||||
audit_map_file=self.audit_map,
|
|
||||||
service_name='pycadf')
|
|
||||||
req = webob.Request.blank('/foo/bar',
|
req = webob.Request.blank('/foo/bar',
|
||||||
environ=self._get_environ_header('GET'))
|
environ=self.get_environ_header('GET'))
|
||||||
with mock.patch('oslo.messaging.Notifier.info',
|
with mock.patch('oslo.messaging.Notifier.info',
|
||||||
side_effect=Exception('error')) as notify:
|
side_effect=Exception('error')) as notify:
|
||||||
middleware._process_request(req)
|
self.middleware._process_request(req)
|
||||||
self.assertTrue(notify.called)
|
self.assertTrue(notify.called)
|
||||||
|
|
||||||
def test_process_response_fail(self):
|
def test_process_response_fail(self):
|
||||||
middleware = audit.AuditMiddleware(
|
|
||||||
FakeApp(),
|
|
||||||
audit_map_file=self.audit_map,
|
|
||||||
service_name='pycadf')
|
|
||||||
req = webob.Request.blank('/foo/bar',
|
req = webob.Request.blank('/foo/bar',
|
||||||
environ=self._get_environ_header('GET'))
|
environ=self.get_environ_header('GET'))
|
||||||
with mock.patch('oslo.messaging.Notifier.info',
|
with mock.patch('oslo.messaging.Notifier.info',
|
||||||
side_effect=Exception('error')) as notify:
|
side_effect=Exception('error')) as notify:
|
||||||
middleware._process_response(req, webob.response.Response())
|
self.middleware._process_response(req, webob.response.Response())
|
||||||
self.assertTrue(notify.called)
|
self.assertTrue(notify.called)
|
||||||
|
|
||||||
def test_ignore_req_opt(self):
|
def test_ignore_req_opt(self):
|
||||||
middleware = audit.AuditMiddleware(FakeApp(),
|
self.middleware = audit.AuditMiddleware(FakeApp(),
|
||||||
audit_map_file=self.audit_map,
|
audit_map_file=self.audit_map,
|
||||||
ignore_req_list='get, PUT')
|
ignore_req_list='get, PUT')
|
||||||
req = webob.Request.blank('/skip/foo',
|
req = webob.Request.blank('/skip/foo',
|
||||||
environ=self._get_environ_header('GET'))
|
environ=self.get_environ_header('GET'))
|
||||||
req1 = webob.Request.blank('/skip/foo',
|
req1 = webob.Request.blank('/skip/foo',
|
||||||
environ=self._get_environ_header('PUT'))
|
environ=self.get_environ_header('PUT'))
|
||||||
req2 = webob.Request.blank('/accept/foo',
|
req2 = webob.Request.blank('/accept/foo',
|
||||||
environ=self._get_environ_header('POST'))
|
environ=self.get_environ_header('POST'))
|
||||||
with mock.patch('oslo.messaging.Notifier.info') as notify:
|
with mock.patch('oslo.messaging.Notifier.info') as notify:
|
||||||
# Check GET/PUT request does not send notification
|
# Check GET/PUT request does not send notification
|
||||||
middleware(req)
|
self.middleware(req)
|
||||||
middleware(req1)
|
self.middleware(req1)
|
||||||
self.assertEqual([], notify.call_args_list)
|
self.assertEqual([], notify.call_args_list)
|
||||||
|
|
||||||
# Check non-GET/PUT request does send notification
|
# Check non-GET/PUT request does send notification
|
||||||
middleware(req2)
|
self.middleware(req2)
|
||||||
self.assertThat(notify.call_args_list, matchers.HasLength(2))
|
self.assertThat(notify.call_args_list, matchers.HasLength(2))
|
||||||
call_args = notify.call_args_list[0][0]
|
call_args = notify.call_args_list[0][0]
|
||||||
self.assertEqual('audit.http.request', call_args[1])
|
self.assertEqual('audit.http.request', call_args[1])
|
||||||
|
@ -175,15 +185,11 @@ class AuditMiddlewareTest(testtools.TestCase):
|
||||||
self.assertEqual('/accept/foo', call_args[2]['requestPath'])
|
self.assertEqual('/accept/foo', call_args[2]['requestPath'])
|
||||||
|
|
||||||
def test_api_request_no_messaging(self):
|
def test_api_request_no_messaging(self):
|
||||||
middleware = audit.AuditMiddleware(
|
|
||||||
FakeApp(),
|
|
||||||
audit_map_file=self.audit_map,
|
|
||||||
service_name='pycadf')
|
|
||||||
req = webob.Request.blank('/foo/bar',
|
req = webob.Request.blank('/foo/bar',
|
||||||
environ=self._get_environ_header('GET'))
|
environ=self.get_environ_header('GET'))
|
||||||
with mock.patch('keystonemiddleware.audit.messaging', None):
|
with mock.patch('keystonemiddleware.audit.messaging', None):
|
||||||
with mock.patch('keystonemiddleware.audit._LOG.info') as log:
|
with mock.patch('keystonemiddleware.audit._LOG.info') as log:
|
||||||
middleware(req)
|
self.middleware(req)
|
||||||
# Check first notification with only 'request'
|
# Check first notification with only 'request'
|
||||||
call_args = log.call_args_list[0][0]
|
call_args = log.call_args_list[0][0]
|
||||||
self.assertEqual('audit.http.request',
|
self.assertEqual('audit.http.request',
|
||||||
|
@ -200,7 +206,7 @@ class AuditMiddlewareTest(testtools.TestCase):
|
||||||
audit_map_file=self.audit_map,
|
audit_map_file=self.audit_map,
|
||||||
service_name='pycadf')
|
service_name='pycadf')
|
||||||
req = webob.Request.blank('/foo/bar',
|
req = webob.Request.blank('/foo/bar',
|
||||||
environ=self._get_environ_header('GET'))
|
environ=self.get_environ_header('GET'))
|
||||||
with mock.patch('oslo.messaging.Notifier.info') as notify:
|
with mock.patch('oslo.messaging.Notifier.info') as notify:
|
||||||
middleware(req)
|
middleware(req)
|
||||||
self.assertIsNotNone(req.environ.get('cadf_event'))
|
self.assertIsNotNone(req.environ.get('cadf_event'))
|
||||||
|
@ -215,16 +221,265 @@ class AuditMiddlewareTest(testtools.TestCase):
|
||||||
audit_map_file=self.audit_map,
|
audit_map_file=self.audit_map,
|
||||||
service_name='pycadf')
|
service_name='pycadf')
|
||||||
req = webob.Request.blank('/foo/bar',
|
req = webob.Request.blank('/foo/bar',
|
||||||
environ=self._get_environ_header('GET'))
|
environ=self.get_environ_header('GET'))
|
||||||
with mock.patch('oslo.messaging.Notifier.info',
|
with mock.patch('oslo.messaging.Notifier.info',
|
||||||
side_effect=Exception('error')) as notify:
|
side_effect=Exception('error')) as notify:
|
||||||
middleware._process_request(req)
|
middleware._process_request(req)
|
||||||
self.assertTrue(notify.called)
|
self.assertTrue(notify.called)
|
||||||
req2 = webob.Request.blank('/foo/bar',
|
req2 = webob.Request.blank('/foo/bar',
|
||||||
environ=self._get_environ_header('GET'))
|
environ=self.get_environ_header('GET'))
|
||||||
with mock.patch('oslo.messaging.Notifier.info') as notify:
|
with mock.patch('oslo.messaging.Notifier.info') as notify:
|
||||||
middleware._process_response(req2, webob.response.Response())
|
middleware._process_response(req2, webob.response.Response())
|
||||||
self.assertTrue(notify.called)
|
self.assertTrue(notify.called)
|
||||||
# ensure event is not the same across requests
|
# ensure event is not the same across requests
|
||||||
self.assertNotEqual(req.environ['cadf_event'].id,
|
self.assertNotEqual(req.environ['cadf_event'].id,
|
||||||
notify.call_args_list[0][0][2]['id'])
|
notify.call_args_list[0][0][2]['id'])
|
||||||
|
|
||||||
|
|
||||||
|
@mock.patch('oslo.messaging', mock.MagicMock())
|
||||||
|
class AuditApiLogicTest(BaseAuditMiddlewareTest):
|
||||||
|
|
||||||
|
def api_request(self, method, url):
|
||||||
|
req = webob.Request.blank(url, environ=self.get_environ_header(method),
|
||||||
|
remote_addr='192.168.0.1')
|
||||||
|
self.middleware._process_request(req)
|
||||||
|
return req
|
||||||
|
|
||||||
|
def test_get_list(self):
|
||||||
|
req = self.api_request('GET', 'http://admin_host:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/servers')
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['action'], 'read/list')
|
||||||
|
self.assertEqual(payload['typeURI'],
|
||||||
|
'http://schemas.dmtf.org/cloud/audit/1.0/event')
|
||||||
|
self.assertEqual(payload['outcome'], 'pending')
|
||||||
|
self.assertEqual(payload['eventType'], 'activity')
|
||||||
|
self.assertEqual(payload['target']['name'], 'nova')
|
||||||
|
self.assertEqual(payload['target']['id'], 'openstack:resource_id')
|
||||||
|
self.assertEqual(payload['target']['typeURI'],
|
||||||
|
'service/compute/servers')
|
||||||
|
self.assertEqual(len(payload['target']['addresses']), 3)
|
||||||
|
self.assertEqual(payload['target']['addresses'][0]['name'], 'admin')
|
||||||
|
self.assertEqual(payload['target']['addresses'][0]['url'],
|
||||||
|
'http://admin_host:8774')
|
||||||
|
self.assertEqual(payload['initiator']['id'], 'openstack:user_id')
|
||||||
|
self.assertEqual(payload['initiator']['name'], 'user_name')
|
||||||
|
self.assertEqual(payload['initiator']['project_id'],
|
||||||
|
'openstack:tenant_id')
|
||||||
|
self.assertEqual(payload['initiator']['host']['address'],
|
||||||
|
'192.168.0.1')
|
||||||
|
self.assertEqual(payload['initiator']['typeURI'],
|
||||||
|
'service/security/account/user')
|
||||||
|
self.assertNotEqual(payload['initiator']['credential']['token'],
|
||||||
|
'token')
|
||||||
|
self.assertEqual(payload['initiator']['credential']['identity_status'],
|
||||||
|
'Confirmed')
|
||||||
|
self.assertNotIn('reason', payload)
|
||||||
|
self.assertNotIn('reporterchain', payload)
|
||||||
|
self.assertEqual(payload['observer']['id'], 'target')
|
||||||
|
self.assertEqual(req.path, payload['requestPath'])
|
||||||
|
|
||||||
|
def test_get_read(self):
|
||||||
|
req = self.api_request('GET', 'http://admin_host:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/servers/'
|
||||||
|
+ str(uuid.uuid4()))
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['target']['typeURI'],
|
||||||
|
'service/compute/servers/server')
|
||||||
|
self.assertEqual(payload['action'], 'read')
|
||||||
|
self.assertEqual(payload['outcome'], 'pending')
|
||||||
|
|
||||||
|
def test_get_unknown_endpoint(self):
|
||||||
|
req = self.api_request('GET', 'http://unknown:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/servers')
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['action'], 'read/list')
|
||||||
|
self.assertEqual(payload['outcome'], 'pending')
|
||||||
|
self.assertEqual(payload['target']['name'], 'unknown')
|
||||||
|
self.assertEqual(payload['target']['id'], 'unknown')
|
||||||
|
self.assertEqual(payload['target']['typeURI'], 'unknown')
|
||||||
|
|
||||||
|
def test_get_unknown_endpoint_default_set(self):
|
||||||
|
with open(self.audit_map, "w") as f:
|
||||||
|
f.write("[DEFAULT]\n")
|
||||||
|
f.write("target_endpoint_type = compute\n")
|
||||||
|
f.write("[path_keywords]\n")
|
||||||
|
f.write("servers = server\n\n")
|
||||||
|
f.write("[service_endpoints]\n")
|
||||||
|
f.write("compute = service/compute")
|
||||||
|
|
||||||
|
self.middleware = audit.AuditMiddleware(
|
||||||
|
FakeApp(), audit_map_file=self.audit_map,
|
||||||
|
service_name='pycadf')
|
||||||
|
|
||||||
|
req = self.api_request('GET', 'http://unknown:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/servers')
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['action'], 'read/list')
|
||||||
|
self.assertEqual(payload['outcome'], 'pending')
|
||||||
|
self.assertEqual(payload['target']['name'], 'nova')
|
||||||
|
self.assertEqual(payload['target']['id'], 'openstack:resource_id')
|
||||||
|
self.assertEqual(payload['target']['typeURI'],
|
||||||
|
'service/compute/servers')
|
||||||
|
|
||||||
|
def test_put(self):
|
||||||
|
req = self.api_request('PUT', 'http://admin_host:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/servers')
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['target']['typeURI'],
|
||||||
|
'service/compute/servers')
|
||||||
|
self.assertEqual(payload['action'], 'update')
|
||||||
|
self.assertEqual(payload['outcome'], 'pending')
|
||||||
|
|
||||||
|
def test_delete(self):
|
||||||
|
req = self.api_request('DELETE', 'http://admin_host:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/servers')
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['target']['typeURI'],
|
||||||
|
'service/compute/servers')
|
||||||
|
self.assertEqual(payload['action'], 'delete')
|
||||||
|
self.assertEqual(payload['outcome'], 'pending')
|
||||||
|
|
||||||
|
def test_head(self):
|
||||||
|
req = self.api_request('HEAD', 'http://admin_host:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/servers')
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['target']['typeURI'],
|
||||||
|
'service/compute/servers')
|
||||||
|
self.assertEqual(payload['action'], 'read')
|
||||||
|
self.assertEqual(payload['outcome'], 'pending')
|
||||||
|
|
||||||
|
def test_post_update(self):
|
||||||
|
req = self.api_request('POST',
|
||||||
|
'http://admin_host:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/servers/'
|
||||||
|
+ str(uuid.uuid4()))
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['target']['typeURI'],
|
||||||
|
'service/compute/servers/server')
|
||||||
|
self.assertEqual(payload['action'], 'update')
|
||||||
|
self.assertEqual(payload['outcome'], 'pending')
|
||||||
|
|
||||||
|
def test_post_create(self):
|
||||||
|
req = self.api_request('POST', 'http://admin_host:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/servers')
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['target']['typeURI'],
|
||||||
|
'service/compute/servers')
|
||||||
|
self.assertEqual(payload['action'], 'create')
|
||||||
|
self.assertEqual(payload['outcome'], 'pending')
|
||||||
|
|
||||||
|
def test_post_action(self):
|
||||||
|
req = webob.Request.blank('http://admin_host:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/servers/action',
|
||||||
|
environ=self.get_environ_header('POST'))
|
||||||
|
req.body = b'{"createImage" : {"name" : "new-image","metadata": ' \
|
||||||
|
b'{"ImageType": "Gold","ImageVersion": "2.0"}}}'
|
||||||
|
self.middleware._process_request(req)
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['target']['typeURI'],
|
||||||
|
'service/compute/servers/action')
|
||||||
|
self.assertEqual(payload['action'], 'update/createImage')
|
||||||
|
self.assertEqual(payload['outcome'], 'pending')
|
||||||
|
|
||||||
|
def test_post_empty_body_action(self):
|
||||||
|
req = self.api_request('POST', 'http://admin_host:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/servers/action')
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['target']['typeURI'],
|
||||||
|
'service/compute/servers/action')
|
||||||
|
self.assertEqual(payload['action'], 'create')
|
||||||
|
self.assertEqual(payload['outcome'], 'pending')
|
||||||
|
|
||||||
|
def test_custom_action(self):
|
||||||
|
req = self.api_request('GET', 'http://admin_host:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/os-hosts/'
|
||||||
|
+ str(uuid.uuid4()) + '/reboot')
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['target']['typeURI'],
|
||||||
|
'service/compute/os-hosts/host/reboot')
|
||||||
|
self.assertEqual(payload['action'], 'start/reboot')
|
||||||
|
self.assertEqual(payload['outcome'], 'pending')
|
||||||
|
|
||||||
|
def test_custom_action_complex(self):
|
||||||
|
req = self.api_request('GET', 'http://admin_host:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/os-migrations')
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['target']['typeURI'],
|
||||||
|
'service/compute/os-migrations')
|
||||||
|
self.assertEqual(payload['action'], 'read')
|
||||||
|
req = self.api_request('POST', 'http://admin_host:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/os-migrations')
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['target']['typeURI'],
|
||||||
|
'service/compute/os-migrations')
|
||||||
|
self.assertEqual(payload['action'], 'create')
|
||||||
|
|
||||||
|
def test_response_mod_msg(self):
|
||||||
|
req = self.api_request('GET', 'http://admin_host:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/servers')
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.middleware._process_response(req, webob.Response())
|
||||||
|
payload2 = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['id'], payload2['id'])
|
||||||
|
self.assertEqual(payload['tags'], payload2['tags'])
|
||||||
|
self.assertEqual(payload2['outcome'], 'success')
|
||||||
|
self.assertEqual(payload2['reason']['reasonType'], 'HTTP')
|
||||||
|
self.assertEqual(payload2['reason']['reasonCode'], '200')
|
||||||
|
self.assertEqual(len(payload2['reporterchain']), 1)
|
||||||
|
self.assertEqual(payload2['reporterchain'][0]['role'], 'modifier')
|
||||||
|
self.assertEqual(payload2['reporterchain'][0]['reporter']['id'],
|
||||||
|
'target')
|
||||||
|
|
||||||
|
def test_no_response(self):
|
||||||
|
req = self.api_request('GET', 'http://admin_host:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/servers')
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.middleware._process_response(req, None)
|
||||||
|
payload2 = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['id'], payload2['id'])
|
||||||
|
self.assertEqual(payload['tags'], payload2['tags'])
|
||||||
|
self.assertEqual(payload2['outcome'], 'unknown')
|
||||||
|
self.assertNotIn('reason', payload2)
|
||||||
|
self.assertEqual(len(payload2['reporterchain']), 1)
|
||||||
|
self.assertEqual(payload2['reporterchain'][0]['role'], 'modifier')
|
||||||
|
self.assertEqual(payload2['reporterchain'][0]['reporter']['id'],
|
||||||
|
'target')
|
||||||
|
|
||||||
|
def test_missing_req(self):
|
||||||
|
req = webob.Request.blank('http://admin_host:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/servers',
|
||||||
|
environ=self.get_environ_header('GET'))
|
||||||
|
self.assertNotIn('cadf_event', req.environ)
|
||||||
|
self.middleware._process_response(req, webob.Response())
|
||||||
|
self.assertIn('cadf_event', req.environ)
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['outcome'], 'success')
|
||||||
|
self.assertEqual(payload['reason']['reasonType'], 'HTTP')
|
||||||
|
self.assertEqual(payload['reason']['reasonCode'], '200')
|
||||||
|
self.assertEqual(payload['observer']['id'], 'target')
|
||||||
|
|
||||||
|
def test_missing_catalog_endpoint_id(self):
|
||||||
|
env_headers = {'HTTP_X_SERVICE_CATALOG':
|
||||||
|
'''[{"endpoints_links": [],
|
||||||
|
"endpoints": [{"adminURL":
|
||||||
|
"http://admin_host:8774",
|
||||||
|
"region": "RegionOne",
|
||||||
|
"publicURL":
|
||||||
|
"http://public_host:8774",
|
||||||
|
"internalURL":
|
||||||
|
"http://internal_host:8774"}],
|
||||||
|
"type": "compute",
|
||||||
|
"name": "nova"},]''',
|
||||||
|
'HTTP_X_USER_ID': 'user_id',
|
||||||
|
'HTTP_X_USER_NAME': 'user_name',
|
||||||
|
'HTTP_X_AUTH_TOKEN': 'token',
|
||||||
|
'HTTP_X_PROJECT_ID': 'tenant_id',
|
||||||
|
'HTTP_X_IDENTITY_STATUS': 'Confirmed',
|
||||||
|
'REQUEST_METHOD': 'GET'}
|
||||||
|
req = webob.Request.blank('http://admin_host:8774/v2/'
|
||||||
|
+ str(uuid.uuid4()) + '/servers',
|
||||||
|
environ=env_headers)
|
||||||
|
self.middleware._process_request(req)
|
||||||
|
payload = req.environ['cadf_event'].as_dict()
|
||||||
|
self.assertEqual(payload['target']['id'], identifier.norm_ns('nova'))
|
||||||
|
|
Loading…
Reference in New Issue