From 6673e2e6f23521c533c5630e6003bef0bab8ab81 Mon Sep 17 00:00:00 2001 From: Kristi Nikolla Date: Wed, 19 Jul 2017 09:59:54 -0400 Subject: [PATCH] Preliminary extension support + Name routing This introduces preliminary support for extensions. Extensions are registered to routes, and if the request matches the registered route they will be called to handle the request and the response. The interface is still not finalized and will progress as more extensions with their different requirements are added. Also included dependency on oslo_serialization for loading the json response. Follow up patch will change all instances of json.loads to jsonutils.loads Change-Id: I9c573ce1d4ebe85c07c8ff219f384e3c6c67b39a --- mixmatch/extend/__init__.py | 44 +++++++++++++++++++ mixmatch/extend/base.py | 65 +++++++++++++++++++++++++++++ mixmatch/extend/name_routing.py | 49 ++++++++++++++++++++++ mixmatch/proxy.py | 21 ++++++---- mixmatch/services.py | 4 +- mixmatch/tests/unit/base.py | 2 + mixmatch/tests/unit/test_images.py | 40 ++++++++++++++++-- mixmatch/tests/unit/test_volumes.py | 30 +++++++++++++ requirements.txt | 2 + setup.cfg | 4 ++ 10 files changed, 248 insertions(+), 13 deletions(-) create mode 100644 mixmatch/extend/__init__.py create mode 100644 mixmatch/extend/base.py create mode 100644 mixmatch/extend/name_routing.py diff --git a/mixmatch/extend/__init__.py b/mixmatch/extend/__init__.py new file mode 100644 index 0000000..55fdea8 --- /dev/null +++ b/mixmatch/extend/__init__.py @@ -0,0 +1,44 @@ +# Copyright 2017 Massachusetts Open Cloud +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from mixmatch import config + +from stevedore import extension + +CONF = config.CONF +LOG = config.LOG + +EXTENSION_MANAGER = None # type: extension.ExtensionManager + + +def load_extensions(): + global EXTENSION_MANAGER + + EXTENSION_MANAGER = extension.ExtensionManager( + namespace='mixmatch.extend', + invoke_on_load=True + ) + + +def get_matched_extensions(request): + """Return list of matched extensions for request + + :type request: Dict[] + :rtype: List[mixmatch.extend.base.Extension] + """ + def _match(e): + return e.obj if e.obj.matches(request) else None + + result = EXTENSION_MANAGER.map(_match) + return filter(bool, result) diff --git a/mixmatch/extend/base.py b/mixmatch/extend/base.py new file mode 100644 index 0000000..ff11bea --- /dev/null +++ b/mixmatch/extend/base.py @@ -0,0 +1,65 @@ +# Copyright 2017 Massachusetts Open Cloud +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class Extension(object): + ROUTES = [] + OPTS = [] + + def matches(self, request): + for route in self.ROUTES: + if route.match(request): + return True + return False + + def handle_request(self, request): + pass + + def handle_response(self, response): + pass + + +class Route(object): + def __init__(self, service=None, version=None, method=None, action=None): + self.service = service + self.version = version + self.method = method + self.action = action + + def _match_service(self, service): + if self.service: + return self.service == service + return True + + def _match_version(self, version): + if self.version: + return self.version == version + return True + + def _match_method(self, method): + if self.method: + return self.method == method + return True + + def _match_action(self, action): + if self.action: + # FIXME(knikolla): More sophisticated matching after PoC + return self.action == action + return True + + def match(self, request): + return (self._match_service(request['service']) and + self._match_version(request['version']) and + self._match_method(request['method']) and + self._match_action(request['action'])) diff --git a/mixmatch/extend/name_routing.py b/mixmatch/extend/name_routing.py new file mode 100644 index 0000000..170c3fb --- /dev/null +++ b/mixmatch/extend/name_routing.py @@ -0,0 +1,49 @@ +# Copyright 2017 Massachusetts Open Cloud +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from mixmatch.extend import base +from mixmatch.session import request as mm_request + +from oslo_serialization import jsonutils + + +class NameRouting(base.Extension): + + ROUTES = [ + base.Route(service='volume', version=None, + action=['volumes'], method='POST'), + base.Route(service='image', version=None, + action=['images'], method='POST'), + ] + + @staticmethod + def _is_targeted(headers): + return 'MM-SERVICE-PROVIDER' in headers + + def handle_request(self, request): + if self._is_targeted(request['headers']): + return + + body = jsonutils.loads(mm_request.data) + if request['service'] == 'image': + if request['version'] == 'v1': + name = request['headers'].get('X-IMAGE-META-NAME', '') + else: + name = body.get('name', '') + elif request['service'] == 'volume': + name = body['volume'].get('name', '') + + name = name.split('@') + if len(name) == 2: + request['headers']['MM-SERVICE-PROVIDER'] = name[1] diff --git a/mixmatch/proxy.py b/mixmatch/proxy.py index 8b80ffc..207879c 100644 --- a/mixmatch/proxy.py +++ b/mixmatch/proxy.py @@ -25,6 +25,7 @@ from mixmatch.session import app from mixmatch.session import chunked_reader from mixmatch.session import request from mixmatch import auth +from mixmatch import extend from mixmatch import model from mixmatch import services from mixmatch import utils @@ -51,9 +52,9 @@ def get_service(a): abort(404) -def get_details(method, path, headers): +def get_details(method, orig_path, headers): """Get details for a request.""" - path = path[:] + path = orig_path.split('/') # NOTE(knikolla): Request usually look like: # ///// # or @@ -66,7 +67,8 @@ def get_details(method, path, headers): 'resource_type': utils.safe_pop(path), # this 'resource_id': utils.pop_if_uuid(path), # and this 'token': headers.get('X-AUTH-TOKEN', None), - 'headers': headers} + 'headers': dict(headers), + 'path': orig_path} def is_token_header_key(string): @@ -93,16 +95,18 @@ def format_for_log(title=None, method=None, url=None, headers=None, class RequestHandler(object): def __init__(self, method, path, headers): - self.details = get_details(method, path.split('/'), headers) - + self.details = get_details(method, path, headers) + self.extensions = extend.get_matched_extensions(self.details) self._set_strip_details(self.details) - self.enabled_sps = filter( lambda sp: (self.details['service'] in service_providers.get(CONF, sp).enabled_services), CONF.service_providers ) + for extension in self.extensions: + extension.handle_request(self.details) + if not self.details['version']: if CONF.aggregation: # unversioned calls with no action @@ -154,8 +158,8 @@ class RequestHandler(object): LOG.info(format_for_log(title="Request to proxy", method=self.details['method'], - url=path, - headers=dict(headers))) + url=self.details['path'], + headers=dict(self.details['headers']))) def _do_request_on(self, sp, project_id=None): headers = self._prepare_headers(self.details['headers']) @@ -348,6 +352,7 @@ def proxy(path): def main(): config.configure() model.create_tables() + extend.load_extensions() if __name__ == "__main__": diff --git a/mixmatch/services.py b/mixmatch/services.py index 532a423..bcab019 100644 --- a/mixmatch/services.py +++ b/mixmatch/services.py @@ -19,6 +19,8 @@ from six.moves.urllib import parse from mixmatch import config +from oslo_serialization import jsonutils + CONF = config.CONF @@ -64,7 +66,7 @@ def aggregate(responses, key, service_type, version=None, resource_list = [] for location, response in responses.items(): - resources = json.loads(response.text) + resources = jsonutils.loads(response.text) if type(resources) == dict: resource_list += resources[key] diff --git a/mixmatch/tests/unit/base.py b/mixmatch/tests/unit/base.py index 4ad7ce9..a84b19e 100644 --- a/mixmatch/tests/unit/base.py +++ b/mixmatch/tests/unit/base.py @@ -22,6 +22,7 @@ from requests_mock.contrib import fixture as requests_fixture from oslo_config import fixture as config_fixture from mixmatch import config +from mixmatch import extend from mixmatch.proxy import app from mixmatch.model import BASE, enginefacade @@ -61,6 +62,7 @@ class BaseTest(testcase.TestCase): image_endpoint='http://images.remote1', volume_endpoint='http://volumes.remote1') config.post_config() + extend.load_extensions() def load_auth_fixtures(self): self.auth = FakeSession(token=uuid.uuid4().hex, diff --git a/mixmatch/tests/unit/test_images.py b/mixmatch/tests/unit/test_images.py index 2e81419..6d782d5 100644 --- a/mixmatch/tests/unit/test_images.py +++ b/mixmatch/tests/unit/test_images.py @@ -32,13 +32,45 @@ class TestImages(base.BaseTest): def _construct_url(self, image_id='', sp=None): if not sp: - prefix = '/image' + url = '/image' else: - prefix = self.service_providers[sp]['image_endpoint'] + url = self.service_providers[sp]['image_endpoint'] + url = '%s/v2/images' % url - return ( - '%s/v2/images/%s' % (prefix, image_id) + if image_id: + url = '%s/%s' % (url, image_id) + + return url + + def test_create_image(self): + image_id = uuid.uuid4().hex + self.requests_fixture.post( + self._construct_url(sp='default'), + request_headers=self.auth.get_headers(), + text=six.u(image_id), + headers={'CONTENT-TYPE': 'application/json'} ) + response = self.app.post( + self._construct_url(), + headers=self.auth.get_headers(), + data=json.dumps({'name': 'local'}) + ) + self.assertEqual(six.b(image_id), response.data) + + def test_create_image_routing(self): + image_id = uuid.uuid4().hex + self.requests_fixture.post( + self._construct_url(sp='remote1'), + request_headers=self.remote_auth.get_headers(), + text=six.u(image_id), + headers={'CONTENT-TYPE': 'application/json'} + ) + response = self.app.post( + self._construct_url(), + headers=self.auth.get_headers(), + data=json.dumps({'name': 'local@remote1'}) + ) + self.assertEqual(six.b(image_id), response.data) def test_get_image_local(self): image_id = uuid.uuid4().hex diff --git a/mixmatch/tests/unit/test_volumes.py b/mixmatch/tests/unit/test_volumes.py index 0c86830..14b40f5 100644 --- a/mixmatch/tests/unit/test_volumes.py +++ b/mixmatch/tests/unit/test_volumes.py @@ -106,6 +106,36 @@ class TestVolumesV2(base.BaseTest): return url + def test_create_volume(self): + volume_id = uuid.uuid4().hex + self.requests_fixture.post( + self._construct_url(self.auth, sp='default'), + request_headers=self.auth.get_headers(), + text=six.u(volume_id), + headers={'CONTENT-TYPE': 'application/json'} + ) + response = self.app.post( + self._construct_url(self.auth), + headers=self.auth.get_headers(), + data=json.dumps({'volume': {'name': 'local'}}) + ) + self.assertEqual(six.b(volume_id), response.data) + + def test_create_volume_routing(self): + volume_id = uuid.uuid4().hex + self.requests_fixture.post( + self._construct_url(self.remote_auth, sp='remote1'), + request_headers=self.remote_auth.get_headers(), + text=six.u(volume_id), + headers={'CONTENT-TYPE': 'application/json'} + ) + response = self.app.post( + self._construct_url(self.auth), + headers=self.auth.get_headers(), + data=json.dumps({'volume': {'name': 'local@remote1'}}) + ) + self.assertEqual(six.b(volume_id), response.data) + def test_get_volume_local_mapping(self): volume_id = uuid.uuid4().hex diff --git a/requirements.txt b/requirements.txt index ef9f95f..8e12dea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,9 @@ oslo.config!=4.3.0,!=4.4.0,>=4.0.0 # Apache-2.0 oslo.messaging!=5.25.0,>=5.24.2 # Apache-2.0 oslo.log>=3.22.0 # Apache-2.0 oslo.db>=4.23.0 # Apache-2.0 +oslo.serialization>=1.10.0,!=2.19.1 # Apache-2.0 keystoneauth1>=2.21.0 # Apache-2.0 python-keystoneclient>=3.8.0 # Apache-2.0 requests>=2.14.2 # Apache-2.0 six>=1.9.0 # MIT +stevedore>=1.20.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index ef672e9..01e1480 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,12 +24,16 @@ classifier = [files] packages = mixmatch + mixmatch.config + mixmatch.extend data_files = etc/ = etc/* [entry_points] oslo.config.opts = mixmatch = mixmatch.config:list_opts +mixmatch.extend = + name_routing = mixmatch.extend.name_routing:NameRouting [build_sphinx] source-dir = doc/source