diff --git a/storlets/swift_middleware/handlers/proxy.py b/storlets/swift_middleware/handlers/proxy.py index 974cae8c..d07282d3 100644 --- a/storlets/swift_middleware/handlers/proxy.py +++ b/storlets/swift_middleware/handlers/proxy.py @@ -41,10 +41,10 @@ class StorletProxyHandler(StorletBaseHandler): def __init__(self, request, conf, gateway_conf, app, logger): super(StorletProxyHandler, self).__init__( request, conf, gateway_conf, app, logger) + self.max_extra_resources = int(conf.get('max_extra_resources', -1)) self.storlet_containers = [self.storlet_container, self.storlet_dependency] self.agent = 'ST' - self.extra_sources = [] # A very initial hook for blocking requests self._should_block(request) @@ -71,6 +71,7 @@ class StorletProxyHandler(StorletBaseHandler): raise NotStorletExecution() elif self.is_storlet_execution: self._setup_gateway() + self.extra_resources = self._parse_extra_resources() else: raise NotStorletExecution() @@ -280,8 +281,6 @@ class StorletProxyHandler(StorletBaseHandler): def _call_gateway(self, resp): sreq = self._build_storlet_request(self.request, resp.headers, resp.app_iter) - if self.extra_sources: - sreq.extra_data_list = self.extra_sources return self.gateway.invocation_flow(sreq) def augment_storlet_request(self, params): @@ -294,36 +293,60 @@ class StorletProxyHandler(StorletBaseHandler): for key, val in params.items(): self.request.headers['X-Storlet-' + key] = val - def gather_extra_sources(self): - # (kota_): I know this is a crazy hack to set the resp - # dynamically so that this is a temporary way to make sure - # the capability, this absolutely needs cleanup more genelic - if 'X-Storlet-Extra-Resources' in self.request.headers: - try: - resources = list_from_csv( - self.request.headers['X-Storlet-Extra-Resources']) - # resourece should be /container/object - for resource in resources: - # sanity check, if it's invalid path ValueError - # will be raisen - swift_path = ['', self.api_version, self.account] - swift_path.extend(split_path(resource, 2, 2, True)) - sub_req = make_subrequest( - self.request.environ, - 'GET', '/'.join(swift_path), - agent=self.agent) - sub_resp = sub_req.get_response(self.app) - # TODO(kota_): make this in another green thread - # expicially, in parallel with primary GET + def _parse_extra_resources(self): + if 'X-Storlet-Extra-Resources' not in self.request.headers: + return [] - self.extra_sources.append( - StorletData( - self._get_user_metadata(sub_resp.headers), - sub_resp.app_iter)) - except ValueError: - raise HTTPBadRequest( - 'X-Storlet-Extra-Resource must be a csv with' - '/container/object format') + try: + resources = list_from_csv( + self.request.headers['X-Storlet-Extra-Resources']) + + if len(resources) == 0: + return [] + + if self.max_extra_resources > 0: + if len(resources) > self.max_extra_resources: + msg = ( + 'Too many extra resources. %d was requested but only ' + '%s is allowed,' % + (len(resources), self.max_extra_resources) + ) + raise HTTPBadRequest(msg.encode('utf-8'), + request=self.request) + elif self.max_extra_resources == 0: + msg = 'Usage of extra resources is not allowed' + raise HTTPForbidden(msg.encode('utf-8'), request=self.request) + + res_list = [] + # resourece should be /container/object + for resource in resources: + # sanity check, if it's invalid path ValueError + # will be raisen + swift_path = ['', self.api_version, self.account] + swift_path.extend(split_path(resource, 2, 2, True)) + res_list.append('/'.join(swift_path)) + return res_list + except ValueError: + msg = ('X-Storlet-Extra-Resource must be a csv with' + '/container/object format') + raise HTTPBadRequest(msg.encode('utf-8'), request=self.request) + + def _build_storlet_request(self, req, sheaders, sbody_iter): + sreq = super(StorletProxyHandler, self)._build_storlet_request( + req, sheaders, sbody_iter) + for resource in self.extra_resources: + # TODO(kota_): make this in another green thread + # expicially, in parallel with primary GET + sub_req = make_subrequest( + self.request.environ, + 'GET', resource, + agent=self.agent) + sub_resp = sub_req.get_response(self.app) + sreq.extra_data_list.append( + StorletData( + self._get_user_metadata(sub_resp.headers), + sub_resp.app_iter)) + return sreq @public def GET(self): @@ -387,7 +410,6 @@ class StorletProxyHandler(StorletBaseHandler): # full object to detect if we are in SLO case, # and invoke Storlet only if in SLO case. if self.is_proxy_runnable(original_resp): - self.gather_extra_sources() return self.apply_storlet(original_resp) else: # Non proxy GET case: Storlet was already invoked at @@ -470,9 +492,6 @@ class StorletProxyHandler(StorletBaseHandler): # Do it and fixup the user metadata headers. sreq = self._build_storlet_request(self.request, src_resp.headers, src_resp.app_iter) - self.gather_extra_sources() - if self.extra_sources: - sreq.extra_data_list = self.extra_sources sresp = self.gateway.invocation_flow(sreq) data_iter = sresp.data.data_iter self._set_metadata_in_headers(self.request.headers, diff --git a/tests/functional/java/test_multiinput_storlet.py b/tests/functional/java/test_multiinput_storlet.py index 3ab39149..91d9171a 100644 --- a/tests/functional/java/test_multiinput_storlet.py +++ b/tests/functional/java/test_multiinput_storlet.py @@ -53,6 +53,29 @@ class TestMultiInputStorlet(StorletJavaFunctionalTest): self.assertEqual(b'0123456789abcdefghijklmnopqr', resp_content) + def test_put_extra_sources(self): + obj = 'small' + obj2 = 'small2' + c.put_object(self.url, self.token, + self.container, obj2, + b'efghijklmnopqr') + + headers = { + 'X-Run-Storlet': self.storlet_name, + 'X-Storlet-Extra-Resources': + '/%s/%s' % (self.container, obj2), + 'X-Storlet-Run-On-Proxy': '' + } + headers.update(self.additional_headers) + c.put_object( + self.url, self.token, self.container, obj, + b'0123456789abcd', headers=headers) + + resp_headers, resp_content = c.get_object( + self.url, self.token, self.container, obj) + self.assertEqual(b'0123456789abcdefghijklmnopqr', + resp_content) + def test_put_x_copy_from_extra_sources(self): obj = 'small' obj2 = 'small2' diff --git a/tests/functional/python/test_multiinput_storlet.py b/tests/functional/python/test_multiinput_storlet.py index f4c48da2..52ab2fdc 100644 --- a/tests/functional/python/test_multiinput_storlet.py +++ b/tests/functional/python/test_multiinput_storlet.py @@ -53,6 +53,28 @@ class TestMultiInputStorlet(StorletPythonFunctionalTest): self.assertEqual(b'0123456789abcdefghijklmnopqr', resp_content) + def test_put_extra_sources(self): + obj = 'small' + obj2 = 'small2' + c.put_object(self.url, self.token, + self.container, obj2, + b'efghijklmnopqr') + + headers = { + 'X-Run-Storlet': self.storlet_name, + 'X-Storlet-Extra-Resources': + os.path.join('/' + self.container, obj2) + } + headers.update(self.additional_headers) + c.put_object( + self.url, self.token, self.container, obj, + b'0123456789abcd', headers=headers) + + resp_headers, resp_content = c.get_object( + self.url, self.token, self.container, obj) + self.assertEqual(b'0123456789abcdefghijklmnopqr', + resp_content) + def test_put_x_copy_from_extra_sources(self): obj = 'small' obj2 = 'small2' diff --git a/tests/unit/swift_middleware/handlers/test_proxy.py b/tests/unit/swift_middleware/handlers/test_proxy.py index 8296d54f..fd455dd9 100644 --- a/tests/unit/swift_middleware/handlers/test_proxy.py +++ b/tests/unit/swift_middleware/handlers/test_proxy.py @@ -175,7 +175,7 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware): self.assertEqual('200 OK', resp.status) self.assertEqual(b'FAKE APP', resp.body) - def test_GET_with_storlets_and_extra_resourece(self): + def test_GET_with_storlets_and_extra_resource(self): target = '/v1/AUTH_a/c/o' self.base_app.register('GET', target, HTTPOk, body=b'FAKE APP') extra_target = '/v1/AUTH_a/c2/o2' @@ -195,6 +195,79 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware): # GET extra target also called self.assertTrue(any(self.base_app.get_calls('GET', extra_target))) + def test_PUT_with_storlets_and_extra_resource(self): + target = '/v1/AUTH_a/c/o' + self.base_app.register('PUT', target, HTTPCreated, body=b'') + extra_target = '/v1/AUTH_a/c2/o2' + self.base_app.register('GET', extra_target, HTTPOk, body=b'Whooa') + storlet = '/v1/AUTH_a/storlet/Storlet-1.0.jar' + self.base_app.register('GET', storlet, HTTPOk, body=b'jar binary') + + with storlet_enabled(): + headers = {'X-Run-Storlet': 'Storlet-1.0.jar', + 'X-Storlet-Extra-Resources': '/c2/o2'} + resp = self.get_request_response(target, 'PUT', headers=headers) + self.assertEqual('201 Created', resp.status) + + # PUT target called + self.assertTrue(any(self.base_app.get_calls('PUT', target))) + # GET extra target also called + self.assertTrue(any(self.base_app.get_calls('GET', extra_target))) + + def test_GET_with_storlets_and_extra_resource_within_limit(self): + target = '/v1/AUTH_a/c/o' + self.base_app.register('GET', target, HTTPOk, body=b'FAKE APP') + extra_targets = ['/v1/AUTH_a/c2/o2', '/v1/AUTH_a/c3/o3'] + for extra_target in extra_targets: + self.base_app.register('GET', extra_target, HTTPOk, body=b'foo') + storlet = '/v1/AUTH_a/storlet/Storlet-1.0.jar' + self.base_app.register('GET', storlet, HTTPOk, body=b'jar binary') + + self.conf['max_extra_resources'] = 2 + + with storlet_enabled(): + headers = {'X-Run-Storlet': 'Storlet-1.0.jar', + 'X-Storlet-Extra-Resources': '/c2/o2,/c3/o3'} + resp = self.get_request_response(target, 'GET', headers=headers) + self.assertEqual('200 OK', resp.status) + self.assertEqual(b'FAKE APP', resp.body) + + # GET target called + self.assertTrue(any(self.base_app.get_calls('GET', target))) + # GET extra targets also called + for extra_target in extra_targets: + self.assertTrue( + any(self.base_app.get_calls('GET', extra_target))) + + def test_GET_with_storlets_and_extra_resource_not_object(self): + target = '/v1/AUTH_a/c/o' + + with storlet_enabled(): + headers = {'X-Run-Storlet': 'Storlet-1.0.jar', + 'X-Storlet-Extra-Resources': '/c2o2'} + resp = self.get_request_response(target, 'GET', headers=headers) + self.assertEqual('400 Bad Request', resp.status) + + def test_GET_with_storlets_and_extra_resource_over_limit(self): + target = '/v1/AUTH_a/c/o' + self.conf['max_extra_resources'] = 1 + + with storlet_enabled(): + headers = {'X-Run-Storlet': 'Storlet-1.0.jar', + 'X-Storlet-Extra-Resources': '/c2/o2,/c3/o3'} + resp = self.get_request_response(target, 'GET', headers=headers) + self.assertEqual('400 Bad Request', resp.status) + + def test_GET_with_storlets_and_extra_resource_disabled(self): + target = '/v1/AUTH_a/c/o' + self.conf['max_extra_resources'] = 0 + + with storlet_enabled(): + headers = {'X-Run-Storlet': 'Storlet-1.0.jar', + 'X-Storlet-Extra-Resources': '/c2/o2'} + resp = self.get_request_response(target, 'GET', headers=headers) + self.assertEqual('403 Forbidden', resp.status) + def test_GET_slo_without_storlets(self): target = '/v1/AUTH_a/c/slo_manifest' self.base_app.register('GET', target, HTTPOk,