openstack
/
storlets
Code Issues Proposed changes

Allow limiting maximum extra resources 

Using many extra resources may cause high load at proxy server, because
the process needs to get all of the requested objects concurrently.
This change introduces a new option to limit maximum number of extra
resources per request to avoid DoS attack by too many extra resources.
If a request contains extra resources over the limit, then the request
is rejected at an early stage. The default value is -1 which means
unlimited. In case this option is set to 0 then users are not allowed
to use extra resources at all.

This also refactors handling of extra-resources header. One side
benefit of the refactoring is that now users can use additional inputs
when executing storlet over PUT requests.

Change-Id: I0ea7d78614f2b1ef5bf4961d2d5fe773264ef448
This commit is contained in:
Takashi Kajinami 2024-02-14 00:13:56 +09:00
parent b75465ab51
commit 8401b6c581
4 changed files with 174 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
storlets/swift_middleware/handlers/proxy.py
View File

@ -41,10 +41,10 @@ class StorletProxyHandler(StorletBaseHandler):
def __init__(self, request, conf, gateway_conf, app, logger):
super(StorletProxyHandler, self).__init__(
request, conf, gateway_conf, app, logger)
self.max_extra_resources = int(conf.get('max_extra_resources', -1))
self.storlet_containers = [self.storlet_container,
self.storlet_dependency]
self.agent = 'ST'
self.extra_sources = []
# A very initial hook for blocking requests
self._should_block(request)
@ -71,6 +71,7 @@ class StorletProxyHandler(StorletBaseHandler):
raise NotStorletExecution()
elif self.is_storlet_execution:
self._setup_gateway()
self.extra_resources = self._parse_extra_resources()
else:
raise NotStorletExecution()
@ -280,8 +281,6 @@ class StorletProxyHandler(StorletBaseHandler):
def _call_gateway(self, resp):
sreq = self._build_storlet_request(self.request, resp.headers,
resp.app_iter)
if self.extra_sources:
sreq.extra_data_list = self.extra_sources
return self.gateway.invocation_flow(sreq)
def augment_storlet_request(self, params):
@ -294,36 +293,60 @@ class StorletProxyHandler(StorletBaseHandler):
for key, val in params.items():
self.request.headers['X-Storlet-' + key] = val
def gather_extra_sources(self):
# (kota_): I know this is a crazy hack to set the resp
# dynamically so that this is a temporary way to make sure
# the capability, this absolutely needs cleanup more genelic
if 'X-Storlet-Extra-Resources' in self.request.headers:
try:
resources = list_from_csv(
self.request.headers['X-Storlet-Extra-Resources'])
# resourece should be /container/object
for resource in resources:
# sanity check, if it's invalid path ValueError
# will be raisen
swift_path = ['', self.api_version, self.account]
swift_path.extend(split_path(resource, 2, 2, True))
sub_req = make_subrequest(
self.request.environ,
'GET', '/'.join(swift_path),
agent=self.agent)
sub_resp = sub_req.get_response(self.app)
# TODO(kota_): make this in another green thread
# expicially, in parallel with primary GET
def _parse_extra_resources(self):
if 'X-Storlet-Extra-Resources' not in self.request.headers:
return []
self.extra_sources.append(
StorletData(
self._get_user_metadata(sub_resp.headers),
sub_resp.app_iter))
except ValueError:
raise HTTPBadRequest(
'X-Storlet-Extra-Resource must be a csv with'
'/container/object format')
try:
resources = list_from_csv(
self.request.headers['X-Storlet-Extra-Resources'])
if len(resources) == 0:
return []
if self.max_extra_resources > 0:
if len(resources) > self.max_extra_resources:
msg = (
'Too many extra resources. %d was requested but only '
'%s is allowed,' %
(len(resources), self.max_extra_resources)
)
raise HTTPBadRequest(msg.encode('utf-8'),
request=self.request)
elif self.max_extra_resources == 0:
msg = 'Usage of extra resources is not allowed'
raise HTTPForbidden(msg.encode('utf-8'), request=self.request)
res_list = []
# resourece should be /container/object
for resource in resources:
# sanity check, if it's invalid path ValueError
# will be raisen
swift_path = ['', self.api_version, self.account]
swift_path.extend(split_path(resource, 2, 2, True))
res_list.append('/'.join(swift_path))
return res_list
except ValueError:
msg = ('X-Storlet-Extra-Resource must be a csv with'
'/container/object format')
raise HTTPBadRequest(msg.encode('utf-8'), request=self.request)
def _build_storlet_request(self, req, sheaders, sbody_iter):
sreq = super(StorletProxyHandler, self)._build_storlet_request(
req, sheaders, sbody_iter)
for resource in self.extra_resources:
# TODO(kota_): make this in another green thread
# expicially, in parallel with primary GET
sub_req = make_subrequest(
self.request.environ,
'GET', resource,
agent=self.agent)
sub_resp = sub_req.get_response(self.app)
sreq.extra_data_list.append(
StorletData(
self._get_user_metadata(sub_resp.headers),
sub_resp.app_iter))
return sreq
@public
def GET(self):
@ -387,7 +410,6 @@ class StorletProxyHandler(StorletBaseHandler):
# full object to detect if we are in SLO case,
# and invoke Storlet only if in SLO case.
if self.is_proxy_runnable(original_resp):
self.gather_extra_sources()
return self.apply_storlet(original_resp)
else:
# Non proxy GET case: Storlet was already invoked at
@ -470,9 +492,6 @@ class StorletProxyHandler(StorletBaseHandler):
# Do it and fixup the user metadata headers.
sreq = self._build_storlet_request(self.request, src_resp.headers,
src_resp.app_iter)
self.gather_extra_sources()
if self.extra_sources:
sreq.extra_data_list = self.extra_sources
sresp = self.gateway.invocation_flow(sreq)
data_iter = sresp.data.data_iter
self._set_metadata_in_headers(self.request.headers,

23
tests/functional/java/test_multiinput_storlet.py
View File

@ -53,6 +53,29 @@ class TestMultiInputStorlet(StorletJavaFunctionalTest):
self.assertEqual(b'0123456789abcdefghijklmnopqr',
resp_content)
def test_put_extra_sources(self):
obj = 'small'
obj2 = 'small2'
c.put_object(self.url, self.token,
self.container, obj2,
b'efghijklmnopqr')
headers = {
'X-Run-Storlet': self.storlet_name,
'X-Storlet-Extra-Resources':
'/%s/%s' % (self.container, obj2),
'X-Storlet-Run-On-Proxy': ''
}
headers.update(self.additional_headers)
c.put_object(
self.url, self.token, self.container, obj,
b'0123456789abcd', headers=headers)
resp_headers, resp_content = c.get_object(
self.url, self.token, self.container, obj)
self.assertEqual(b'0123456789abcdefghijklmnopqr',
resp_content)
def test_put_x_copy_from_extra_sources(self):
obj = 'small'
obj2 = 'small2'

22
tests/functional/python/test_multiinput_storlet.py
View File

@ -53,6 +53,28 @@ class TestMultiInputStorlet(StorletPythonFunctionalTest):
self.assertEqual(b'0123456789abcdefghijklmnopqr',
resp_content)
def test_put_extra_sources(self):
obj = 'small'
obj2 = 'small2'
c.put_object(self.url, self.token,
self.container, obj2,
b'efghijklmnopqr')
headers = {
'X-Run-Storlet': self.storlet_name,
'X-Storlet-Extra-Resources':
os.path.join('/' + self.container, obj2)
}
headers.update(self.additional_headers)
c.put_object(
self.url, self.token, self.container, obj,
b'0123456789abcd', headers=headers)
resp_headers, resp_content = c.get_object(
self.url, self.token, self.container, obj)
self.assertEqual(b'0123456789abcdefghijklmnopqr',
resp_content)
def test_put_x_copy_from_extra_sources(self):
obj = 'small'
obj2 = 'small2'

75
tests/unit/swift_middleware/handlers/test_proxy.py
View File

@ -175,7 +175,7 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware):
self.assertEqual('200 OK', resp.status)
self.assertEqual(b'FAKE APP', resp.body)
def test_GET_with_storlets_and_extra_resourece(self):
def test_GET_with_storlets_and_extra_resource(self):
target = '/v1/AUTH_a/c/o'
self.base_app.register('GET', target, HTTPOk, body=b'FAKE APP')
extra_target = '/v1/AUTH_a/c2/o2'
@ -195,6 +195,79 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware):
# GET extra target also called
self.assertTrue(any(self.base_app.get_calls('GET', extra_target)))
def test_PUT_with_storlets_and_extra_resource(self):
target = '/v1/AUTH_a/c/o'
self.base_app.register('PUT', target, HTTPCreated, body=b'')
extra_target = '/v1/AUTH_a/c2/o2'
self.base_app.register('GET', extra_target, HTTPOk, body=b'Whooa')
storlet = '/v1/AUTH_a/storlet/Storlet-1.0.jar'
self.base_app.register('GET', storlet, HTTPOk, body=b'jar binary')
with storlet_enabled():
headers = {'X-Run-Storlet': 'Storlet-1.0.jar',
'X-Storlet-Extra-Resources': '/c2/o2'}
resp = self.get_request_response(target, 'PUT', headers=headers)
self.assertEqual('201 Created', resp.status)
# PUT target called
self.assertTrue(any(self.base_app.get_calls('PUT', target)))
# GET extra target also called
self.assertTrue(any(self.base_app.get_calls('GET', extra_target)))
def test_GET_with_storlets_and_extra_resource_within_limit(self):
target = '/v1/AUTH_a/c/o'
self.base_app.register('GET', target, HTTPOk, body=b'FAKE APP')
extra_targets = ['/v1/AUTH_a/c2/o2', '/v1/AUTH_a/c3/o3']
for extra_target in extra_targets:
self.base_app.register('GET', extra_target, HTTPOk, body=b'foo')
storlet = '/v1/AUTH_a/storlet/Storlet-1.0.jar'
self.base_app.register('GET', storlet, HTTPOk, body=b'jar binary')
self.conf['max_extra_resources'] = 2
with storlet_enabled():
headers = {'X-Run-Storlet': 'Storlet-1.0.jar',
'X-Storlet-Extra-Resources': '/c2/o2,/c3/o3'}
resp = self.get_request_response(target, 'GET', headers=headers)
self.assertEqual('200 OK', resp.status)
self.assertEqual(b'FAKE APP', resp.body)
# GET target called
self.assertTrue(any(self.base_app.get_calls('GET', target)))
# GET extra targets also called
for extra_target in extra_targets:
self.assertTrue(
any(self.base_app.get_calls('GET', extra_target)))
def test_GET_with_storlets_and_extra_resource_not_object(self):
target = '/v1/AUTH_a/c/o'
with storlet_enabled():
headers = {'X-Run-Storlet': 'Storlet-1.0.jar',
'X-Storlet-Extra-Resources': '/c2o2'}
resp = self.get_request_response(target, 'GET', headers=headers)
self.assertEqual('400 Bad Request', resp.status)
def test_GET_with_storlets_and_extra_resource_over_limit(self):
target = '/v1/AUTH_a/c/o'
self.conf['max_extra_resources'] = 1
with storlet_enabled():
headers = {'X-Run-Storlet': 'Storlet-1.0.jar',
'X-Storlet-Extra-Resources': '/c2/o2,/c3/o3'}
resp = self.get_request_response(target, 'GET', headers=headers)
self.assertEqual('400 Bad Request', resp.status)
def test_GET_with_storlets_and_extra_resource_disabled(self):
target = '/v1/AUTH_a/c/o'
self.conf['max_extra_resources'] = 0
with storlet_enabled():
headers = {'X-Run-Storlet': 'Storlet-1.0.jar',
'X-Storlet-Extra-Resources': '/c2/o2'}
resp = self.get_request_response(target, 'GET', headers=headers)
self.assertEqual('403 Forbidden', resp.status)
def test_GET_slo_without_storlets(self):
target = '/v1/AUTH_a/c/slo_manifest'
self.base_app.register('GET', target, HTTPOk,