From 6adbeb40365b88c721294e72c8a95accb1b1d4f7 Mon Sep 17 00:00:00 2001 From: indianwhocodes Date: Mon, 11 Sep 2023 11:48:00 -0700 Subject: [PATCH] slo: part-number=N query parameter support This change allows individual SLO segments to be downloaded by adding an extra 'part-number' query parameter to the GET request. You can also retrieve the Content-Length of an individual segment with a HEAD request. Co-Authored-By: Clay Gerrard Co-Authored-By: Alistair Coles Change-Id: I7af0dc9898ca35f042b52dd5db000072f2c7512e --- swift/common/middleware/slo.py | 205 ++++-- swift/common/request_helpers.py | 33 + test/functional/swift_test_client.py | 4 +- test/functional/test_object_versioning.py | 131 +++- test/functional/test_slo.py | 157 +++++ test/functional/tests.py | 10 +- test/unit/common/middleware/test_slo.py | 731 +++++++++++++++++++++- 7 files changed, 1206 insertions(+), 65 deletions(-) diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index d8b86218dc..146f72e4c8 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -270,11 +270,27 @@ A GET request with the query parameters:: will return the contents of the original manifest as it was sent by the client. The main purpose for both calls is solely debugging. -When the manifest object is uploaded you are more or less guaranteed that -every segment in the manifest exists and matched the specifications. -However, there is nothing that prevents the user from breaking the -SLO download by deleting/replacing a segment referenced in the manifest. It is -left to the user to use caution in handling the segments. +A GET request to a manifest object with the query parameter:: + + ?part-number= + +will return the contents of the ``nth`` segment. Segments are indexed from 1, +so ``n`` must be an integer between 1 and the total number of segments in the +manifest. The response status will be ``206 Partial Content`` and its headers +will include: an ``X-Parts-Count`` header equal to the total number of +segments; a ``Content-Length`` header equal to the length of the specified +segment; a ``Content-Range`` header describing the byte range of the specified +part within the SLO. A HEAD request with a ``part-number`` parameter will also +return a response with status ``206 Partial Content`` and the same headers. + +.. note:: + + When the manifest object is uploaded you are more or less guaranteed that + every segment in the manifest exists and matched the specifications. + However, there is nothing that prevents the user from breaking the SLO + download by deleting/replacing a segment referenced in the manifest. It is + left to the user to use caution in handling the segments. + ----------------------- Deleting a Large Object @@ -353,7 +369,7 @@ from swift.common.registry import register_swift_info from swift.common.request_helpers import SegmentedIterable, \ get_sys_meta_prefix, update_etag_is_at_header, resolve_etag_is_at_header, \ get_container_update_override_key, update_ignore_range_header, \ - get_param + get_param, get_valid_part_num from swift.common.constraints import check_utf8, AUTO_CREATE_ACCOUNT_PREFIX from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED from swift.common.wsgi import WSGIContext, make_subrequest, make_env, \ @@ -564,6 +580,60 @@ def _annotate_segments(segments, logger=None): seg_dict['segment_length'] = segment_length +def calculate_byterange_for_part_num(req, segments, part_num): + """ + Helper function to calculate the byterange for a part_num response. + + N.B. as a side-effect of calculating the single tuple representing the + byterange required for a part_num response this function will also mutate + the request's Range header so that swob knows to return 206. + + :param req: the request object + :param segments: the list of seg_dicts + :param part_num: the part number of the object to return + + :returns: a tuple representing the byterange + """ + start = 0 + for seg in segments[:part_num - 1]: + start += seg['segment_length'] + last = start + segments[part_num - 1]['segment_length'] + # We need to mutate the request's Range header so that swob knows to + # handle these partial content requests correctly. + req.range = "bytes=%d-%d" % (start, last - 1) + return start, last - 1 + + +def calculate_byteranges(req, segments, resp_attrs, part_num): + """ + Calculate the byteranges based on the request, segments, and part number. + + N.B. as a side-effect of calculating the single tuple representing the + byterange required for a part_num response this function will also mutate + the request's Range header so that swob knows to return 206. + + :param req: the request object + :param segments: the list of seg_dicts + :param resp_attrs: the slo response attributes + :param part_num: the part number of the object to return + + :returns: a list of tuples representing byteranges + """ + if req.range: + byteranges = [ + # For some reason, swob.Range.ranges_for_length adds 1 to the + # last byte's position. + (start, end - 1) for start, end + in req.range.ranges_for_length(resp_attrs.slo_size)] + elif part_num: + byteranges = [ + calculate_byterange_for_part_num(req, segments, part_num)] + else: + byteranges = [(0, resp_attrs.slo_size - 1)] + + return byteranges + + class RespAttrs(object): """ Encapsulate properties of a GET or HEAD response that are pertinent to @@ -684,6 +754,9 @@ class SloGetContext(WSGIContext): method='GET', headers={'x-auth-token': req.headers.get('x-auth-token')}, agent='%(orig)s SLO MultipartGET', swift_source='SLO') + params_copy = dict(req.params) + params_copy.pop('part-number', None) + sub_req.params = params_copy sub_resp = sub_req.get_response(self.slo.app) if not sub_resp.is_success: @@ -847,8 +920,7 @@ class SloGetContext(WSGIContext): # we can avoid re-fetching the object. return first_byte == 0 and last_byte == length - 1 - def _is_manifest_and_need_to_refetch(self, req, resp_attrs, - is_manifest_get): + def _need_to_refetch_manifest(self, req, resp_attrs, is_part_num_request): """ Check if the segments will be needed to service the request and update the segment_listing_needed attribute. @@ -856,19 +928,11 @@ class SloGetContext(WSGIContext): :return: boolean indicating if we need to refetch, only if the segments ARE needed we MAY need to refetch them! """ - if not resp_attrs.is_slo: - # Not a static large object manifest, maybe an error, regardless - # no refetch needed - return False - - if is_manifest_get: - # Any manifest json object response will do - return False - if req.method == 'HEAD': # There may be some cases in the future where a HEAD resp on even a # modern manifest should refetch, e.g. lp bug #2029174 - self.segment_listing_needed = resp_attrs.is_legacy + self.segment_listing_needed = (resp_attrs.is_legacy or + is_part_num_request) # it will always be the case that a HEAD must re-fetch iff # segment_listing_needed return self.segment_listing_needed @@ -965,22 +1029,56 @@ class SloGetContext(WSGIContext): replace_headers) def _return_slo_response(self, req, start_response, resp_iter, resp_attrs): + headers = { + 'Etag': '"%s"' % resp_attrs.slo_etag, + 'X-Manifest-Etag': resp_attrs.json_md5, + # swob will fix this for a GET with Range + 'Content-Length': str(resp_attrs.slo_size), + # ignore bogus content-range, make swob figure it out + 'Content-Range': None, + } if self.segment_listing_needed: # consume existing resp_iter; we'll create a new one segments = self._parse_segments(resp_iter) resp_attrs.update_from_segments(segments) - if req.method == 'HEAD': + headers['Etag'] = '"%s"' % resp_attrs.slo_etag + headers['Content-Length'] = str(resp_attrs.slo_size) + part_num = get_valid_part_num(req) + if part_num: + headers['X-Parts-Count'] = len(segments) + + if part_num and part_num > len(segments): + if req.method == 'HEAD': + resp_iter = [] + headers['Content-Length'] = '0' + else: + body = b'The requested part number is not satisfiable' + resp_iter = [body] + headers['Content-Length'] = len(body) + headers['Content-Range'] = 'bytes */%d' % resp_attrs.slo_size + self._response_status = '416 Requested Range Not Satisfiable' + elif part_num and req.method == 'HEAD': + resp_iter = [] + headers['Content-Length'] = \ + segments[part_num - 1].get('segment_length') + start, end = calculate_byterange_for_part_num( + req, segments, part_num) + headers['Content-Range'] = \ + 'bytes {}-{}/{}'.format(start, end, + resp_attrs.slo_size) + # The RFC specifies 206 in the context of Range requests, and + # Range headers MUST be ignored for HEADs [1], so a HEAD will + # not normally return a 206. However, a part-number HEAD + # returns Content-Length equal to the part size, rather than + # the whole object size, so in this case we do return 206. + # [1] https://www.rfc-editor.org/rfc/rfc9110#name-range + self._response_status = '206 Partial Content' + elif req.method == 'HEAD': resp_iter = [] else: - resp_iter = self._build_resp_iter(req, segments, resp_attrs) - headers = { - 'Etag': '"%s"' % resp_attrs.slo_etag, - 'X-Manifest-Etag': resp_attrs.json_md5, - # This isn't correct for range requests, but swob will fix it? - 'Content-Length': str(resp_attrs.slo_size), - # ignore bogus content-range, make swob figure it out - 'Content-Range': None - } + byteranges = calculate_byteranges( + req, segments, resp_attrs, part_num) + resp_iter = self._build_resp_iter(req, segments, byteranges) return self._return_response(req, start_response, resp_iter, replace_headers=headers) @@ -1046,21 +1144,32 @@ class SloGetContext(WSGIContext): update_ignore_range_header(req, 'X-Static-Large-Object') # process original request + orig_path_info = req.path_info resp_iter = self._app_call(req.environ) resp_attrs = RespAttrs.from_headers(self._response_headers) - # the next two calls hide a couple side-effects, sorry: - # - # 1) regardless of the return value the "need_to_refetch" check *may* - # also set self.segment_listing_needed = True (it's commented to - # help you wrap your head around that one, good luck) - # 2) if we refetch, we overwrite the current resp_iter and resp_attrs - # variables, partly because we *might* get back a NOT - # resp_attrs.is_slo response (even if we had one to start), but - # hopefully they're just the manifest resp we needed to refetch! - if self._is_manifest_and_need_to_refetch(req, resp_attrs, - is_manifest_get): - resp_attrs, resp_iter = self._refetch_manifest( - req, resp_iter, resp_attrs) + if resp_attrs.is_slo and not is_manifest_get: + try: + # only validate part-number if the request is to an SLO + part_num = get_valid_part_num(req) + except HTTPException: + friendly_close(resp_iter) + raise + # the next two calls hide a couple side effects, sorry: + # + # 1) regardless of the return value the "need_to_refetch" check + # *may* also set self.segment_listing_needed = True (it's + # commented to help you wrap your head around that one, + # good luck) + # 2) if we refetch, we overwrite the current resp_iter and + # resp_attrs variables, partly because we *might* get back a NOT + # resp_attrs.is_slo response (even if we had one to start), but + # hopefully they're just the manifest resp we needed to refetch! + if self._need_to_refetch_manifest(req, resp_attrs, part_num): + # reset path in case it was modified during original request + # (e.g. object versioning might re-write the path) + req.path_info = orig_path_info + resp_attrs, resp_iter = self._refetch_manifest( + req, resp_iter, resp_attrs) if not resp_attrs.is_slo: # even if the original resp_attrs may have been SLO we may have @@ -1115,24 +1224,16 @@ class SloGetContext(WSGIContext): raise HTTPServerError(msg) return segments - def _build_resp_iter(self, req, segments, resp_attrs): + def _build_resp_iter(self, req, segments, byteranges): """ Build a response iterable for a GET request. :param req: the request object - :param resp_attrs: the slo attributes + :param segments: the list of seg_dicts + :param byteranges: a list of tuples representing byteranges :returns: a segmented iterable """ - if req.range: - byteranges = [ - # For some reason, swob.Range.ranges_for_length adds 1 to the - # last byte's position. - (start, end - 1) for start, end - in req.range.ranges_for_length(resp_attrs.slo_size)] - else: - byteranges = [(0, resp_attrs.slo_size - 1)] - ver, account, _junk = req.split_path(3, 3, rest_with_last=True) account = wsgi_to_str(account) plain_listing_iter = self._segment_listing_iterator( diff --git a/swift/common/request_helpers.py b/swift/common/request_helpers.py index bcc36f501a..e785941a23 100644 --- a/swift/common/request_helpers.py +++ b/swift/common/request_helpers.py @@ -95,6 +95,39 @@ def get_param(req, name, default=None): return value +def get_valid_part_num(req): + """ + Any non-range GET or HEAD request for a SLO object may include a + part-number parameter in query string. If the passed in request + includes a part-number parameter it will be parsed into a valid integer + and returned. If the passed in request does not include a part-number + param we will return None. If the part-number parameter is invalid for + the given request we will raise the appropriate HTTP exception + + :param req: the request object + + :returns: validated part-number value or None + :raises HTTPBadRequest: if request or part-number param is not valid + """ + part_number_param = get_param(req, 'part-number') + if part_number_param is None: + return None + try: + part_number = int(part_number_param) + if part_number <= 0: + raise ValueError + except ValueError: + raise HTTPBadRequest('Part number must be an integer greater ' + 'than 0') + + if req.range: + raise HTTPBadRequest(req=req, + body='Range requests are not supported ' + 'with part number queries') + + return part_number + + def validate_params(req, names): """ Get list of parameters from an HTTP request, validating the encoding of diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index e3b3dd93ec..3b90862b84 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -938,7 +938,7 @@ class File(Base): return True - def info(self, hdrs=None, parms=None, cfg=None): + def info(self, hdrs=None, parms=None, cfg=None, exp_status=200): if hdrs is None: hdrs = {} if parms is None: @@ -946,7 +946,7 @@ class File(Base): if cfg is None: cfg = {} if self.conn.make_request('HEAD', self.path, hdrs=hdrs, - parms=parms, cfg=cfg) != 200: + parms=parms, cfg=cfg) != exp_status: raise ResponseError(self.conn.response, 'HEAD', self.conn.make_path(self.path)) diff --git a/test/functional/test_object_versioning.py b/test/functional/test_object_versioning.py index 703521374b..32d0641165 100644 --- a/test/functional/test_object_versioning.py +++ b/test/functional/test_object_versioning.py @@ -2311,11 +2311,12 @@ class TestSloWithVersioning(TestObjectVersioningBase): '/', 1)[-1] return self._account_name - def _create_manifest(self, seg_name): + def _create_manifest(self, seg_names): # create a manifest in the versioning container file_item = self.container.file("my-slo-manifest") + manifest = [self.seg_info[seg_name] for seg_name in seg_names] resp = file_item.write( - json.dumps([self.seg_info[seg_name]]).encode('ascii'), + json.dumps(manifest).encode('ascii'), parms={'multipart-manifest': 'put'}, return_resp=True) version_id = resp.getheader('x-object-version-id') @@ -2340,9 +2341,10 @@ class TestSloWithVersioning(TestObjectVersioningBase): self.assertEqual(1, len(manifest)) key_map = {'etag': 'hash', 'size_bytes': 'bytes', 'path': 'name'} + for k_client, k_slo in key_map.items(): self.assertEqual(self.seg_info[seg_name][k_client], - manifest[0][k_slo]) + Utils.encode_if_py2(manifest[0][k_slo])) def _assert_is_object(self, file_item, seg_data, version_id=None): if version_id: @@ -2357,13 +2359,13 @@ class TestSloWithVersioning(TestObjectVersioningBase): self._tear_down_files(self.container) def test_slo_manifest_version(self): - file_item, v1_version_id = self._create_manifest('a') + file_item, v1_version_id = self._create_manifest(['a']) # sanity check: read the manifest, then the large object self._assert_is_manifest(file_item, 'a') self._assert_is_object(file_item, b'a') # upload new manifest - file_item, v2_version_id = self._create_manifest('b') + file_item, v2_version_id = self._create_manifest(['b']) # sanity check: read the manifest, then the large object self._assert_is_manifest(file_item, 'b') self._assert_is_object(file_item, b'b') @@ -2445,7 +2447,7 @@ class TestSloWithVersioning(TestObjectVersioningBase): self.assertEqual(409, caught.exception.status) def test_links_to_slo(self): - file_item, v1_version_id = self._create_manifest('a') + file_item, v1_version_id = self._create_manifest(['a']) slo_info = file_item.info() symlink_name = Utils.create_name() @@ -2463,6 +2465,123 @@ class TestSloWithVersioning(TestObjectVersioningBase): symlink.write(b'', hdrs=sym_headers) self.assertEqual(slo_info, symlink.info()) + def test_slo_HEAD_part_number_with_version(self): + file_item, version_id = self._create_manifest(['a', 'b']) + file_item.info(parms={'part-number': '1', + 'version-id': version_id}, + exp_status=206) + sizes = [seg['size_bytes'] + for seg in (self.seg_info['a'], self.seg_info['b'])] + total_size = sum(sizes) + resp = file_item.conn.response + self.assertEqual(version_id, resp.getheader('X-Object-Version-Id')) + self.assertEqual('2', resp.getheader('X-Parts-Count')) + self.assertEqual('bytes 0-%s/%s' % (sizes[0] - 1, total_size), + resp.getheader('Content-Range')) + + file_item.info(parms={'part-number': '2', + 'version-id': version_id}, + exp_status=206) + resp = file_item.conn.response + self.assertEqual(version_id, resp.getheader('X-Object-Version-Id')) + self.assertEqual('2', resp.getheader('X-Parts-Count')) + self.assertEqual('bytes %s-%s/%s' + % (sizes[1], total_size - 1, total_size), + resp.getheader('Content-Range')) + + file_item.info(parms={'part-number': '3', + 'version-id': version_id}, + exp_status=416) + resp = file_item.conn.response + self.assertEqual(version_id, resp.getheader('X-Object-Version-Id')) + self.assertEqual('2', resp.getheader('X-Parts-Count')) + self.assertEqual('bytes */%s' % total_size, + resp.getheader('Content-Range')) + + def test_slo_GET_part_number_with_version(self): + file_item, version_id = self._create_manifest(['a', 'b']) + body = file_item.read(parms={'part-number': '1', + 'version-id': version_id}) + sizes = [seg['size_bytes'] + for seg in (self.seg_info['a'], self.seg_info['b'])] + total_size = sum(sizes) + resp = file_item.conn.response + self.assertEqual(version_id, resp.getheader('X-Object-Version-Id')) + self.assertEqual('2', resp.getheader('X-Parts-Count')) + self.assertEqual('bytes 0-%s/%s' % (sizes[0] - 1, total_size), + resp.getheader('Content-Range')) + self.assertEqual(('a' * sizes[0]).encode('ascii'), body) + + body = file_item.read(parms={'part-number': '2', + 'version-id': version_id}) + resp = file_item.conn.response + self.assertEqual(version_id, resp.getheader('X-Object-Version-Id')) + self.assertEqual('2', resp.getheader('X-Parts-Count')) + self.assertEqual('bytes %s-%s/%s' + % (sizes[1], total_size - 1, total_size), + resp.getheader('Content-Range')) + self.assertEqual(('b' * sizes[0]).encode('ascii'), body) + + with self.assertRaises(ResponseError): + file_item.read(parms={'part-number': '3', + 'version-id': version_id}) + self.assertEqual(416, file_item.conn.response.status) + resp = file_item.conn.response + self.assertEqual(version_id, resp.getheader('X-Object-Version-Id')) + self.assertEqual('2', resp.getheader('X-Parts-Count')) + self.assertEqual('bytes */%s' % total_size, + resp.getheader('Content-Range')) + + def test_slo_HEAD_part_number_multiple_versions(self): + file_item, version_id_1 = self._create_manifest(['a', 'b']) + file_item, version_id_2 = self._create_manifest(['a']) + # older version has 2 parts + file_item.info(parms={'part-number': '2', + 'version-id': version_id_1}, + exp_status=206) + sizes = [seg['size_bytes'] + for seg in (self.seg_info['a'], self.seg_info['b'])] + total_size = sum(sizes) + resp = file_item.conn.response + self.assertEqual(version_id_1, resp.getheader('x-object-version-id')) + self.assertEqual('2', resp.getheader('X-Parts-Count')) + self.assertEqual('bytes %s-%s/%s' + % (sizes[1], total_size - 1, total_size), + resp.getheader('Content-Range')) + + # newer version has only 1 part + file_item.info(parms={'part-number': '1', + 'version-id': version_id_2}, + exp_status=206) + resp = file_item.conn.response + self.assertEqual(version_id_2, resp.getheader('X-Object-Version-Id')) + self.assertEqual('1', resp.getheader('X-Parts-Count')) + self.assertEqual('bytes %s-%s/%s' + % (0, sizes[0] - 1, sizes[0]), + resp.getheader('Content-Range')) + + file_item.info(parms={'part-number': '2', + 'version-id': version_id_2}, + exp_status=416) + resp = file_item.conn.response + self.assertEqual(version_id_2, resp.getheader('X-Object-Version-Id')) + self.assertEqual('1', resp.getheader('X-Parts-Count')) + self.assertEqual('bytes */%s' % sizes[0], + resp.getheader('Content-Range')) + + # current version == newer version has only 1 part + file_item.info(parms={'part-number': '2'}, + exp_status=416) + resp = file_item.conn.response + self.assertEqual(version_id_2, resp.getheader('X-Object-Version-Id')) + self.assertEqual('1', resp.getheader('X-Parts-Count')) + self.assertEqual('bytes */%s' % sizes[0], + resp.getheader('Content-Range')) + + +class TestSloWithVersioningUTF8(Base2, TestSloWithVersioning): + pass + class TestVersionsLocationWithVersioning(TestObjectVersioningBase): diff --git a/test/functional/test_slo.py b/test/functional/test_slo.py index d97954c6b1..21a855cfd2 100644 --- a/test/functional/test_slo.py +++ b/test/functional/test_slo.py @@ -293,6 +293,163 @@ class TestSlo(Base): (b'e', 1), ], group_file_contents(file_contents)) + def test_slo_multipart_delete_part_number_ignored(self): + # create a container just for this test because we're going to delete + # objects that we create + container = self.env.account.container(Utils.create_name()) + self.assertTrue(container.create()) + # create segments in same container + seg_info = self.env.create_segments(container) + file_item = container.file("manifest-abcde") + self.assertTrue(file_item.write( + json.dumps([seg_info['seg_a'], seg_info['seg_b'], + seg_info['seg_c'], seg_info['seg_d'], + seg_info['seg_e']]).encode('ascii'), + parms={'multipart-manifest': 'put'})) + # sanity check, we have SLO... + file_item.initialize(parms={'part-number': '5'}) + self.assertEqual( + file_item.conn.response.getheader('X-Static-Large-Object'), 'True') + self.assertEqual( + file_item.conn.response.getheader('X-Parts-Count'), '5') + self.assertEqual(6, len(container.files())) + + # part-number should be ignored + status = file_item.conn.make_request( + 'DELETE', file_item.path, + parms={'multipart-manifest': 'delete', + 'part-number': '2'}) + self.assertEqual(200, status) + # everything is gone + self.assertFalse(container.files()) + + def test_get_head_part_number_invalid(self): + file_item = self.env.container.file('manifest-abcde') + file_item.initialize() + ok_resp = file_item.conn.response + self.assertEqual(ok_resp.getheader('X-Static-Large-Object'), 'True') + self.assertEqual(ok_resp.getheader('Etag'), file_item.etag) + self.assertEqual(ok_resp.getheader('Content-Length'), + str(file_item.size)) + + # part-number is 1-indexed + self.assertRaises(ResponseError, file_item.read, + parms={'part-number': '0'}) + resp_body = file_item.conn.response.read() + self.assertEqual(400, file_item.conn.response.status, resp_body) + self.assertEqual(b'Part number must be an integer greater than 0', + resp_body) + + self.assertRaises(ResponseError, file_item.initialize, + parms={'part-number': '0'}) + resp_body = file_item.conn.response.read() + self.assertEqual(400, file_item.conn.response.status) + self.assertEqual(b'', resp_body) + + def test_get_head_part_number_out_of_range(self): + file_item = self.env.container.file('manifest-abcde') + file_item.initialize() + ok_resp = file_item.conn.response + self.assertEqual(ok_resp.getheader('X-Static-Large-Object'), 'True') + self.assertEqual(ok_resp.getheader('Etag'), file_item.etag) + self.assertEqual(ok_resp.getheader('Content-Length'), + str(file_item.size)) + manifest_etag = ok_resp.getheader('Manifest-Etag') + + def check_headers(resp): + self.assertEqual(resp.getheader('X-Static-Large-Object'), 'True') + self.assertEqual(resp.getheader('Etag'), file_item.etag) + self.assertEqual(resp.getheader('Manifest-Etag'), manifest_etag) + self.assertEqual(resp.getheader('X-Parts-Count'), '5') + self.assertEqual(resp.getheader('Content-Range'), + 'bytes */%s' % file_item.size) + + self.assertRaises(ResponseError, file_item.read, + parms={'part-number': '10001'}) + resp_body = file_item.conn.response.read() + self.assertEqual(416, file_item.conn.response.status, resp_body) + self.assertEqual(b'The requested part number is not satisfiable', + resp_body) + check_headers(file_item.conn.response) + self.assertEqual(file_item.conn.response.getheader('Content-Length'), + str(len(resp_body))) + + self.assertRaises(ResponseError, file_item.info, + parms={'part-number': '10001'}) + resp_body = file_item.conn.response.read() + self.assertEqual(416, file_item.conn.response.status) + self.assertEqual(b'', resp_body) + check_headers(file_item.conn.response) + self.assertEqual(file_item.conn.response.getheader('Content-Length'), + '0') + + def test_get_part_number_simple_manifest(self): + file_item = self.env.container.file('manifest-abcde') + seg_info_list = [ + self.env.seg_info["seg_%s" % letter] + for letter in ['a', 'b', 'c', 'd', 'e'] + ] + checksum = md5(usedforsecurity=False) + total_size = 0 + for seg_info in seg_info_list: + checksum.update(seg_info['etag'].encode('ascii')) + total_size += seg_info['size_bytes'] + slo_etag = checksum.hexdigest() + start = 0 + manifest_etag = None + for i, seg_info in enumerate(seg_info_list, start=1): + part_contents = file_item.read(parms={'part-number': i}) + self.assertEqual(len(part_contents), seg_info['size_bytes']) + headers = dict( + (h.lower(), v) + for h, v in file_item.conn.response.getheaders()) + self.assertEqual(headers['content-length'], + str(seg_info['size_bytes'])) + self.assertEqual(headers['etag'], '"%s"' % slo_etag) + if not manifest_etag: + manifest_etag = headers['x-manifest-etag'] + else: + self.assertEqual(headers['x-manifest-etag'], manifest_etag) + end = start + seg_info['size_bytes'] - 1 + self.assertEqual(headers['content-range'], + 'bytes %d-%d/%d' % (start, end, total_size), i) + self.assertEqual(headers['x-parts-count'], '5') + start = end + 1 + + def test_head_part_number_simple_manifest(self): + file_item = self.env.container.file('manifest-abcde') + seg_info_list = [ + self.env.seg_info["seg_%s" % letter] + for letter in ['a', 'b', 'c', 'd', 'e'] + ] + checksum = md5(usedforsecurity=False) + total_size = 0 + for seg_info in seg_info_list: + checksum.update(seg_info['etag'].encode('ascii')) + total_size += seg_info['size_bytes'] + slo_etag = checksum.hexdigest() + start = 0 + manifest_etag = None + for i, seg_info in enumerate(seg_info_list, start=1): + part_info = file_item.info(parms={'part-number': i}, + exp_status=206) + headers = dict( + (h.lower(), v) + for h, v in file_item.conn.response.getheaders()) + self.assertEqual(headers['content-length'], + str(seg_info['size_bytes'])) + self.assertEqual(headers['etag'], '"%s"' % slo_etag) + self.assertEqual(headers['etag'], part_info['etag']) + if not manifest_etag: + manifest_etag = headers['x-manifest-etag'] + else: + self.assertEqual(headers['x-manifest-etag'], manifest_etag) + end = start + seg_info['size_bytes'] - 1 + self.assertEqual(headers['content-range'], + 'bytes %d-%d/%d' % (start, end, total_size), i) + self.assertEqual(headers['x-parts-count'], '5') + start = end + 1 + def test_slo_container_listing(self): # the listing object size should equal the sum of the size of the # segments, not the size of the manifest body diff --git a/test/functional/tests.py b/test/functional/tests.py index ff3cf5c934..948647feba 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -53,6 +53,12 @@ def tearDownModule(): class Utils(object): + @classmethod + def encode_if_py2(cls, value): + if six.PY2 and isinstance(value, six.text_type): + return value.encode('utf8') + return value + @classmethod def create_ascii_name(cls, length=None): return uuid.uuid4().hex @@ -71,9 +77,7 @@ class Utils(object): u'\u5608\u3706\u1804\u0903\u03A9\u2603' ustr = u''.join([random.choice(utf8_chars) for x in range(length)]) - if six.PY2: - return ustr.encode('utf-8') - return ustr + return cls.encode_if_py2(ustr) create_name = create_ascii_name diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index 21f170fb1a..fbe250198f 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -19,6 +19,7 @@ from datetime import datetime import json import time import unittest +import string import mock from mock import patch @@ -1905,7 +1906,10 @@ class SloGETorHEADTestCase(SloTestCase): They're nothing special, just small regular objects with names that describe their content and size. """ - for letter, size in zip(letters, range(5, 5 * len(letters) + 1, 5)): + for i, letter in enumerate(string.ascii_lowercase): + if letter not in letters: + continue + size = (i + 1) * 5 body = letter * size path = '/v1/AUTH_test/%s/%s_%s' % (container, letter, size) self.app.register('GET', path, swob.HTTPOk, { @@ -1949,7 +1953,7 @@ class SloGETorHEADTestCase(SloTestCase): # has *no* content-type, both empty or missing Content-Type header # on ?multipart-manifest=put result in a default # "application/octet-stream" value being stored in the manifest - # metadata; sitll I wouldn't assert on this value in these tests or + # metadata; still I wouldn't assert on this value in these tests, # you may not be testing what you think you are - N.B. some tests # will override this value with the "extra_headers" param. 'Content-Type': 'application/octet-stream', @@ -1964,6 +1968,34 @@ class SloGETorHEADTestCase(SloTestCase): 'GET', '/v1/AUTH_test/%s/%s' % (container, obj_key), swob.HTTPOk, manifest_headers, manifest_json.encode('ascii')) + def _setup_manifest_single_segment(self): + """ + This manifest's segments are all regular objects. + """ + _single_segment_manifest = [ + {'name': '/gettest/b_50', 'hash': md5hex('b' * 50), 'bytes': '50', + 'content_type': 'text/plain'}, + ] + self._setup_manifest( + 'single-segment', _single_segment_manifest, + extra_headers={'X-Object-Meta-Nature': 'Regular'}, + container='gettest') + + def _setup_manifest_data(self): + _data_manifest = [ + { + 'data': base64.b64encode(b'123456').decode('ascii') + }, { + 'name': '/gettest/a_5', + 'hash': md5hex('a' * 5), + 'content_type': 'text/plain', + 'bytes': '5', + }, { + 'data': base64.b64encode(b'ABCDEF').decode('ascii') + }, + ] + self._setup_manifest('data', _data_manifest) + def _setup_manifest_bc(self): """ This manifest's segments are all regular objects. @@ -5632,6 +5664,701 @@ class TestSloConditionalGetNewManifest(TestSloConditionalGetOldManifest): modern_manifest_headers = True +class TestPartNumber(SloGETorHEADTestCase): + + modern_manifest_headers = True + + def setUp(self): + super(TestPartNumber, self).setUp() + self._setup_alphabet_objects('bcdj') + self._setup_manifest_bc() + self._setup_manifest_abcd() + self._setup_manifest_abcdefghijkl() + self._setup_manifest_bc_ranges() + self._setup_manifest_abcd_ranges() + self._setup_manifest_abcd_subranges() + self._setup_manifest_aabbccdd() + self._setup_manifest_single_segment() + + # this b_50 object doesn't follow the alphabet convention + self.app.register( + 'GET', '/v1/AUTH_test/gettest/b_50', + swob.HTTPPartialContent, {'Content-Length': '50', + 'Etag': md5hex('b' * 50)}, + 'b' * 50) + + self._setup_manifest_data() + + def test_head_part_number(self): + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-bc?part-number=1', + environ={'REQUEST_METHOD': 'HEAD'}) + status, headers, body = self.call_slo(req) + expected_calls = [ + ('HEAD', '/v1/AUTH_test/gettest/manifest-bc?part-number=1'), + ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=1') + ] + + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag) + self.assertEqual(headers['Content-Length'], '10') + self.assertEqual(headers['Content-Range'], 'bytes 0-9/25') + self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5) + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['X-Parts-Count'], '2') + self.assertEqual(headers['Content-Type'], 'application/octet-stream') + self.assertEqual(body, b'') # it's a HEAD request, after all + self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus') + self.assertEqual(self.app.calls, expected_calls) + + def test_head_part_number_refetch_path(self): + # verify that any modification of the request path by a downstream + # middleware is ignored when refetching + req = Request.blank( + '/v1/AUTH_test/gettest/mani?part-number=1', + environ={'REQUEST_METHOD': 'HEAD'}) + captured_calls = [] + orig_call = FakeSwift.__call__ + + def pseudo_middleware(app, env, start_response): + captured_calls.append((env['REQUEST_METHOD'], env['PATH_INFO'])) + # pretend another middleware modified the path + # note: for convenience, the path "modification" actually results + # in one of the pre-registered paths + env['PATH_INFO'] += 'fest-bc' + return orig_call(app, env, start_response) + + with patch.object(FakeSwift, '__call__', pseudo_middleware): + status, headers, body = self.call_slo(req) + + # pseudo-middleware gets the original path for the refetch + self.assertEqual([('HEAD', '/v1/AUTH_test/gettest/mani'), + ('GET', '/v1/AUTH_test/gettest/mani')], + captured_calls) + self.assertEqual(status, '206 Partial Content') + expected_calls = [ + # original path is modified... + ('HEAD', '/v1/AUTH_test/gettest/manifest-bc?part-number=1'), + # refetch: the *original* path is modified... + ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=1') + ] + self.assertEqual(self.app.calls, expected_calls) + + def test_get_part_number(self): + # part number 1 is b_10 + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-bc?part-number=1') + status, headers, body = self.call_slo(req) + expected_calls = [ + ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=1'), + ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get') + ] + + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag) + self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5) + self.assertEqual(headers['Content-Length'], '10') + self.assertEqual(headers['Content-Range'], 'bytes 0-9/25') + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['X-Parts-Count'], '2') + self.assertEqual(headers['Content-Type'], 'application/octet-stream') + self.assertEqual(body, b'b' * 10) + self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus') + self.assertEqual(self.app.calls, expected_calls) + + # part number 2 is c_15 + self.app.clear_calls() + expected_calls = [ + ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=2'), + ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get') + ] + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-bc?part-number=2') + status, headers, body = self.call_slo(req) + + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag) + self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5) + self.assertEqual(headers['Content-Length'], '15') + self.assertEqual(headers['Content-Range'], 'bytes 10-24/25') + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['X-Parts-Count'], '2') + self.assertEqual(headers['Content-Type'], 'application/octet-stream') + self.assertEqual(body, b'c' * 15) + self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus') + self.assertEqual(self.app.calls, expected_calls) + + # we now test it with single segment slo + self.app.clear_calls() + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-single-segment?part-number=1') + status, headers, body = self.call_slo(req) + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['Etag'], '"%s"' % + self.manifest_single_segment_slo_etag) + self.assertEqual(headers['X-Manifest-Etag'], + self.manifest_single_segment_json_md5) + self.assertEqual(headers['Content-Length'], '50') + self.assertEqual(headers['Content-Range'], 'bytes 0-49/50') + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['X-Object-Meta-Nature'], 'Regular') + self.assertEqual(headers['X-Parts-Count'], '1') + self.assertEqual(headers['Content-Type'], 'application/octet-stream') + expected_calls = [ + ('GET', '/v1/AUTH_test/gettest/manifest-single-segment?' + 'part-number=1'), + ('GET', '/v1/AUTH_test/gettest/b_50?multipart-manifest=get') + ] + self.assertEqual(self.app.calls, expected_calls) + + def test_get_part_number_sub_slo(self): + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-abcd?part-number=3') + status, headers, body = self.call_slo(req) + expected_calls = [ + ('GET', '/v1/AUTH_test/gettest/manifest-abcd?part-number=3'), + ('GET', '/v1/AUTH_test/gettest/d_20?multipart-manifest=get') + ] + + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_slo_etag) + self.assertEqual(headers['X-Manifest-Etag'], + self.manifest_abcd_json_md5) + self.assertEqual(headers['Content-Length'], '20') + self.assertEqual(headers['Content-Range'], 'bytes 30-49/50') + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['X-Parts-Count'], '3') + self.assertEqual(headers['Content-Type'], 'application/json') + self.assertEqual(body, b'd' * 20) + self.assertEqual(self.app.calls, expected_calls) + + self.app.clear_calls() + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-abcd?part-number=2') + status, headers, body = self.call_slo(req) + expected_calls = [ + ('GET', '/v1/AUTH_test/gettest/manifest-abcd?part-number=2'), + ('GET', '/v1/AUTH_test/gettest/manifest-bc'), + ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'), + ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get') + ] + + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_slo_etag) + self.assertEqual(headers['X-Manifest-Etag'], + self.manifest_abcd_json_md5) + self.assertEqual(headers['Content-Length'], '25') + self.assertEqual(headers['Content-Range'], 'bytes 5-29/50') + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['X-Parts-Count'], '3') + self.assertEqual(headers['Content-Type'], 'application/json') + self.assertEqual(body, b'b' * 10 + b'c' * 15) + self.assertEqual(self.app.calls, expected_calls) + + def test_get_part_number_large_manifest(self): + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-abcdefghijkl?part-number=10') + status, headers, body = self.call_slo(req) + expected_calls = [ + ('GET', '/v1/AUTH_test/gettest/manifest-abcdefghijkl?' + 'part-number=10'), + ('GET', '/v1/AUTH_test/gettest/j_50?multipart-manifest=get') + ] + + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['Etag'], '"%s"' % + self.manifest_abcdefghijkl_slo_etag) + self.assertEqual(headers['X-Manifest-Etag'], + self.manifest_abcdefghijkl_json_md5) + self.assertEqual(headers['Content-Length'], '50') + self.assertEqual(headers['Content-Range'], 'bytes 225-274/390') + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['X-Parts-Count'], '12') + self.assertEqual(headers['Content-Type'], 'application/octet-stream') + self.assertEqual(body, b'j' * 50) + self.assertEqual(self.app.calls, expected_calls) + + def test_part_number_with_range_segments(self): + req = Request.blank('/v1/AUTH_test/gettest/manifest-bc-ranges', + params={'part-number': 1}) + status, headers, body = self.call_slo(req) + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['Etag'], '"%s"' % + self.manifest_bc_ranges_slo_etag) + self.assertEqual(headers['X-Manifest-Etag'], + self.manifest_bc_ranges_json_md5) + self.assertEqual(headers['Content-Length'], '4') + self.assertEqual(headers['Content-Range'], + 'bytes 0-3/%s' % self.manifest_bc_ranges_slo_size) + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['X-Parts-Count'], '4') + self.assertEqual(body, b'b' * 4) + expected_calls = [ + ('GET', '/v1/AUTH_test/gettest/manifest-bc-ranges?part-number=1'), + ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get') + ] + self.assertEqual(self.app.calls, expected_calls) + # since the our requested part-number is range-segment we expect Range + # header on b_10 segment subrequest + self.assertEqual('bytes=4-7', + self.app.calls_with_headers[1].headers['Range']) + + def test_part_number_sub_ranges_manifest(self): + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=3') + + status, headers, body = self.call_slo(req) + expected_calls = [ + ('GET', '/v1/AUTH_test/gettest/manifest-abcd-subranges?' + 'part-number=3'), + ('GET', '/v1/AUTH_test/gettest/manifest-abcd-ranges'), + ('GET', '/v1/AUTH_test/gettest/manifest-bc-ranges'), + ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'), + ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get') + ] + + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['Etag'], '"%s"' % + self.manifest_abcd_subranges_slo_etag) + self.assertEqual(headers['X-Manifest-Etag'], + self.manifest_abcd_subranges_json_md5) + self.assertEqual(headers['Content-Length'], '5') + self.assertEqual(headers['Content-Range'], 'bytes 6-10/17') + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['X-Parts-Count'], '5') + self.assertEqual(headers['Content-Type'], 'application/json') + self.assertEqual(body, b'c' * 2 + b'b' * 3) + self.assertEqual(self.app.calls, expected_calls) + + def test_get_part_num_with_repeated_segments(self): + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-aabbccdd?part-number=3', + environ={'REQUEST_METHOD': 'GET'}) + + status, headers, body = self.call_slo(req) + expected_calls = [ + ('GET', '/v1/AUTH_test/gettest/manifest-aabbccdd?part-number=3'), + ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get') + ] + + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['Etag'], '"%s"' % + self.manifest_aabbccdd_slo_etag) + self.assertEqual(headers['X-Manifest-Etag'], + self.manifest_aabbccdd_json_md5) + self.assertEqual(headers['Content-Length'], '10') + self.assertEqual(headers['Content-Range'], 'bytes 10-19/100') + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['X-Parts-Count'], '8') + self.assertEqual(headers['Content-Type'], 'application/octet-stream') + self.assertEqual(body, b'b' * 10) + self.assertEqual(self.app.calls, expected_calls) + + def test_part_number_zero_invalid(self): + # part-number query param is 1-indexed, part-number=0 is no joy + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-bc?part-number=0') + status, headers, body = self.call_slo(req) + + self.assertEqual(status, '400 Bad Request') + self.assertNotIn('Content-Range', headers) + self.assertNotIn('Etag', headers) + self.assertNotIn('X-Static-Large-Object', headers) + self.assertNotIn('X-Parts-Count', headers) + self.assertEqual(body, + b'Part number must be an integer greater than 0') + expected_calls = [ + ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=0') + ] + self.assertEqual(expected_calls, self.app.calls) + + self.app.clear_calls() + self.slo.max_manifest_segments = 3999 + req = Request.blank('/v1/AUTH_test/gettest/manifest-bc', + params={'part-number': 0}) + status, headers, body = self.call_slo(req) + self.assertEqual(status, '400 Bad Request') + self.assertNotIn('Content-Range', headers) + self.assertNotIn('Etag', headers) + self.assertNotIn('X-Static-Large-Object', headers) + self.assertNotIn('X-Parts-Count', headers) + self.assertEqual(body, + b'Part number must be an integer greater than 0') + self.assertEqual(expected_calls, self.app.calls) + + def test_head_part_number_zero_invalid(self): + # you can HEAD part-number=0 either + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-bc', method='HEAD', + params={'part-number': 0}) + status, headers, body = self.call_slo(req) + + self.assertEqual(status, '400 Bad Request') + self.assertNotIn('Content-Range', headers) + self.assertNotIn('Etag', headers) + self.assertNotIn('X-Static-Large-Object', headers) + self.assertNotIn('X-Parts-Count', headers) + self.assertEqual(body, b'') # HEAD response, makes sense + expected_calls = [ + ('HEAD', '/v1/AUTH_test/gettest/manifest-bc?part-number=0') + ] + self.assertEqual(expected_calls, self.app.calls) + + def test_part_number_zero_invalid_on_subrange(self): + # either manifest, doesn't matter, part-number=0 is always invalid + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=0') + status, headers, body = self.call_slo(req) + self.assertEqual(status, '400 Bad Request') + self.assertNotIn('Content-Range', headers) + self.assertNotIn('Etag', headers) + self.assertNotIn('X-Static-Large-Object', headers) + self.assertNotIn('X-Parts-Count', headers) + self.assertEqual(body, + b'Part number must be an integer greater than 0') + expected_calls = [ + ('GET', + '/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=0') + ] + self.assertEqual(expected_calls, self.app.calls) + + def test_negative_part_number_invalid(self): + # negative numbers are never any good + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-bc?part-number=-1') + status, headers, body = self.call_slo(req) + self.assertEqual(status, '400 Bad Request') + self.assertNotIn('Content-Range', headers) + self.assertNotIn('Etag', headers) + self.assertNotIn('X-Static-Large-Object', headers) + self.assertNotIn('X-Parts-Count', headers) + self.assertEqual(body, + b'Part number must be an integer greater than 0') + expected_calls = [ + ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=-1') + ] + self.assertEqual(expected_calls, self.app.calls) + + def test_head_negative_part_number_invalid_on_subrange(self): + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-abcd-subranges', method='HEAD', + params={'part-number': '-1'}) + status, headers, body = self.call_slo(req) + self.assertEqual(status, '400 Bad Request') + self.assertNotIn('Content-Range', headers) + self.assertNotIn('Etag', headers) + self.assertNotIn('X-Static-Large-Object', headers) + self.assertNotIn('X-Parts-Count', headers) + self.assertEqual(body, b'') + expected_calls = [ + ('HEAD', + '/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=-1') + ] + self.assertEqual(expected_calls, self.app.calls) + + def test_head_non_integer_part_number_invalid(self): + # some kind of string is bad too + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-bc', method='HEAD', + params={'part-number': 'foo'}) + status, headers, body = self.call_slo(req) + self.assertEqual(status, '400 Bad Request') + self.assertEqual(body, b'') + expected_calls = [ + ('HEAD', '/v1/AUTH_test/gettest/manifest-bc?part-number=foo') + ] + self.assertEqual(expected_calls, self.app.calls) + + def test_get_non_integer_part_number_invalid(self): + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-bc', params={'part-number': 'foo'}) + status, headers, body = self.call_slo(req) + self.assertEqual(status, '400 Bad Request') + self.assertNotIn('Content-Range', headers) + self.assertNotIn('Etag', headers) + self.assertNotIn('X-Static-Large-Object', headers) + self.assertNotIn('X-Parts-Count', headers) + self.assertEqual(body, b'Part number must be an integer greater' + b' than 0') + expected_calls = [ + ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=foo') + ] + self.assertEqual(expected_calls, self.app.calls) + + def test_get_out_of_range_part_number(self): + # you can't go past the actual number of parts either + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-bc?part-number=4') + status, headers, body = self.call_slo(req) + self.assertEqual(status, '416 Requested Range Not Satisfiable') + self.assertEqual(headers['Content-Range'], + 'bytes */%d' % self.manifest_bc_slo_size) + self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag) + self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5) + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['X-Parts-Count'], '2') + self.assertEqual(int(headers['Content-Length']), len(body)) + self.assertEqual(body, b'The requested part number is not ' + b'satisfiable') + self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus') + expected_app_calls = [ + ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=4'), + ] + self.assertEqual(self.app.calls, expected_app_calls) + + self.app.clear_calls() + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-single-segment?part-number=2') + status, headers, body = self.call_slo(req) + self.assertEqual(status, '416 Requested Range Not Satisfiable') + self.assertEqual(headers['Content-Range'], + 'bytes */%d' % self.manifest_single_segment_slo_size) + self.assertEqual(int(headers['Content-Length']), len(body)) + self.assertEqual(headers['Etag'], + '"%s"' % self.manifest_single_segment_slo_etag) + self.assertEqual(headers['X-Manifest-Etag'], + self.manifest_single_segment_json_md5) + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['X-Parts-Count'], '1') + self.assertEqual(body, b'The requested part number is not ' + b'satisfiable') + self.assertEqual(headers['X-Object-Meta-Nature'], 'Regular') + expected_app_calls = [ + ('GET', '/v1/AUTH_test/gettest/manifest-single-segment?' + 'part-number=2'), + ] + self.assertEqual(self.app.calls, expected_app_calls) + + def test_head_out_of_range_part_number(self): + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-bc?part-number=4') + req.method = 'HEAD' + status, headers, body = self.call_slo(req) + self.assertEqual(status, '416 Requested Range Not Satisfiable') + self.assertEqual(headers['Content-Range'], + 'bytes */%d' % self.manifest_bc_slo_size) + self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag) + self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5) + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['X-Parts-Count'], '2') + self.assertEqual(int(headers['Content-Length']), len(body)) + self.assertEqual(body, b'') + self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus') + expected_app_calls = [ + ('HEAD', '/v1/AUTH_test/gettest/manifest-bc?part-number=4'), + # segments needed + ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=4') + ] + self.assertEqual(self.app.calls, expected_app_calls) + + def test_part_number_exceeds_max_manifest_segments_is_ok(self): + # verify that an existing part can be fetched regardless of the current + # max_manifest_segments + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-bc?part-number=2') + self.slo.max_manifest_segments = 1 + status, headers, body = self.call_slo(req) + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['Etag'], '"%s"' % self.manifest_bc_slo_etag) + self.assertEqual(headers['X-Manifest-Etag'], self.manifest_bc_json_md5) + self.assertEqual(headers['Content-Length'], '15') + self.assertEqual(headers['Content-Range'], 'bytes 10-24/25') + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['X-Parts-Count'], '2') + self.assertEqual(headers['Content-Type'], 'application/octet-stream') + self.assertEqual(body, b'c' * 15) + expected_calls = [ + ('GET', '/v1/AUTH_test/gettest/manifest-bc?part-number=2'), + ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get') + ] + self.assertEqual(self.app.calls, expected_calls) + + def test_part_number_ignored_for_non_slo_object(self): + # verify that a part-number param is ignored for a non-slo object + def do_test(query_string): + self.app.clear_calls() + req = Request.blank( + '/v1/AUTH_test/gettest/c_15?%s' % query_string) + self.slo.max_manifest_segments = 1 + status, headers, body = self.call_slo(req) + self.assertEqual(status, '200 OK') + self.assertEqual(headers['Etag'], '%s' % md5hex('c' * 15)) + self.assertEqual(headers['Content-Length'], '15') + self.assertEqual(body, b'c' * 15) + self.assertEqual(1, self.app.call_count) + method, path = self.app.calls[0] + actual_req = Request.blank(path, method=method) + self.assertEqual(req.path, actual_req.path) + self.assertEqual(req.params, actual_req.params) + + do_test('part-number=-1') + do_test('part-number=0') + do_test('part-number=1') + do_test('part-number=2') + do_test('part-number=foo') + do_test('part-number=foo&multipart-manifest=get') + + def test_part_number_ignored_for_non_slo_object_with_range(self): + # verify that a part-number param is ignored for a non-slo object + def do_test(query_string): + self.app.clear_calls() + req = Request.blank( + '/v1/AUTH_test/gettest/c_15?%s' % query_string, + headers={'Range': 'bytes=1-2'}) + self.slo.max_manifest_segments = 1 + status, headers, body = self.call_slo(req) + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['Etag'], '%s' % md5hex('c' * 15)) + self.assertEqual(headers['Content-Length'], '2') + self.assertEqual(headers['Content-Range'], 'bytes 1-2/15') + self.assertEqual(body, b'c' * 2) + self.assertEqual(1, self.app.call_count) + method, path = self.app.calls[0] + actual_req = Request.blank(path, method=method) + self.assertEqual(req.path, actual_req.path) + self.assertEqual(req.params, actual_req.params) + + do_test('part-number=-1') + do_test('part-number=0') + do_test('part-number=1') + do_test('part-number=2') + do_test('part-number=foo') + do_test('part-number=foo&multipart-manifest=get') + + def test_part_number_ignored_for_manifest_get(self): + def do_test(query_string): + self.app.clear_calls() + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-bc?%s' % query_string) + self.slo.max_manifest_segments = 1 + status, headers, body = self.call_slo(req) + self.assertEqual(status, '200 OK') + self.assertEqual(headers['Etag'], self.manifest_bc_json_md5) + self.assertEqual(headers['Content-Length'], + str(self.manifest_bc_json_size)) + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['Content-Type'], + 'application/json; charset=utf-8') + self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus') + self.assertEqual(1, self.app.call_count) + method, path = self.app.calls[0] + actual_req = Request.blank(path, method=method) + self.assertEqual(req.path, actual_req.path) + self.assertEqual(req.params, actual_req.params) + + do_test('part-number=-1&multipart-manifest=get') + do_test('part-number=0&multipart-manifest=get') + do_test('part-number=1&multipart-manifest=get') + do_test('part-number=2&multipart-manifest=get') + do_test('part-number=foo&multipart-manifest=get') + + def test_head_out_of_range_part_number_on_subrange(self): + # you can't go past the actual number of parts either + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-abcd-subranges', + method='HEAD', + params={'part-number': 6}) + expected_calls = [ + ('HEAD', '/v1/AUTH_test/gettest/manifest-abcd-subranges?' + 'part-number=6'), + ('GET', '/v1/AUTH_test/gettest/manifest-abcd-subranges?' + 'part-number=6')] + status, headers, body = self.call_slo(req) + self.assertEqual(status, '416 Requested Range Not Satisfiable') + self.assertEqual(headers['Content-Range'], + 'bytes */%d' % self.manifest_abcd_subranges_slo_size) + self.assertEqual(headers['Etag'], + '"%s"' % self.manifest_abcd_subranges_slo_etag) + self.assertEqual(headers['X-Manifest-Etag'], + self.manifest_abcd_subranges_json_md5) + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['X-Parts-Count'], '5') + self.assertEqual(int(headers['Content-Length']), len(body)) + self.assertEqual(body, b'') + self.assertEqual(self.app.calls, expected_calls) + + def test_range_with_part_number_is_error(self): + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=2', + headers={'Range': 'bytes=4-12'}) + status, headers, body = self.call_slo(req) + self.assertEqual(status, '400 Bad Request') + self.assertNotIn('Content-Range', headers) + self.assertNotIn('Etag', headers) + self.assertNotIn('X-Static-Large-Object', headers) + self.assertNotIn('X-Parts-Count', headers) + self.assertEqual(body, b'Range requests are not supported with ' + b'part number queries') + expected_calls = [ + ('GET', + '/v1/AUTH_test/gettest/manifest-abcd-subranges?part-number=2') + ] + self.assertEqual(expected_calls, self.app.calls) + + def test_head_part_number_subrange(self): + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-abcd-subranges', + method='HEAD', params={'part-number': 2}) + status, headers, body = self.call_slo(req) + + # Range header can be ignored in a HEAD request + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['Etag'], + '"%s"' % self.manifest_abcd_subranges_slo_etag) + self.assertEqual(headers['Content-Length'], '1') + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['Content-Type'], 'application/json') + self.assertEqual(headers['X-Parts-Count'], '5') + self.assertEqual(body, b'') # it's a HEAD request, after all + expected_calls = [ + ('HEAD', '/v1/AUTH_test/gettest/manifest-abcd-subranges' + '?part-number=2'), + ('GET', '/v1/AUTH_test/gettest/manifest-abcd-subranges' + '?part-number=2'), + ] + self.assertEqual(self.app.calls, expected_calls) + + def test_head_part_number_data_manifest(self): + req = Request.blank( + '/v1/AUTH_test/c/manifest-data', + method='HEAD', params={'part-number': 1}) + status, headers, body = self.call_slo(req) + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['Etag'], + '"%s"' % self.manifest_data_slo_etag) + self.assertEqual(headers['Content-Length'], '6') + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['X-Parts-Count'], '3') + self.assertEqual(body, b'') # it's a HEAD request, after all + expected_calls = [ + ('HEAD', '/v1/AUTH_test/c/manifest-data?part-number=1'), + ('GET', '/v1/AUTH_test/c/manifest-data?part-number=1'), + ] + self.assertEqual(self.app.calls, expected_calls) + + def test_get_part_number_data_manifest(self): + req = Request.blank( + '/v1/AUTH_test/c/manifest-data', + params={'part-number': 3}) + status, headers, body = self.call_slo(req) + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['Etag'], + '"%s"' % self.manifest_data_slo_etag) + self.assertEqual(headers['Content-Length'], '6') + self.assertEqual(headers['X-Static-Large-Object'], 'true') + self.assertEqual(headers['X-Parts-Count'], '3') + self.assertEqual(body, b'ABCDEF') + expected_calls = [ + ('GET', '/v1/AUTH_test/c/manifest-data?part-number=3'), + ] + self.assertEqual(self.app.calls, expected_calls) + + +class TestPartNumberLegacyManifest(TestPartNumber): + + modern_manifest_headers = False + + class TestSloBulkDeleter(unittest.TestCase): def test_reused_logger(self): slo_mware = slo.filter_factory({})('fake app')