From 1457155212acde05b4e020f40c9645eda9481077 Mon Sep 17 00:00:00 2001 From: Kota Tsuyuzaki Date: Thu, 29 Mar 2018 15:32:20 +0900 Subject: [PATCH] Change sysmeta name from swift3 to s3api And add a magic to translate existing swift3 sysmeta to s3api sysmeta at swift.common.middleware.s3api.response.Response to upgrade easy from swift3 to s3api. Change-Id: I4c11274a69863be04fd2b1ca4c1d4d9aadbff2de --- etc/proxy-server.conf-sample | 2 +- swift/common/middleware/s3api/response.py | 32 ++++++++++-- swift/common/middleware/s3api/utils.py | 6 +-- .../common/middleware/s3api/test_bucket.py | 10 ++-- .../middleware/s3api/test_multi_upload.py | 12 ++--- .../common/middleware/s3api/test_response.py | 49 ++++++++++++++++++- 6 files changed, 91 insertions(+), 20 deletions(-) diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index b3af10b3bd..9b8b61a41b 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -480,7 +480,7 @@ use = egg:swift#s3api # max_multi_delete_objects = 1000 # # If set to 'true', s3api uses its own metadata for ACL -# (e.g. X-Container-Sysmeta-Swift3-Acl) to achieve the best S3 compatibility. +# (e.g. X-Container-Sysmeta-S3Api-Acl) to achieve the best S3 compatibility. # If set to 'false', s3api tries to use Swift ACL (e.g. X-Container-Read) # instead of S3 ACL as far as possible. If you want to keep backward # compatibility with Swift3 1.7 or earlier, set false here diff --git a/swift/common/middleware/s3api/response.py b/swift/common/middleware/s3api/response.py index f48969eff9..ea4eb3cb59 100644 --- a/swift/common/middleware/s3api/response.py +++ b/swift/common/middleware/s3api/response.py @@ -19,6 +19,7 @@ from functools import partial from swift.common import swob from swift.common.utils import config_true_value +from swift.common.request_helpers import is_sys_meta from swift.common.middleware.s3api.utils import snake_to_camel, sysmeta_prefix from swift.common.middleware.s3api.etree import Element, SubElement, tostring @@ -87,11 +88,34 @@ class Response(ResponseBase, swob.Response): headers = HeaderKeyDict() self.is_slo = False + def is_swift3_sysmeta(sysmeta_key, server_type): + swift3_sysmeta_prefix = ( + 'x-%s-sysmeta-swift3' % server_type).lower() + if sysmeta_key.lower().startswith(swift3_sysmeta_prefix): + return True + return False + + def is_s3api_sysmeta(sysmeta_key, server_type): + s3api_sysmeta_prefix = sysmeta_prefix(_server_type).lower() + if sysmeta_key.lower().startswith(s3api_sysmeta_prefix): + return True + return False + for key, val in self.headers.iteritems(): - _key = key.lower() - if _key.startswith(sysmeta_prefix('object')) or \ - _key.startswith(sysmeta_prefix('container')): - sw_sysmeta_headers[key] = val + if is_sys_meta('object', key) or is_sys_meta('container', key): + _server_type = key.split('-')[1] + if is_swift3_sysmeta(key, _server_type): + # To be compatible with older swift3, translate swift3 + # sysmeta to s3api sysmeta here + key = sysmeta_prefix(_server_type) + \ + key[len('x-%s-sysmeta-swift3-' % _server_type):] + + if key not in sw_sysmeta_headers: + # To avoid overwrite s3api sysmeta by older swift3 + # sysmeta set the key only when the key does not exist + sw_sysmeta_headers[key] = val + elif is_s3api_sysmeta(key, _server_type): + sw_sysmeta_headers[key] = val else: sw_headers[key] = val diff --git a/swift/common/middleware/s3api/utils.py b/swift/common/middleware/s3api/utils.py index 9d4bd4aea1..813d979181 100644 --- a/swift/common/middleware/s3api/utils.py +++ b/swift/common/middleware/s3api/utils.py @@ -30,10 +30,10 @@ def sysmeta_prefix(resource): """ Returns the system metadata prefix for given resource type. """ - if resource == 'object': - return 'x-object-sysmeta-swift3-' + if resource.lower() == 'object': + return 'x-object-sysmeta-s3api-' else: - return 'x-container-sysmeta-swift3-' + return 'x-container-sysmeta-s3api-' def sysmeta_header(resource, name): diff --git a/test/unit/common/middleware/s3api/test_bucket.py b/test/unit/common/middleware/s3api/test_bucket.py index 0711c32657..e98e0e03ac 100644 --- a/test/unit/common/middleware/s3api/test_bucket.py +++ b/test/unit/common/middleware/s3api/test_bucket.py @@ -571,7 +571,7 @@ class TestS3ApiBucket(S3ApiTestCase): _, _, headers = self.swift.calls_with_headers[-1] self.assertTrue('X-Container-Read' in headers) self.assertEqual(headers.get('X-Container-Read'), '.r:*,.rlistings') - self.assertTrue('X-Container-Sysmeta-Swift3-Acl' not in headers) + self.assertNotIn('X-Container-Sysmeta-S3api-Acl', headers) @s3acl(s3acl_only=True) def test_bucket_PUT_with_canned_s3acl(self): @@ -586,10 +586,10 @@ class TestS3ApiBucket(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') _, _, headers = self.swift.calls_with_headers[-1] - self.assertTrue('X-Container-Read' not in headers) - self.assertTrue('X-Container-Sysmeta-Swift3-Acl' in headers) - self.assertEqual(headers.get('X-Container-Sysmeta-Swift3-Acl'), - acl['x-container-sysmeta-swift3-acl']) + self.assertNotIn('X-Container-Read', headers) + self.assertIn('X-Container-Sysmeta-S3api-Acl', headers) + self.assertEqual(headers.get('X-Container-Sysmeta-S3api-Acl'), + acl['x-container-sysmeta-s3api-acl']) @s3acl def test_bucket_PUT_with_location_error(self): diff --git a/test/unit/common/middleware/s3api/test_multi_upload.py b/test/unit/common/middleware/s3api/test_multi_upload.py index 4facf1b615..51cb58f0da 100644 --- a/test/unit/common/middleware/s3api/test_multi_upload.py +++ b/test/unit/common/middleware/s3api/test_multi_upload.py @@ -92,8 +92,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase): swob.HTTPOk, {'x-object-meta-foo': 'bar', 'content-type': 'application/directory', - 'x-object-sysmeta-swift3-has-content-type': 'yes', - 'x-object-sysmeta-swift3-content-type': + 'x-object-sysmeta-s3api-has-content-type': 'yes', + 'x-object-sysmeta-s3api-content-type': 'baz/quux'}, None) self.swift.register('PUT', segment_bucket + '/object/X', swob.HTTPCreated, {}, None) @@ -580,9 +580,9 @@ class TestS3ApiMultiUpload(S3ApiTestCase): _, _, req_headers = self.swift.calls_with_headers[-1] self.assertEqual(req_headers.get('X-Object-Meta-Foo'), 'bar') self.assertEqual(req_headers.get( - 'X-Object-Sysmeta-Swift3-Has-Content-Type'), 'yes') + 'X-Object-Sysmeta-S3api-Has-Content-Type'), 'yes') self.assertEqual(req_headers.get( - 'X-Object-Sysmeta-Swift3-Content-Type'), 'cat/picture') + 'X-Object-Sysmeta-S3api-Content-Type'), 'cat/picture') tmpacl_header = req_headers.get(sysmeta_header('object', 'tmpacl')) self.assertTrue(tmpacl_header) acl_header = encode_acl('object', @@ -609,7 +609,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase): _, _, req_headers = self.swift.calls_with_headers[-1] self.assertEqual(req_headers.get('X-Object-Meta-Foo'), 'bar') self.assertEqual(req_headers.get( - 'X-Object-Sysmeta-Swift3-Has-Content-Type'), 'no') + 'X-Object-Sysmeta-S3api-Has-Content-Type'), 'no') tmpacl_header = req_headers.get(sysmeta_header('object', 'tmpacl')) self.assertTrue(tmpacl_header) acl_header = encode_acl('object', @@ -708,7 +708,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase): def test_object_multipart_upload_complete_no_content_type(self): self.swift.register_unconditionally( 'HEAD', '/v1/AUTH_test/bucket+segments/object/X', - swob.HTTPOk, {"X-Object-Sysmeta-Swift3-Has-Content-Type": "no"}, + swob.HTTPOk, {"X-Object-Sysmeta-S3api-Has-Content-Type": "no"}, None) req = Request.blank('/bucket/object?uploadId=X', diff --git a/test/unit/common/middleware/s3api/test_response.py b/test/unit/common/middleware/s3api/test_response.py index e342bfc496..de442f9d13 100644 --- a/test/unit/common/middleware/s3api/test_response.py +++ b/test/unit/common/middleware/s3api/test_response.py @@ -16,10 +16,12 @@ import unittest from swift.common.swob import Response +from swift.common.utils import HeaderKeyDict from swift.common.middleware.s3api.response import Response as S3Response +from swift.common.middleware.s3api.utils import sysmeta_prefix -class TestRequest(unittest.TestCase): +class TestResponse(unittest.TestCase): def test_from_swift_resp_slo(self): for expected, header_vals in \ ((True, ('true', '1')), (False, ('false', 'ugahhh', None))): @@ -28,6 +30,51 @@ class TestRequest(unittest.TestCase): s3resp = S3Response.from_swift_resp(resp) self.assertEqual(expected, s3resp.is_slo) + def test_response_s3api_sysmeta_headers(self): + for _server_type in ('object', 'container'): + swift_headers = HeaderKeyDict( + {sysmeta_prefix(_server_type) + 'test': 'ok'}) + resp = Response(headers=swift_headers) + s3resp = S3Response.from_swift_resp(resp) + self.assertEqual(swift_headers, s3resp.sysmeta_headers) + + def test_response_s3api_sysmeta_headers_ignore_other_sysmeta(self): + for _server_type in ('object', 'container'): + swift_headers = HeaderKeyDict( + # sysmeta not leading sysmeta_prefix even including s3api word + {'x-%s-sysmeta-test-s3api' % _server_type: 'ok', + sysmeta_prefix(_server_type) + 'test': 'ok'}) + resp = Response(headers=swift_headers) + s3resp = S3Response.from_swift_resp(resp) + expected_headers = HeaderKeyDict( + {sysmeta_prefix(_server_type) + 'test': 'ok'}) + self.assertEqual(expected_headers, s3resp.sysmeta_headers) + + def test_response_s3api_sysmeta_from_swift3_sysmeta(self): + for _server_type in ('object', 'container'): + # swift could return older swift3 sysmeta + swift_headers = HeaderKeyDict( + {('x-%s-sysmeta-swift3-' % _server_type) + 'test': 'ok'}) + resp = Response(headers=swift_headers) + s3resp = S3Response.from_swift_resp(resp) + expected_headers = HeaderKeyDict( + {sysmeta_prefix(_server_type) + 'test': 'ok'}) + # but Response class should translates as s3api sysmeta + self.assertEqual(expected_headers, s3resp.sysmeta_headers) + + def test_response_swift3_sysmeta_does_not_overwrite_s3api_sysmeta(self): + for _server_type in ('object', 'container'): + # same key name except sysmeta prefix + swift_headers = HeaderKeyDict( + {('x-%s-sysmeta-swift3-' % _server_type) + 'test': 'ng', + sysmeta_prefix(_server_type) + 'test': 'ok'}) + resp = Response(headers=swift_headers) + s3resp = S3Response.from_swift_resp(resp) + expected_headers = HeaderKeyDict( + {sysmeta_prefix(_server_type) + 'test': 'ok'}) + # but only s3api sysmeta remains in the response sysmeta_headers + self.assertEqual(expected_headers, s3resp.sysmeta_headers) + if __name__ == '__main__': unittest.main()