openstack
/
swift
Code Issues Proposed changes

Change sysmeta name from swift3 to s3api 

And add a magic to translate existing swift3 sysmeta to s3api sysmeta
at swift.common.middleware.s3api.response.Response to upgrade easy from swift3
to s3api.

Change-Id: I4c11274a69863be04fd2b1ca4c1d4d9aadbff2de
This commit is contained in:
Kota Tsuyuzaki 2018-03-29 15:32:20 +09:00
parent c019720275
commit 1457155212
6 changed files with 91 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
etc/proxy-server.conf-sample
View File

@ -480,7 +480,7 @@ use = egg:swift#s3api
# max_multi_delete_objects = 1000
#
# If set to 'true', s3api uses its own metadata for ACL
# (e.g. X-Container-Sysmeta-Swift3-Acl) to achieve the best S3 compatibility.
# (e.g. X-Container-Sysmeta-S3Api-Acl) to achieve the best S3 compatibility.
# If set to 'false', s3api tries to use Swift ACL (e.g. X-Container-Read)
# instead of S3 ACL as far as possible. If you want to keep backward
# compatibility with Swift3 1.7 or earlier, set false here

32
swift/common/middleware/s3api/response.py
View File

@ -19,6 +19,7 @@ from functools import partial
from swift.common import swob
from swift.common.utils import config_true_value
from swift.common.request_helpers import is_sys_meta
from swift.common.middleware.s3api.utils import snake_to_camel, sysmeta_prefix
from swift.common.middleware.s3api.etree import Element, SubElement, tostring
@ -87,11 +88,34 @@ class Response(ResponseBase, swob.Response):
headers = HeaderKeyDict()
self.is_slo = False
def is_swift3_sysmeta(sysmeta_key, server_type):
swift3_sysmeta_prefix = (
'x-%s-sysmeta-swift3' % server_type).lower()
if sysmeta_key.lower().startswith(swift3_sysmeta_prefix):
return True
return False
def is_s3api_sysmeta(sysmeta_key, server_type):
s3api_sysmeta_prefix = sysmeta_prefix(_server_type).lower()
if sysmeta_key.lower().startswith(s3api_sysmeta_prefix):
return True
return False
for key, val in self.headers.iteritems():
_key = key.lower()
if _key.startswith(sysmeta_prefix('object')) or \
_key.startswith(sysmeta_prefix('container')):
sw_sysmeta_headers[key] = val
if is_sys_meta('object', key) or is_sys_meta('container', key):
_server_type = key.split('-')[1]
if is_swift3_sysmeta(key, _server_type):
# To be compatible with older swift3, translate swift3
# sysmeta to s3api sysmeta here
key = sysmeta_prefix(_server_type) + \
key[len('x-%s-sysmeta-swift3-' % _server_type):]
if key not in sw_sysmeta_headers:
# To avoid overwrite s3api sysmeta by older swift3
# sysmeta set the key only when the key does not exist
sw_sysmeta_headers[key] = val
elif is_s3api_sysmeta(key, _server_type):
sw_sysmeta_headers[key] = val
else:
sw_headers[key] = val

6
swift/common/middleware/s3api/utils.py
View File

@ -30,10 +30,10 @@ def sysmeta_prefix(resource):
"""
Returns the system metadata prefix for given resource type.
"""
if resource == 'object':
return 'x-object-sysmeta-swift3-'
if resource.lower() == 'object':
return 'x-object-sysmeta-s3api-'
else:
return 'x-container-sysmeta-swift3-'
return 'x-container-sysmeta-s3api-'
def sysmeta_header(resource, name):

10
test/unit/common/middleware/s3api/test_bucket.py
View File

@ -571,7 +571,7 @@ class TestS3ApiBucket(S3ApiTestCase):
_, _, headers = self.swift.calls_with_headers[-1]
self.assertTrue('X-Container-Read' in headers)
self.assertEqual(headers.get('X-Container-Read'), '.r:*,.rlistings')
self.assertTrue('X-Container-Sysmeta-Swift3-Acl' not in headers)
self.assertNotIn('X-Container-Sysmeta-S3api-Acl', headers)
@s3acl(s3acl_only=True)
def test_bucket_PUT_with_canned_s3acl(self):
@ -586,10 +586,10 @@ class TestS3ApiBucket(S3ApiTestCase):
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
_, _, headers = self.swift.calls_with_headers[-1]
self.assertTrue('X-Container-Read' not in headers)
self.assertTrue('X-Container-Sysmeta-Swift3-Acl' in headers)
self.assertEqual(headers.get('X-Container-Sysmeta-Swift3-Acl'),
acl['x-container-sysmeta-swift3-acl'])
self.assertNotIn('X-Container-Read', headers)
self.assertIn('X-Container-Sysmeta-S3api-Acl', headers)
self.assertEqual(headers.get('X-Container-Sysmeta-S3api-Acl'),
acl['x-container-sysmeta-s3api-acl'])
@s3acl
def test_bucket_PUT_with_location_error(self):

12
test/unit/common/middleware/s3api/test_multi_upload.py
View File

@ -92,8 +92,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
swob.HTTPOk,
{'x-object-meta-foo': 'bar',
'content-type': 'application/directory',
'x-object-sysmeta-swift3-has-content-type': 'yes',
'x-object-sysmeta-swift3-content-type':
'x-object-sysmeta-s3api-has-content-type': 'yes',
'x-object-sysmeta-s3api-content-type':
'baz/quux'}, None)
self.swift.register('PUT', segment_bucket + '/object/X',
swob.HTTPCreated, {}, None)
@ -580,9 +580,9 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
_, _, req_headers = self.swift.calls_with_headers[-1]
self.assertEqual(req_headers.get('X-Object-Meta-Foo'), 'bar')
self.assertEqual(req_headers.get(
'X-Object-Sysmeta-Swift3-Has-Content-Type'), 'yes')
'X-Object-Sysmeta-S3api-Has-Content-Type'), 'yes')
self.assertEqual(req_headers.get(
'X-Object-Sysmeta-Swift3-Content-Type'), 'cat/picture')
'X-Object-Sysmeta-S3api-Content-Type'), 'cat/picture')
tmpacl_header = req_headers.get(sysmeta_header('object', 'tmpacl'))
self.assertTrue(tmpacl_header)
acl_header = encode_acl('object',
@ -609,7 +609,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
_, _, req_headers = self.swift.calls_with_headers[-1]
self.assertEqual(req_headers.get('X-Object-Meta-Foo'), 'bar')
self.assertEqual(req_headers.get(
'X-Object-Sysmeta-Swift3-Has-Content-Type'), 'no')
'X-Object-Sysmeta-S3api-Has-Content-Type'), 'no')
tmpacl_header = req_headers.get(sysmeta_header('object', 'tmpacl'))
self.assertTrue(tmpacl_header)
acl_header = encode_acl('object',
@ -708,7 +708,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
def test_object_multipart_upload_complete_no_content_type(self):
self.swift.register_unconditionally(
'HEAD', '/v1/AUTH_test/bucket+segments/object/X',
swob.HTTPOk, {"X-Object-Sysmeta-Swift3-Has-Content-Type": "no"},
swob.HTTPOk, {"X-Object-Sysmeta-S3api-Has-Content-Type": "no"},
None)
req = Request.blank('/bucket/object?uploadId=X',

49
test/unit/common/middleware/s3api/test_response.py
View File

@ -16,10 +16,12 @@
import unittest
from swift.common.swob import Response
from swift.common.utils import HeaderKeyDict
from swift.common.middleware.s3api.response import Response as S3Response
from swift.common.middleware.s3api.utils import sysmeta_prefix
class TestRequest(unittest.TestCase):
class TestResponse(unittest.TestCase):
def test_from_swift_resp_slo(self):
for expected, header_vals in \
((True, ('true', '1')), (False, ('false', 'ugahhh', None))):
@ -28,6 +30,51 @@ class TestRequest(unittest.TestCase):
s3resp = S3Response.from_swift_resp(resp)
self.assertEqual(expected, s3resp.is_slo)
def test_response_s3api_sysmeta_headers(self):
for _server_type in ('object', 'container'):
swift_headers = HeaderKeyDict(
{sysmeta_prefix(_server_type) + 'test': 'ok'})
resp = Response(headers=swift_headers)
s3resp = S3Response.from_swift_resp(resp)
self.assertEqual(swift_headers, s3resp.sysmeta_headers)
def test_response_s3api_sysmeta_headers_ignore_other_sysmeta(self):
for _server_type in ('object', 'container'):
swift_headers = HeaderKeyDict(
# sysmeta not leading sysmeta_prefix even including s3api word
{'x-%s-sysmeta-test-s3api' % _server_type: 'ok',
sysmeta_prefix(_server_type) + 'test': 'ok'})
resp = Response(headers=swift_headers)
s3resp = S3Response.from_swift_resp(resp)
expected_headers = HeaderKeyDict(
{sysmeta_prefix(_server_type) + 'test': 'ok'})
self.assertEqual(expected_headers, s3resp.sysmeta_headers)
def test_response_s3api_sysmeta_from_swift3_sysmeta(self):
for _server_type in ('object', 'container'):
# swift could return older swift3 sysmeta
swift_headers = HeaderKeyDict(
{('x-%s-sysmeta-swift3-' % _server_type) + 'test': 'ok'})
resp = Response(headers=swift_headers)
s3resp = S3Response.from_swift_resp(resp)
expected_headers = HeaderKeyDict(
{sysmeta_prefix(_server_type) + 'test': 'ok'})
# but Response class should translates as s3api sysmeta
self.assertEqual(expected_headers, s3resp.sysmeta_headers)
def test_response_swift3_sysmeta_does_not_overwrite_s3api_sysmeta(self):
for _server_type in ('object', 'container'):
# same key name except sysmeta prefix
swift_headers = HeaderKeyDict(
{('x-%s-sysmeta-swift3-' % _server_type) + 'test': 'ng',
sysmeta_prefix(_server_type) + 'test': 'ok'})
resp = Response(headers=swift_headers)
s3resp = S3Response.from_swift_resp(resp)
expected_headers = HeaderKeyDict(
{sysmeta_prefix(_server_type) + 'test': 'ok'})
# but only s3api sysmeta remains in the response sysmeta_headers
self.assertEqual(expected_headers, s3resp.sysmeta_headers)
if __name__ == '__main__':
unittest.main()