Copy headers correctly when copying object
Fix copy middleware so that all client-defined object headers that object servers allow to be persisted are copied. For example, content-encoding and content-disposition will now be copied. Fix treatment of x-fresh-metadata header so that, when it is used, new object sysmeta is applied to the object copy in the same way as a copy without x-fresh-metadata. Remove unnecessary passing of original request headers to sink PUT request constructor: passing the environ is sufficient to have the new request inherit the original's headers. Add tests for this change and to verify that content-type gets either copied or updated if supplied with the copy request. Add tests for x-fresh-metadata treatment. Closes-Bug: #1391826 Closes-Bug: #1600247 Co-Authored-By: Thiago da Silva <thiago@redhat.com> Change-Id: I917fb0b4e831c13e04ade1c5e0b9821802dec967
This commit is contained in:
Alistair Coles
parent
3c30fad957
commit
afe3968dd1
|
|
@ -455,10 +455,33 @@ class ServerSideCopyMiddleware(object):
|
|||
close_if_possible(source_resp.app_iter)
|
||||
return source_resp(source_resp.environ, start_response)
|
||||
|
||||
# Create a new Request object based on the original req instance.
|
||||
# This will preserve env and headers.
|
||||
sink_req = Request.blank(req.path_info,
|
||||
environ=req.environ, headers=req.headers)
|
||||
# Create a new Request object based on the original request instance.
|
||||
# This will preserve original request environ including headers.
|
||||
sink_req = Request.blank(req.path_info, environ=req.environ)
|
||||
|
||||
def is_object_sysmeta(k):
|
||||
return is_sys_meta('object', k)
|
||||
|
||||
if 'swift.post_as_copy' in sink_req.environ:
|
||||
# Post-as-copy: ignore new sysmeta, copy existing sysmeta
|
||||
remove_items(sink_req.headers, is_object_sysmeta)
|
||||
copy_header_subset(source_resp, sink_req, is_object_sysmeta)
|
||||
elif config_true_value(req.headers.get('x-fresh-metadata', 'false')):
|
||||
# x-fresh-metadata only applies to copy, not post-as-copy: ignore
|
||||
# existing user metadata, update existing sysmeta with new
|
||||
copy_header_subset(source_resp, sink_req, is_object_sysmeta)
|
||||
copy_header_subset(req, sink_req, is_object_sysmeta)
|
||||
else:
|
||||
# First copy existing sysmeta, user meta and other headers from the
|
||||
# source to the sink, apart from headers that are conditionally
|
||||
# copied below and timestamps.
|
||||
exclude_headers = ('x-static-large-object', 'x-object-manifest',
|
||||
'etag', 'content-type', 'x-timestamp',
|
||||
'x-backend-timestamp')
|
||||
copy_header_subset(source_resp, sink_req,
|
||||
lambda k: k.lower() not in exclude_headers)
|
||||
# now update with original req headers
|
||||
sink_req.headers.update(req.headers)
|
||||
|
||||
params = sink_req.params
|
||||
if params.get('multipart-manifest') == 'get':
|
||||
|
|
@ -489,32 +512,19 @@ class ServerSideCopyMiddleware(object):
|
|||
else:
|
||||
# since we're not copying the source etag, make sure that any
|
||||
# container update override values are not copied.
|
||||
remove_items(source_resp.headers, lambda k: k.startswith(
|
||||
remove_items(sink_req.headers, lambda k: k.startswith(
|
||||
'X-Object-Sysmeta-Container-Update-Override-'))
|
||||
|
||||
# We no longer need these headers
|
||||
sink_req.headers.pop('X-Copy-From', None)
|
||||
sink_req.headers.pop('X-Copy-From-Account', None)
|
||||
|
||||
# If the copy request does not explicitly override content-type,
|
||||
# use the one present in the source object.
|
||||
if not req.headers.get('content-type'):
|
||||
sink_req.headers['Content-Type'] = \
|
||||
source_resp.headers['Content-Type']
|
||||
|
||||
fresh_meta_flag = config_true_value(
|
||||
sink_req.headers.get('x-fresh-metadata', 'false'))
|
||||
|
||||
if fresh_meta_flag or 'swift.post_as_copy' in sink_req.environ:
|
||||
# Post-as-copy: ignore new sysmeta, copy existing sysmeta
|
||||
condition = lambda k: is_sys_meta('object', k)
|
||||
remove_items(sink_req.headers, condition)
|
||||
copy_header_subset(source_resp, sink_req, condition)
|
||||
else:
|
||||
# Copy/update existing sysmeta, transient-sysmeta and user meta
|
||||
_copy_headers(source_resp.headers, sink_req.headers)
|
||||
# Copy/update new metadata provided in request if any
|
||||
_copy_headers(req.headers, sink_req.headers)
|
||||
|
||||
# Create response headers for PUT response
|
||||
resp_headers = self._create_response_headers(source_path,
|
||||
source_resp, sink_req)
|
||||
|
|
|
|||
|
|
@ -1186,11 +1186,24 @@ class TestFile(Base):
|
|||
file_item = self.env.container.file(source_filename)
|
||||
|
||||
metadata = {}
|
||||
for i in range(1):
|
||||
metadata[Utils.create_ascii_name()] = Utils.create_name()
|
||||
metadata[Utils.create_ascii_name()] = Utils.create_name()
|
||||
put_headers = {'Content-Type': 'application/test',
|
||||
'Content-Encoding': 'gzip',
|
||||
'Content-Disposition': 'attachment; filename=myfile'}
|
||||
file_item.metadata = metadata
|
||||
data = file_item.write_random(hdrs=put_headers)
|
||||
|
||||
data = file_item.write_random()
|
||||
file_item.sync_metadata(metadata)
|
||||
# the allowed headers are configurable in object server, so we cannot
|
||||
# assert that content-encoding and content-disposition get *copied*
|
||||
# unless they were successfully set on the original PUT, so populate
|
||||
# expected_headers by making a HEAD on the original object
|
||||
file_item.initialize()
|
||||
self.assertEqual('application/test', file_item.content_type)
|
||||
resp_headers = dict(file_item.conn.response.getheaders())
|
||||
expected_headers = {}
|
||||
for k, v in put_headers.items():
|
||||
if k.lower() in resp_headers:
|
||||
expected_headers[k] = v
|
||||
|
||||
dest_cont = self.env.account.container(Utils.create_name())
|
||||
self.assertTrue(dest_cont.create())
|
||||
|
|
@ -1201,16 +1214,71 @@ class TestFile(Base):
|
|||
for prefix in ('', '/'):
|
||||
dest_filename = Utils.create_name()
|
||||
|
||||
file_item = self.env.container.file(source_filename)
|
||||
file_item.copy('%s%s' % (prefix, cont), dest_filename)
|
||||
extra_hdrs = {'X-Object-Meta-Extra': 'fresh'}
|
||||
self.assertTrue(file_item.copy(
|
||||
'%s%s' % (prefix, cont), dest_filename, hdrs=extra_hdrs))
|
||||
|
||||
self.assertIn(dest_filename, cont.files())
|
||||
|
||||
file_item = cont.file(dest_filename)
|
||||
file_copy = cont.file(dest_filename)
|
||||
|
||||
self.assertEqual(data, file_item.read())
|
||||
self.assertTrue(file_item.initialize())
|
||||
self.assertEqual(metadata, file_item.metadata)
|
||||
self.assertEqual(data, file_copy.read())
|
||||
self.assertTrue(file_copy.initialize())
|
||||
expected_metadata = dict(metadata)
|
||||
# new metadata should be merged with existing
|
||||
expected_metadata['extra'] = 'fresh'
|
||||
self.assertDictEqual(expected_metadata, file_copy.metadata)
|
||||
resp_headers = dict(file_copy.conn.response.getheaders())
|
||||
for k, v in expected_headers.items():
|
||||
self.assertIn(k.lower(), resp_headers)
|
||||
self.assertEqual(v, resp_headers[k.lower()])
|
||||
|
||||
# repeat copy with updated content-type, content-encoding and
|
||||
# content-disposition, which should get updated
|
||||
extra_hdrs = {
|
||||
'X-Object-Meta-Extra': 'fresher',
|
||||
'Content-Type': 'application/test-changed',
|
||||
'Content-Encoding': 'not_gzip',
|
||||
'Content-Disposition': 'attachment; filename=notmyfile'}
|
||||
self.assertTrue(file_item.copy(
|
||||
'%s%s' % (prefix, cont), dest_filename, hdrs=extra_hdrs))
|
||||
|
||||
self.assertIn(dest_filename, cont.files())
|
||||
|
||||
file_copy = cont.file(dest_filename)
|
||||
|
||||
self.assertEqual(data, file_copy.read())
|
||||
self.assertTrue(file_copy.initialize())
|
||||
expected_metadata['extra'] = 'fresher'
|
||||
self.assertDictEqual(expected_metadata, file_copy.metadata)
|
||||
resp_headers = dict(file_copy.conn.response.getheaders())
|
||||
# if k is in expected_headers then we can assert its new value
|
||||
for k, v in expected_headers.items():
|
||||
v = extra_hdrs.get(k, v)
|
||||
self.assertIn(k.lower(), resp_headers)
|
||||
self.assertEqual(v, resp_headers[k.lower()])
|
||||
|
||||
# repeat copy with X-Fresh-Metadata header - existing user
|
||||
# metadata should not be copied, new completely replaces it.
|
||||
extra_hdrs = {'Content-Type': 'application/test-updated',
|
||||
'X-Object-Meta-Extra': 'fresher',
|
||||
'X-Fresh-Metadata': 'true'}
|
||||
self.assertTrue(file_item.copy(
|
||||
'%s%s' % (prefix, cont), dest_filename, hdrs=extra_hdrs))
|
||||
|
||||
self.assertIn(dest_filename, cont.files())
|
||||
|
||||
file_copy = cont.file(dest_filename)
|
||||
|
||||
self.assertEqual(data, file_copy.read())
|
||||
self.assertTrue(file_copy.initialize())
|
||||
self.assertEqual('application/test-updated',
|
||||
file_copy.content_type)
|
||||
expected_metadata = {'extra': 'fresher'}
|
||||
self.assertDictEqual(expected_metadata, file_copy.metadata)
|
||||
resp_headers = dict(file_copy.conn.response.getheaders())
|
||||
for k in ('Content-Disposition', 'Content-Encoding'):
|
||||
self.assertNotIn(k.lower(), resp_headers)
|
||||
|
||||
def testCopyAccount(self):
|
||||
# makes sure to test encoded characters
|
||||
|
|
|
|||
|
|
@ -1198,6 +1198,218 @@ class TestServerSideCopyMiddleware(unittest.TestCase):
|
|||
self.assertEqual('OPTIONS', self.authorized[0].method)
|
||||
self.assertEqual('/v1/a/c/o', self.authorized[0].path)
|
||||
|
||||
def _test_COPY_source_headers(self, extra_put_headers):
|
||||
# helper method to perform a COPY with some metadata headers that
|
||||
# should always be sent to the destination
|
||||
put_headers = {'Destination': '/c1/o',
|
||||
'X-Object-Meta-Test2': 'added',
|
||||
'X-Object-Sysmeta-Test2': 'added',
|
||||
'X-Object-Transient-Sysmeta-Test2': 'added'}
|
||||
put_headers.update(extra_put_headers)
|
||||
get_resp_headers = {
|
||||
'X-Timestamp': '1234567890.12345',
|
||||
'X-Backend-Timestamp': '1234567890.12345',
|
||||
'Content-Type': 'text/original',
|
||||
'Content-Encoding': 'gzip',
|
||||
'Content-Disposition': 'attachment; filename=myfile',
|
||||
'X-Object-Meta-Test': 'original',
|
||||
'X-Object-Sysmeta-Test': 'original',
|
||||
'X-Object-Transient-Sysmeta-Test': 'original',
|
||||
'X-Foo': 'Bar'}
|
||||
self.app.register(
|
||||
'GET', '/v1/a/c/o', swob.HTTPOk, headers=get_resp_headers)
|
||||
self.app.register('PUT', '/v1/a/c1/o', swob.HTTPCreated, {})
|
||||
req = Request.blank('/v1/a/c/o', method='COPY', headers=put_headers)
|
||||
status, headers, body = self.call_ssc(req)
|
||||
self.assertEqual(status, '201 Created')
|
||||
calls = self.app.calls_with_headers
|
||||
self.assertEqual(2, len(calls))
|
||||
method, path, req_headers = calls[1]
|
||||
self.assertEqual('PUT', method)
|
||||
# these headers should always be applied to the destination
|
||||
self.assertEqual('added', req_headers.get('X-Object-Meta-Test2'))
|
||||
self.assertEqual('added', req_headers.get('X-Object-Sysmeta-Test2'))
|
||||
self.assertEqual('added',
|
||||
req_headers.get('X-Object-Transient-Sysmeta-Test2'))
|
||||
return req_headers
|
||||
|
||||
def test_COPY_source_headers_no_updates(self):
|
||||
# copy should preserve existing metadata if not updated
|
||||
req_headers = self._test_COPY_source_headers({})
|
||||
self.assertEqual('text/original', req_headers.get('Content-Type'))
|
||||
self.assertEqual('gzip', req_headers.get('Content-Encoding'))
|
||||
self.assertEqual('attachment; filename=myfile',
|
||||
req_headers.get('Content-Disposition'))
|
||||
self.assertEqual('original', req_headers.get('X-Object-Meta-Test'))
|
||||
self.assertEqual('original', req_headers.get('X-Object-Sysmeta-Test'))
|
||||
self.assertEqual('original',
|
||||
req_headers.get('X-Object-Transient-Sysmeta-Test'))
|
||||
self.assertEqual('Bar', req_headers.get('X-Foo'))
|
||||
self.assertNotIn('X-Timestamp', req_headers)
|
||||
self.assertNotIn('X-Backend-Timestamp', req_headers)
|
||||
|
||||
def test_COPY_source_headers_with_updates(self):
|
||||
# copy should apply any updated values to existing metadata
|
||||
put_headers = {
|
||||
'Content-Type': 'text/not_original',
|
||||
'Content-Encoding': 'not_gzip',
|
||||
'Content-Disposition': 'attachment; filename=notmyfile',
|
||||
'X-Object-Meta-Test': 'not_original',
|
||||
'X-Object-Sysmeta-Test': 'not_original',
|
||||
'X-Object-Transient-Sysmeta-Test': 'not_original',
|
||||
'X-Foo': 'Not Bar'}
|
||||
req_headers = self._test_COPY_source_headers(put_headers)
|
||||
self.assertEqual('text/not_original', req_headers.get('Content-Type'))
|
||||
self.assertEqual('not_gzip', req_headers.get('Content-Encoding'))
|
||||
self.assertEqual('attachment; filename=notmyfile',
|
||||
req_headers.get('Content-Disposition'))
|
||||
self.assertEqual('not_original', req_headers.get('X-Object-Meta-Test'))
|
||||
self.assertEqual('not_original',
|
||||
req_headers.get('X-Object-Sysmeta-Test'))
|
||||
self.assertEqual('not_original',
|
||||
req_headers.get('X-Object-Transient-Sysmeta-Test'))
|
||||
self.assertEqual('Not Bar', req_headers.get('X-Foo'))
|
||||
self.assertNotIn('X-Timestamp', req_headers)
|
||||
self.assertNotIn('X-Backend-Timestamp', req_headers)
|
||||
|
||||
def test_COPY_x_fresh_metadata_no_updates(self):
|
||||
# existing user metadata should not be copied, sysmeta is copied
|
||||
put_headers = {
|
||||
'X-Fresh-Metadata': 'true',
|
||||
'X-Extra': 'Fresh'}
|
||||
req_headers = self._test_COPY_source_headers(put_headers)
|
||||
self.assertEqual('text/original', req_headers.get('Content-Type'))
|
||||
self.assertEqual('Fresh', req_headers.get('X-Extra'))
|
||||
self.assertEqual('original',
|
||||
req_headers.get('X-Object-Sysmeta-Test'))
|
||||
self.assertIn('X-Fresh-Metadata', req_headers)
|
||||
self.assertNotIn('X-Object-Meta-Test', req_headers)
|
||||
self.assertNotIn('X-Object-Transient-Sysmeta-Test', req_headers)
|
||||
self.assertNotIn('X-Timestamp', req_headers)
|
||||
self.assertNotIn('X-Backend-Timestamp', req_headers)
|
||||
self.assertNotIn('Content-Encoding', req_headers)
|
||||
self.assertNotIn('Content-Disposition', req_headers)
|
||||
self.assertNotIn('X-Foo', req_headers)
|
||||
|
||||
def test_COPY_x_fresh_metadata_with_updates(self):
|
||||
# existing user metadata should not be copied, new metadata replaces it
|
||||
put_headers = {
|
||||
'X-Fresh-Metadata': 'true',
|
||||
'Content-Type': 'text/not_original',
|
||||
'Content-Encoding': 'not_gzip',
|
||||
'Content-Disposition': 'attachment; filename=notmyfile',
|
||||
'X-Object-Meta-Test': 'not_original',
|
||||
'X-Object-Sysmeta-Test': 'not_original',
|
||||
'X-Object-Transient-Sysmeta-Test': 'not_original',
|
||||
'X-Foo': 'Not Bar',
|
||||
'X-Extra': 'Fresh'}
|
||||
req_headers = self._test_COPY_source_headers(put_headers)
|
||||
self.assertEqual('Fresh', req_headers.get('X-Extra'))
|
||||
self.assertEqual('text/not_original', req_headers.get('Content-Type'))
|
||||
self.assertEqual('not_gzip', req_headers.get('Content-Encoding'))
|
||||
self.assertEqual('attachment; filename=notmyfile',
|
||||
req_headers.get('Content-Disposition'))
|
||||
self.assertEqual('not_original', req_headers.get('X-Object-Meta-Test'))
|
||||
self.assertEqual('not_original',
|
||||
req_headers.get('X-Object-Sysmeta-Test'))
|
||||
self.assertEqual('not_original',
|
||||
req_headers.get('X-Object-Transient-Sysmeta-Test'))
|
||||
self.assertEqual('Not Bar', req_headers.get('X-Foo'))
|
||||
|
||||
def _test_POST_source_headers(self, extra_post_headers):
|
||||
# helper method to perform a POST with metadata headers that should
|
||||
# always be sent to the destination
|
||||
post_headers = {'X-Object-Meta-Test2': 'added',
|
||||
'X-Object-Sysmeta-Test2': 'added',
|
||||
'X-Object-Transient-Sysmeta-Test2': 'added'}
|
||||
post_headers.update(extra_post_headers)
|
||||
get_resp_headers = {
|
||||
'X-Timestamp': '1234567890.12345',
|
||||
'X-Backend-Timestamp': '1234567890.12345',
|
||||
'Content-Type': 'text/original',
|
||||
'Content-Encoding': 'gzip',
|
||||
'Content-Disposition': 'attachment; filename=myfile',
|
||||
'X-Object-Meta-Test': 'original',
|
||||
'X-Object-Sysmeta-Test': 'original',
|
||||
'X-Object-Transient-Sysmeta-Test': 'original',
|
||||
'X-Foo': 'Bar'}
|
||||
self.app.register(
|
||||
'GET', '/v1/a/c/o', swob.HTTPOk, headers=get_resp_headers)
|
||||
self.app.register('PUT', '/v1/a/c/o', swob.HTTPCreated, {})
|
||||
req = Request.blank('/v1/a/c/o', method='POST', headers=post_headers)
|
||||
status, headers, body = self.call_ssc(req)
|
||||
self.assertEqual(status, '202 Accepted')
|
||||
calls = self.app.calls_with_headers
|
||||
self.assertEqual(2, len(calls))
|
||||
method, path, req_headers = calls[1]
|
||||
self.assertEqual('PUT', method)
|
||||
# these headers should always be applied to the destination
|
||||
self.assertEqual('added', req_headers.get('X-Object-Meta-Test2'))
|
||||
self.assertEqual('added',
|
||||
req_headers.get('X-Object-Transient-Sysmeta-Test2'))
|
||||
# POSTed sysmeta should never be applied to the destination
|
||||
self.assertNotIn('X-Object-Sysmeta-Test2', req_headers)
|
||||
# existing sysmeta should always be preserved
|
||||
self.assertEqual('original',
|
||||
req_headers.get('X-Object-Sysmeta-Test'))
|
||||
return req_headers
|
||||
|
||||
def test_POST_no_updates(self):
|
||||
post_headers = {}
|
||||
req_headers = self._test_POST_source_headers(post_headers)
|
||||
self.assertEqual('text/original', req_headers.get('Content-Type'))
|
||||
self.assertNotIn('X-Object-Meta-Test', req_headers)
|
||||
self.assertNotIn('X-Object-Transient-Sysmeta-Test', req_headers)
|
||||
self.assertNotIn('X-Timestamp', req_headers)
|
||||
self.assertNotIn('X-Backend-Timestamp', req_headers)
|
||||
self.assertNotIn('Content-Encoding', req_headers)
|
||||
self.assertNotIn('Content-Disposition', req_headers)
|
||||
self.assertNotIn('X-Foo', req_headers)
|
||||
|
||||
def test_POST_with_updates(self):
|
||||
post_headers = {
|
||||
'Content-Type': 'text/not_original',
|
||||
'Content-Encoding': 'not_gzip',
|
||||
'Content-Disposition': 'attachment; filename=notmyfile',
|
||||
'X-Object-Meta-Test': 'not_original',
|
||||
'X-Object-Sysmeta-Test': 'not_original',
|
||||
'X-Object-Transient-Sysmeta-Test': 'not_original',
|
||||
'X-Foo': 'Not Bar',
|
||||
}
|
||||
req_headers = self._test_POST_source_headers(post_headers)
|
||||
self.assertEqual('text/not_original', req_headers.get('Content-Type'))
|
||||
self.assertEqual('not_gzip', req_headers.get('Content-Encoding'))
|
||||
self.assertEqual('attachment; filename=notmyfile',
|
||||
req_headers.get('Content-Disposition'))
|
||||
self.assertEqual('not_original', req_headers.get('X-Object-Meta-Test'))
|
||||
self.assertEqual('not_original',
|
||||
req_headers.get('X-Object-Transient-Sysmeta-Test'))
|
||||
self.assertEqual('Not Bar', req_headers.get('X-Foo'))
|
||||
|
||||
def test_POST_x_fresh_metadata_with_updates(self):
|
||||
# post-as-copy trumps x-fresh-metadata i.e. existing user metadata
|
||||
# should not be copied, sysmeta is copied *and not updated with new*
|
||||
post_headers = {
|
||||
'X-Fresh-Metadata': 'true',
|
||||
'Content-Type': 'text/not_original',
|
||||
'Content-Encoding': 'not_gzip',
|
||||
'Content-Disposition': 'attachment; filename=notmyfile',
|
||||
'X-Object-Meta-Test': 'not_original',
|
||||
'X-Object-Sysmeta-Test': 'not_original',
|
||||
'X-Object-Transient-Sysmeta-Test': 'not_original',
|
||||
'X-Foo': 'Not Bar',
|
||||
}
|
||||
req_headers = self._test_POST_source_headers(post_headers)
|
||||
self.assertEqual('text/not_original', req_headers.get('Content-Type'))
|
||||
self.assertEqual('not_gzip', req_headers.get('Content-Encoding'))
|
||||
self.assertEqual('attachment; filename=notmyfile',
|
||||
req_headers.get('Content-Disposition'))
|
||||
self.assertEqual('not_original', req_headers.get('X-Object-Meta-Test'))
|
||||
self.assertEqual('not_original',
|
||||
req_headers.get('X-Object-Transient-Sysmeta-Test'))
|
||||
self.assertEqual('Not Bar', req_headers.get('X-Foo'))
|
||||
self.assertIn('X-Fresh-Metadata', req_headers)
|
||||
|
||||
|
||||
class TestServerSideCopyConfiguration(unittest.TestCase):
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue