diff --git a/swift/common/internal_client.py b/swift/common/internal_client.py index 04114ef59d..7908424939 100644 --- a/swift/common/internal_client.py +++ b/swift/common/internal_client.py @@ -789,13 +789,16 @@ class InternalClient(object): def upload_object( self, fobj, account, container, obj, headers=None, - acceptable_statuses=(2,)): + acceptable_statuses=(2,), params=None): """ :param fobj: File object to read object's content from. :param account: The object's account. :param container: The object's container. :param obj: The object. :param headers: Headers to send with request, defaults to empty dict. + :param acceptable_statuses: List of acceptable statuses for request. + :param params: A dict of params to be set in request query string, + defaults to None. :raises UnexpectedResponse: Exception raised when requests fail to get a response with an acceptable status @@ -807,7 +810,8 @@ class InternalClient(object): if 'Content-Length' not in headers: headers['Transfer-Encoding'] = 'chunked' path = self.make_path(account, container, obj) - self.make_request('PUT', path, headers, acceptable_statuses, fobj) + self.make_request('PUT', path, headers, acceptable_statuses, fobj, + params=params) def get_auth(url, user, key, auth_version='1.0', **kwargs): diff --git a/swift/common/request_helpers.py b/swift/common/request_helpers.py index bdc234c9ea..c8399f0936 100644 --- a/swift/common/request_helpers.py +++ b/swift/common/request_helpers.py @@ -29,6 +29,7 @@ import six from swift.common.header_key_dict import HeaderKeyDict from swift import gettext_ as _ +from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX from swift.common.storage_policy import POLICIES from swift.common.exceptions import ListingIterError, SegmentError from swift.common.http import is_success @@ -41,6 +42,7 @@ from swift.common.utils import split_path, validate_device_partition, \ parse_content_range, csv_append, list_from_csv, Spliterator, quote, \ RESERVED from swift.common.wsgi import make_subrequest +from swift.container.reconciler import MISPLACED_OBJECTS_ACCOUNT OBJECT_TRANSIENT_SYSMETA_PREFIX = 'x-object-transient-sysmeta-' @@ -121,7 +123,8 @@ def validate_internal_obj(account, container, obj): if not container: raise ValueError('Container is required') validate_internal_container(account, container) - if obj: + if obj and not (account.startswith(AUTO_CREATE_ACCOUNT_PREFIX) or + account == MISPLACED_OBJECTS_ACCOUNT): _validate_internal_name(obj, 'object') if container.startswith(RESERVED) and not obj.startswith(RESERVED): raise HTTPBadRequest(body='Invalid user-namespace object ' diff --git a/test/probe/brain.py b/test/probe/brain.py index 106749e182..6b94a3122b 100644 --- a/test/probe/brain.py +++ b/test/probe/brain.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from __future__ import print_function +import functools import sys import itertools import uuid @@ -19,10 +20,12 @@ from optparse import OptionParser import random import six -from six.moves.urllib.parse import urlparse +from six import StringIO +from six.moves.urllib.parse import urlparse, parse_qs, quote from swift.common.manager import Manager from swift.common import utils, ring +from swift.common.internal_client import InternalClient, UnexpectedResponse from swift.common.storage_policy import POLICIES from swift.common.http import HTTP_NOT_FOUND @@ -64,13 +67,10 @@ def command(f): return f -@six.add_metaclass(meta_command) -class BrainSplitter(object): - def __init__(self, url, token, container_name='test', object_name='test', - server_type='container', policy=None): - self.url = url - self.token = token - self.account = utils.split_path(urlparse(url).path, 2, 2)[1] +class BaseBrain(object): + def _setup(self, account, container_name, object_name, + server_type, policy): + self.account = account self.container_name = container_name self.object_name = object_name server_list = ['%s-server' % server_type] if server_type else ['all'] @@ -153,36 +153,168 @@ class BrainSplitter(object): policy = self.policy headers = {'X-Storage-Policy': policy.name} - client.put_container(self.url, self.token, self.container_name, - headers=headers) + self.client.put_container(self.container_name, headers=headers) @command def delete_container(self): """ delete container """ - client.delete_container(self.url, self.token, self.container_name) + self.client.delete_container(self.container_name) @command def put_object(self, headers=None, contents=None): """ issue put for test object """ - client.put_object(self.url, self.token, self.container_name, - self.object_name, headers=headers, contents=contents) + self.client.put_object(self.container_name, self.object_name, + headers=headers, contents=contents) @command def delete_object(self): """ issue delete for test object """ + self.client.delete_object(self.container_name, self.object_name) + + @command + def get_object(self): + """ + issue GET for test object + """ + return self.client.get_object(self.container_name, self.object_name) + + +class PublicBrainClient(object): + def __init__(self, url, token): + self.url = url + self.token = token + self.account = utils.split_path(urlparse(url).path, 2, 2)[1] + + def put_container(self, container_name, headers): + return client.put_container(self.url, self.token, container_name, + headers=headers) + + def post_container(self, container_name, headers): + return client.post_container(self.url, self.token, container_name, + headers) + + def delete_container(self, container_name): + return client.delete_container(self.url, self.token, container_name) + + def put_object(self, container_name, object_name, headers, contents, + query_string=None): + return client.put_object(self.url, self.token, container_name, + object_name, headers=headers, + contents=contents, query_string=query_string) + + def delete_object(self, container_name, object_name): try: - client.delete_object(self.url, self.token, self.container_name, - self.object_name) + client.delete_object(self.url, self.token, + container_name, object_name) except ClientException as err: if err.http_status != HTTP_NOT_FOUND: raise + def head_object(self, container_name, object_name): + return client.head_object(self.url, self.token, container_name, + object_name) + + def get_object(self, container_name, object_name, query_string=None): + return client.get_object(self.url, self.token, + container_name, object_name, + query_string=query_string) + + +def translate_client_exception(m): + @functools.wraps(m) + def wrapper(*args, **kwargs): + try: + return m(*args, **kwargs) + except UnexpectedResponse as err: + raise ClientException( + err.message, + http_scheme=err.resp.environ['wsgi.url_scheme'], + http_host=err.resp.environ['SERVER_NAME'], + http_port=err.resp.environ['SERVER_PORT'], + http_path=quote(err.resp.environ['PATH_INFO']), + http_query=err.resp.environ['QUERY_STRING'], + http_status=err.resp.status_int, + http_reason=err.resp.explanation, + http_response_content=err.resp.body, + http_response_headers=err.resp.headers, + ) + return wrapper + + +class InternalBrainClient(object): + + def __init__(self, conf_file, account='AUTH_test'): + self.swift = InternalClient(conf_file, 'probe-test', 3) + self.account = account + + @translate_client_exception + def put_container(self, container_name, headers): + return self.swift.create_container(self.account, container_name, + headers=headers) + + @translate_client_exception + def post_container(self, container_name, headers): + return self.swift.set_container_metadata(self.account, container_name, + headers) + + @translate_client_exception + def delete_container(self, container_name): + return self.swift.delete_container(self.account, container_name) + + def parse_qs(self, query_string): + if query_string is not None: + return {k: v[-1] for k, v in parse_qs(query_string).items()} + + @translate_client_exception + def put_object(self, container_name, object_name, headers, contents, + query_string=None): + return self.swift.upload_object(StringIO(contents), self.account, + container_name, object_name, + headers=headers, + params=self.parse_qs(query_string)) + + @translate_client_exception + def delete_object(self, container_name, object_name): + return self.swift.delete_object( + self.account, container_name, object_name) + + @translate_client_exception + def head_object(self, container_name, object_name): + return self.swift.get_object_metadata( + self.account, container_name, object_name) + + @translate_client_exception + def get_object(self, container_name, object_name, query_string=None): + status, headers, resp_iter = self.swift.get_object( + self.account, container_name, object_name, + params=self.parse_qs(query_string)) + return headers, ''.join(resp_iter) + + +@six.add_metaclass(meta_command) +class BrainSplitter(BaseBrain): + def __init__(self, url, token, container_name='test', object_name='test', + server_type='container', policy=None): + self.client = PublicBrainClient(url, token) + self._setup(self.client.account, container_name, object_name, + server_type, policy) + + +@six.add_metaclass(meta_command) +class InternalBrainSplitter(BaseBrain): + def __init__(self, conf, container_name='test', object_name='test', + server_type='container', policy=None): + self.client = InternalBrainClient(conf) + self._setup(self.client.account, container_name, object_name, + server_type, policy) + + parser = OptionParser('%prog [options] ' '[:[,...]] [...]') parser.usage += '\n\nCommands:\n\t' + \ diff --git a/test/probe/test_container_merge_policy_index.py b/test/probe/test_container_merge_policy_index.py index 4b62804f13..f4a499d09d 100644 --- a/test/probe/test_container_merge_policy_index.py +++ b/test/probe/test_container_merge_policy_index.py @@ -24,11 +24,12 @@ from swift.common import utils, direct_client from swift.common.storage_policy import POLICIES from swift.common.http import HTTP_NOT_FOUND from swift.container.reconciler import MISPLACED_OBJECTS_ACCOUNT -from test.probe.brain import BrainSplitter +from test.probe.brain import BrainSplitter, InternalBrainSplitter +from swift.common.request_helpers import get_reserved_name from test.probe.common import (ReplProbeTest, ENABLED_POLICIES, POLICIES_BY_TYPE, REPL_POLICY) -from swiftclient import client, ClientException +from swiftclient import ClientException TIMEOUT = 60 @@ -48,15 +49,13 @@ class TestContainerMergePolicyIndex(ReplProbeTest): timeout = time.time() + TIMEOUT while time.time() < timeout: try: - return client.get_object(self.url, self.token, - self.container_name, - self.object_name) + return self.brain.get_object() except ClientException as err: if err.http_status != HTTP_NOT_FOUND: raise time.sleep(1) else: - self.fail('could not HEAD /%s/%s/%s/ from policy %s ' + self.fail('could not GET /%s/%s/%s/ from policy %s ' 'after %s seconds.' % ( self.account, self.container_name, self.object_name, int(policy_index), TIMEOUT)) @@ -238,6 +237,12 @@ class TestContainerMergePolicyIndex(ReplProbeTest): self.account, self.container_name, self.object_name, orig_policy_index, node)) + def get_object_name(self, name): + """ + hook for sublcass to translate object names + """ + return name + def test_reconcile_manifest(self): if 'slo' not in self.cluster_info: raise unittest.SkipTest( @@ -258,14 +263,14 @@ class TestContainerMergePolicyIndex(ReplProbeTest): def write_part(i): body = b'VERIFY%0.2d' % i + b'\x00' * 1048576 - part_name = 'manifest_part_%0.2d' % i + part_name = self.get_object_name('manifest_part_%0.2d' % i) manifest_entry = { "path": "/%s/%s" % (self.container_name, part_name), "etag": md5(body).hexdigest(), "size_bytes": len(body), } - client.put_object(self.url, self.token, self.container_name, - part_name, contents=body) + self.brain.client.put_object(self.container_name, part_name, {}, + body) manifest_data.append(manifest_entry) # get an old container stashed @@ -284,10 +289,10 @@ class TestContainerMergePolicyIndex(ReplProbeTest): # write manifest with self.assertRaises(ClientException) as catcher: - client.put_object(self.url, self.token, self.container_name, - self.object_name, - contents=utils.json.dumps(manifest_data), - query_string='multipart-manifest=put') + self.brain.client.put_object( + self.container_name, self.object_name, + {}, utils.json.dumps(manifest_data), + query_string='multipart-manifest=put') # so as it works out, you can't really upload a multi-part # manifest for objects that are currently misplaced - you have to @@ -334,18 +339,18 @@ class TestContainerMergePolicyIndex(ReplProbeTest): self.get_to_final_state() Manager(['container-reconciler']).once() # clear proxy cache - client.post_container(self.url, self.token, self.container_name, {}) + self.brain.client.post_container(self.container_name, {}) # let's see how that direct upload worked out... - metadata, body = client.get_object( - self.url, self.token, self.container_name, direct_manifest_name, + metadata, body = self.brain.client.get_object( + self.container_name, direct_manifest_name, query_string='multipart-manifest=get') self.assertEqual(metadata['x-static-large-object'].lower(), 'true') for i, entry in enumerate(utils.json.loads(body)): for key in ('hash', 'bytes', 'name'): self.assertEqual(entry[key], direct_manifest_data[i][key]) - metadata, body = client.get_object( - self.url, self.token, self.container_name, direct_manifest_name) + metadata, body = self.brain.client.get_object( + self.container_name, direct_manifest_name) self.assertEqual(metadata['x-static-large-object'].lower(), 'true') self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data)) @@ -353,13 +358,11 @@ class TestContainerMergePolicyIndex(ReplProbeTest): for i in range(20))) # and regular upload should work now too - client.put_object(self.url, self.token, self.container_name, - self.object_name, - contents=utils.json.dumps(manifest_data), - query_string='multipart-manifest=put') - metadata = client.head_object(self.url, self.token, - self.container_name, - self.object_name) + self.brain.client.put_object(self.container_name, self.object_name, {}, + utils.json.dumps(manifest_data), + query_string='multipart-manifest=put') + metadata = self.brain.client.head_object(self.container_name, + self.object_name) self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data)) @@ -376,66 +379,69 @@ class TestContainerMergePolicyIndex(ReplProbeTest): self.brain.put_container(int(policy)) self.brain.start_primary_half() # write some target data - client.put_object(self.url, self.token, self.container_name, 'target', - contents=b'this is the target data') + target_name = self.get_object_name('target') + self.brain.client.put_object(self.container_name, target_name, {}, + b'this is the target data') # write the symlink self.brain.stop_handoff_half() self.brain.put_container(int(wrong_policy)) - client.put_object( - self.url, self.token, self.container_name, 'symlink', - headers={ - 'X-Symlink-Target': '%s/target' % self.container_name, + symlink_name = self.get_object_name('symlink') + self.brain.client.put_object( + self.container_name, symlink_name, { + 'X-Symlink-Target': '%s/%s' % ( + self.container_name, target_name), 'Content-Type': 'application/symlink', - }) + }, '') # at this point we have a broken symlink (the container_info has the # proxy looking for the target in the wrong policy) with self.assertRaises(ClientException) as ctx: - client.get_object(self.url, self.token, self.container_name, - 'symlink') + self.brain.client.get_object(self.container_name, symlink_name) self.assertEqual(ctx.exception.http_status, 404) # of course the symlink itself is fine - metadata, body = client.get_object(self.url, self.token, - self.container_name, 'symlink', - query_string='symlink=get') + metadata, body = self.brain.client.get_object( + self.container_name, symlink_name, query_string='symlink=get') self.assertEqual(metadata['x-symlink-target'], - '%s/target' % self.container_name) + utils.quote('%s/%s' % ( + self.container_name, target_name))) self.assertEqual(metadata['content-type'], 'application/symlink') self.assertEqual(body, b'') # ... although in the wrong policy object_ring = POLICIES.get_object_ring(int(wrong_policy), '/etc/swift') part, nodes = object_ring.get_nodes( - self.account, self.container_name, 'symlink') + self.account, self.container_name, symlink_name) for node in nodes: metadata = direct_client.direct_head_object( - node, part, self.account, self.container_name, 'symlink', + node, part, self.account, self.container_name, symlink_name, headers={'X-Backend-Storage-Policy-Index': int(wrong_policy)}) self.assertEqual(metadata['X-Object-Sysmeta-Symlink-Target'], - '%s/target' % self.container_name) + utils.quote('%s/%s' % ( + self.container_name, target_name))) # let the reconciler run self.brain.start_handoff_half() self.get_to_final_state() Manager(['container-reconciler']).once() # clear proxy cache - client.post_container(self.url, self.token, self.container_name, {}) + self.brain.client.post_container(self.container_name, {}) # now the symlink works - metadata, body = client.get_object(self.url, self.token, - self.container_name, 'symlink') + metadata, body = self.brain.client.get_object( + self.container_name, symlink_name) self.assertEqual(body, b'this is the target data') # and it's in the correct policy object_ring = POLICIES.get_object_ring(int(policy), '/etc/swift') part, nodes = object_ring.get_nodes( - self.account, self.container_name, 'symlink') + self.account, self.container_name, symlink_name) for node in nodes: metadata = direct_client.direct_head_object( - node, part, self.account, self.container_name, 'symlink', + node, part, self.account, self.container_name, symlink_name, headers={'X-Backend-Storage-Policy-Index': int(policy)}) self.assertEqual(metadata['X-Object-Sysmeta-Symlink-Target'], - '%s/target' % self.container_name) + utils.quote('%s/%s' % ( + self.container_name, target_name))) def test_reconciler_move_object_twice(self): # select some policies @@ -552,5 +558,24 @@ class TestContainerMergePolicyIndex(ReplProbeTest): self.assertEqual('custom-meta', headers['x-object-meta-test']) +class TestReservedNamespaceMergePolicyIndex(TestContainerMergePolicyIndex): + + @unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy") + def setUp(self): + super(TestReservedNamespaceMergePolicyIndex, self).setUp() + self.container_name = get_reserved_name('container', str(uuid.uuid4())) + self.object_name = get_reserved_name('object', str(uuid.uuid4())) + self.brain = InternalBrainSplitter('/etc/swift/internal-client.conf', + self.container_name, + self.object_name, 'container') + + def get_object_name(self, name): + return get_reserved_name(name) + + def test_reconcile_manifest(self): + raise unittest.SkipTest( + 'SLO does not allow parts in the reserved namespace') + + if __name__ == "__main__": unittest.main() diff --git a/test/unit/common/test_internal_client.py b/test/unit/common/test_internal_client.py index 99ee0b319a..597e30e3d9 100644 --- a/test/unit/common/test_internal_client.py +++ b/test/unit/common/test_internal_client.py @@ -1330,7 +1330,7 @@ class TestInternalClient(unittest.TestCase): def make_request( self, method, path, headers, acceptable_statuses, - body_file=None): + body_file=None, params=None): self.make_request_called += 1 self.test.assertEqual(self.path, path) exp_headers = dict(self.headers) @@ -1358,7 +1358,7 @@ class TestInternalClient(unittest.TestCase): def make_request( self, method, path, headers, acceptable_statuses, - body_file=None): + body_file=None, params=None): self.make_request_called += 1 self.test.assertEqual(self.path, path) exp_headers = dict(self.headers) diff --git a/test/unit/common/test_request_helpers.py b/test/unit/common/test_request_helpers.py index bbdc3be7ff..86ff500df3 100644 --- a/test/unit/common/test_request_helpers.py +++ b/test/unit/common/test_request_helpers.py @@ -19,6 +19,7 @@ import unittest from swift.common.swob import Request, HTTPException, HeaderKeyDict from swift.common.storage_policy import POLICIES, EC_POLICY, REPL_POLICY from swift.common import request_helpers as rh +from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX from test.unit import patch_policies from test.unit.common.test_utils import FakeResponse @@ -280,6 +281,11 @@ class TestRequestHelpers(unittest.TestCase): 'AUTH_foo', cont, 'baz') self.assertEqual(raised.exception.args[0], 'Container is required') + def test_invalid_names_in_system_accounts(self): + self.assertIsNone(rh.validate_internal_obj( + AUTO_CREATE_ACCOUNT_PREFIX + 'system_account', 'foo', + 'crazy%stown' % rh.RESERVED)) + def test_invalid_reserved_names(self): with self.assertRaises(HTTPException) as raised: rh.validate_internal_obj('AUTH_foo' + rh.RESERVED, 'bar', 'baz')