Merge "Allow reconciler to handle reserved names"

swift/common/internal_client.py

@@ -789,13 +789,16 @@ class InternalClient(object):
 
     def upload_object(
             self, fobj, account, container, obj, headers=None,
-            acceptable_statuses=(2,)):
+            acceptable_statuses=(2,), params=None):
         """
         :param fobj: File object to read object's content from.
         :param account: The object's account.
         :param container: The object's container.
         :param obj: The object.
         :param headers: Headers to send with request, defaults to empty dict.
+        :param acceptable_statuses: List of acceptable statuses for request.
+        :param params: A dict of params to be set in request query string,
+                       defaults to None.
 
         :raises UnexpectedResponse: Exception raised when requests fail
                                     to get a response with an acceptable status
@@ -807,7 +810,8 @@ class InternalClient(object):
         if 'Content-Length' not in headers:
             headers['Transfer-Encoding'] = 'chunked'
         path = self.make_path(account, container, obj)
-        self.make_request('PUT', path, headers, acceptable_statuses, fobj)
+        self.make_request('PUT', path, headers, acceptable_statuses, fobj,
+                          params=params)
 
 
 def get_auth(url, user, key, auth_version='1.0', **kwargs):

swift/common/request_helpers.py

@@ -29,6 +29,7 @@ import six
 from swift.common.header_key_dict import HeaderKeyDict
 
 from swift import gettext_ as _
+from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX
 from swift.common.storage_policy import POLICIES
 from swift.common.exceptions import ListingIterError, SegmentError
 from swift.common.http import is_success
@@ -41,6 +42,7 @@ from swift.common.utils import split_path, validate_device_partition, \
     parse_content_range, csv_append, list_from_csv, Spliterator, quote, \
     RESERVED
 from swift.common.wsgi import make_subrequest
+from swift.container.reconciler import MISPLACED_OBJECTS_ACCOUNT
 
 
 OBJECT_TRANSIENT_SYSMETA_PREFIX = 'x-object-transient-sysmeta-'
@@ -121,7 +123,8 @@ def validate_internal_obj(account, container, obj):
     if not container:
         raise ValueError('Container is required')
     validate_internal_container(account, container)
-    if obj:
+    if obj and not (account.startswith(AUTO_CREATE_ACCOUNT_PREFIX) or
+                    account == MISPLACED_OBJECTS_ACCOUNT):
         _validate_internal_name(obj, 'object')
         if container.startswith(RESERVED) and not obj.startswith(RESERVED):
             raise HTTPBadRequest(body='Invalid user-namespace object '

test/probe/brain.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from __future__ import print_function
+import functools
 import sys
 import itertools
 import uuid
@@ -19,10 +20,12 @@ from optparse import OptionParser
 import random
 
 import six
-from six.moves.urllib.parse import urlparse
+from six import StringIO
+from six.moves.urllib.parse import urlparse, parse_qs, quote
 
 from swift.common.manager import Manager
 from swift.common import utils, ring
+from swift.common.internal_client import InternalClient, UnexpectedResponse
 from swift.common.storage_policy import POLICIES
 from swift.common.http import HTTP_NOT_FOUND
 
@@ -64,13 +67,10 @@ def command(f):
     return f
 
 
-@six.add_metaclass(meta_command)
-class BrainSplitter(object):
-    def __init__(self, url, token, container_name='test', object_name='test',
-                 server_type='container', policy=None):
-        self.url = url
-        self.token = token
-        self.account = utils.split_path(urlparse(url).path, 2, 2)[1]
+class BaseBrain(object):
+    def _setup(self, account, container_name, object_name,
+               server_type, policy):
+        self.account = account
         self.container_name = container_name
         self.object_name = object_name
         server_list = ['%s-server' % server_type] if server_type else ['all']
@@ -153,36 +153,168 @@ class BrainSplitter(object):
             policy = self.policy
 
         headers = {'X-Storage-Policy': policy.name}
-        client.put_container(self.url, self.token, self.container_name,
-                             headers=headers)
+        self.client.put_container(self.container_name, headers=headers)
 
     @command
     def delete_container(self):
         """
         delete container
         """
-        client.delete_container(self.url, self.token, self.container_name)
+        self.client.delete_container(self.container_name)
 
     @command
     def put_object(self, headers=None, contents=None):
         """
         issue put for test object
         """
-        client.put_object(self.url, self.token, self.container_name,
-                          self.object_name, headers=headers, contents=contents)
+        self.client.put_object(self.container_name, self.object_name,
+                               headers=headers, contents=contents)
 
     @command
     def delete_object(self):
         """
         issue delete for test object
         """
+        self.client.delete_object(self.container_name, self.object_name)
+
+    @command
+    def get_object(self):
+        """
+        issue GET for test object
+        """
+        return self.client.get_object(self.container_name, self.object_name)
+
+
+class PublicBrainClient(object):
+    def __init__(self, url, token):
+        self.url = url
+        self.token = token
+        self.account = utils.split_path(urlparse(url).path, 2, 2)[1]
+
+    def put_container(self, container_name, headers):
+        return client.put_container(self.url, self.token, container_name,
+                                    headers=headers)
+
+    def post_container(self, container_name, headers):
+        return client.post_container(self.url, self.token, container_name,
+                                     headers)
+
+    def delete_container(self, container_name):
+        return client.delete_container(self.url, self.token, container_name)
+
+    def put_object(self, container_name, object_name, headers, contents,
+                   query_string=None):
+        return client.put_object(self.url, self.token, container_name,
+                                 object_name, headers=headers,
+                                 contents=contents, query_string=query_string)
+
+    def delete_object(self, container_name, object_name):
         try:
-            client.delete_object(self.url, self.token, self.container_name,
-                                 self.object_name)
+            client.delete_object(self.url, self.token,
+                                 container_name, object_name)
         except ClientException as err:
             if err.http_status != HTTP_NOT_FOUND:
                 raise
 
+    def head_object(self, container_name, object_name):
+        return client.head_object(self.url, self.token, container_name,
+                                  object_name)
+
+    def get_object(self, container_name, object_name, query_string=None):
+        return client.get_object(self.url, self.token,
+                                 container_name, object_name,
+                                 query_string=query_string)
+
+
+def translate_client_exception(m):
+    @functools.wraps(m)
+    def wrapper(*args, **kwargs):
+        try:
+            return m(*args, **kwargs)
+        except UnexpectedResponse as err:
+            raise ClientException(
+                err.message,
+                http_scheme=err.resp.environ['wsgi.url_scheme'],
+                http_host=err.resp.environ['SERVER_NAME'],
+                http_port=err.resp.environ['SERVER_PORT'],
+                http_path=quote(err.resp.environ['PATH_INFO']),
+                http_query=err.resp.environ['QUERY_STRING'],
+                http_status=err.resp.status_int,
+                http_reason=err.resp.explanation,
+                http_response_content=err.resp.body,
+                http_response_headers=err.resp.headers,
+            )
+    return wrapper
+
+
+class InternalBrainClient(object):
+
+    def __init__(self, conf_file, account='AUTH_test'):
+        self.swift = InternalClient(conf_file, 'probe-test', 3)
+        self.account = account
+
+    @translate_client_exception
+    def put_container(self, container_name, headers):
+        return self.swift.create_container(self.account, container_name,
+                                           headers=headers)
+
+    @translate_client_exception
+    def post_container(self, container_name, headers):
+        return self.swift.set_container_metadata(self.account, container_name,
+                                                 headers)
+
+    @translate_client_exception
+    def delete_container(self, container_name):
+        return self.swift.delete_container(self.account, container_name)
+
+    def parse_qs(self, query_string):
+        if query_string is not None:
+            return {k: v[-1] for k, v in parse_qs(query_string).items()}
+
+    @translate_client_exception
+    def put_object(self, container_name, object_name, headers, contents,
+                   query_string=None):
+        return self.swift.upload_object(StringIO(contents), self.account,
+                                        container_name, object_name,
+                                        headers=headers,
+                                        params=self.parse_qs(query_string))
+
+    @translate_client_exception
+    def delete_object(self, container_name, object_name):
+        return self.swift.delete_object(
+            self.account, container_name, object_name)
+
+    @translate_client_exception
+    def head_object(self, container_name, object_name):
+        return self.swift.get_object_metadata(
+            self.account, container_name, object_name)
+
+    @translate_client_exception
+    def get_object(self, container_name, object_name, query_string=None):
+        status, headers, resp_iter = self.swift.get_object(
+            self.account, container_name, object_name,
+            params=self.parse_qs(query_string))
+        return headers, ''.join(resp_iter)
+
+
+@six.add_metaclass(meta_command)
+class BrainSplitter(BaseBrain):
+    def __init__(self, url, token, container_name='test', object_name='test',
+                 server_type='container', policy=None):
+        self.client = PublicBrainClient(url, token)
+        self._setup(self.client.account, container_name, object_name,
+                    server_type, policy)
+
+
+@six.add_metaclass(meta_command)
+class InternalBrainSplitter(BaseBrain):
+    def __init__(self, conf, container_name='test', object_name='test',
+                 server_type='container', policy=None):
+        self.client = InternalBrainClient(conf)
+        self._setup(self.client.account, container_name, object_name,
+                    server_type, policy)
+
+
 parser = OptionParser('%prog [options] '
                       '<command>[:<args>[,<args>...]] [<command>...]')
 parser.usage += '\n\nCommands:\n\t' + \

test/probe/test_container_merge_policy_index.py

@@ -24,11 +24,12 @@ from swift.common import utils, direct_client
 from swift.common.storage_policy import POLICIES
 from swift.common.http import HTTP_NOT_FOUND
 from swift.container.reconciler import MISPLACED_OBJECTS_ACCOUNT
-from test.probe.brain import BrainSplitter
+from test.probe.brain import BrainSplitter, InternalBrainSplitter
+from swift.common.request_helpers import get_reserved_name
 from test.probe.common import (ReplProbeTest, ENABLED_POLICIES,
                                POLICIES_BY_TYPE, REPL_POLICY)
 
-from swiftclient import client, ClientException
+from swiftclient import ClientException
 
 TIMEOUT = 60
 
@@ -48,15 +49,13 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
         timeout = time.time() + TIMEOUT
         while time.time() < timeout:
             try:
-                return client.get_object(self.url, self.token,
-                                         self.container_name,
-                                         self.object_name)
+                return self.brain.get_object()
             except ClientException as err:
                 if err.http_status != HTTP_NOT_FOUND:
                     raise
                 time.sleep(1)
         else:
-            self.fail('could not HEAD /%s/%s/%s/ from policy %s '
+            self.fail('could not GET /%s/%s/%s/ from policy %s '
                       'after %s seconds.' % (
                           self.account, self.container_name, self.object_name,
                           int(policy_index), TIMEOUT))
@@ -238,6 +237,12 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
                         self.account, self.container_name, self.object_name,
                         orig_policy_index, node))
 
+    def get_object_name(self, name):
+        """
+        hook for sublcass to translate object names
+        """
+        return name
+
     def test_reconcile_manifest(self):
         if 'slo' not in self.cluster_info:
             raise unittest.SkipTest(
@@ -258,14 +263,14 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
 
         def write_part(i):
             body = b'VERIFY%0.2d' % i + b'\x00' * 1048576
-            part_name = 'manifest_part_%0.2d' % i
+            part_name = self.get_object_name('manifest_part_%0.2d' % i)
             manifest_entry = {
                 "path": "/%s/%s" % (self.container_name, part_name),
                 "etag": md5(body).hexdigest(),
                 "size_bytes": len(body),
             }
-            client.put_object(self.url, self.token, self.container_name,
-                              part_name, contents=body)
+            self.brain.client.put_object(self.container_name, part_name, {},
+                                         body)
             manifest_data.append(manifest_entry)
 
         # get an old container stashed
@@ -284,10 +289,10 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
 
         # write manifest
         with self.assertRaises(ClientException) as catcher:
-            client.put_object(self.url, self.token, self.container_name,
-                              self.object_name,
-                              contents=utils.json.dumps(manifest_data),
-                              query_string='multipart-manifest=put')
+            self.brain.client.put_object(
+                self.container_name, self.object_name,
+                {}, utils.json.dumps(manifest_data),
+                query_string='multipart-manifest=put')
 
         # so as it works out, you can't really upload a multi-part
         # manifest for objects that are currently misplaced - you have to
@@ -334,18 +339,18 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
         self.get_to_final_state()
         Manager(['container-reconciler']).once()
         # clear proxy cache
-        client.post_container(self.url, self.token, self.container_name, {})
+        self.brain.client.post_container(self.container_name, {})
 
         # let's see how that direct upload worked out...
-        metadata, body = client.get_object(
-            self.url, self.token, self.container_name, direct_manifest_name,
+        metadata, body = self.brain.client.get_object(
+            self.container_name, direct_manifest_name,
             query_string='multipart-manifest=get')
         self.assertEqual(metadata['x-static-large-object'].lower(), 'true')
         for i, entry in enumerate(utils.json.loads(body)):
             for key in ('hash', 'bytes', 'name'):
                 self.assertEqual(entry[key], direct_manifest_data[i][key])
-        metadata, body = client.get_object(
-            self.url, self.token, self.container_name, direct_manifest_name)
+        metadata, body = self.brain.client.get_object(
+            self.container_name, direct_manifest_name)
         self.assertEqual(metadata['x-static-large-object'].lower(), 'true')
         self.assertEqual(int(metadata['content-length']),
                          sum(part['size_bytes'] for part in manifest_data))
@@ -353,13 +358,11 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
                                         for i in range(20)))
 
         # and regular upload should work now too
-        client.put_object(self.url, self.token, self.container_name,
-                          self.object_name,
-                          contents=utils.json.dumps(manifest_data),
-                          query_string='multipart-manifest=put')
-        metadata = client.head_object(self.url, self.token,
-                                      self.container_name,
-                                      self.object_name)
+        self.brain.client.put_object(self.container_name, self.object_name, {},
+                                     utils.json.dumps(manifest_data),
+                                     query_string='multipart-manifest=put')
+        metadata = self.brain.client.head_object(self.container_name,
+                                                 self.object_name)
         self.assertEqual(int(metadata['content-length']),
                          sum(part['size_bytes'] for part in manifest_data))
 
@@ -376,66 +379,69 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
         self.brain.put_container(int(policy))
         self.brain.start_primary_half()
         # write some target data
-        client.put_object(self.url, self.token, self.container_name, 'target',
-                          contents=b'this is the target data')
+        target_name = self.get_object_name('target')
+        self.brain.client.put_object(self.container_name, target_name, {},
+                                     b'this is the target data')
 
         # write the symlink
         self.brain.stop_handoff_half()
         self.brain.put_container(int(wrong_policy))
-        client.put_object(
-            self.url, self.token, self.container_name, 'symlink',
-            headers={
-                'X-Symlink-Target': '%s/target' % self.container_name,
+        symlink_name = self.get_object_name('symlink')
+        self.brain.client.put_object(
+            self.container_name, symlink_name, {
+                'X-Symlink-Target': '%s/%s' % (
+                    self.container_name, target_name),
                 'Content-Type': 'application/symlink',
-            })
+            }, '')
 
         # at this point we have a broken symlink (the container_info has the
         # proxy looking for the target in the wrong policy)
         with self.assertRaises(ClientException) as ctx:
-            client.get_object(self.url, self.token, self.container_name,
-                              'symlink')
+            self.brain.client.get_object(self.container_name, symlink_name)
         self.assertEqual(ctx.exception.http_status, 404)
 
         # of course the symlink itself is fine
-        metadata, body = client.get_object(self.url, self.token,
-                                           self.container_name, 'symlink',
-                                           query_string='symlink=get')
+        metadata, body = self.brain.client.get_object(
+            self.container_name, symlink_name, query_string='symlink=get')
         self.assertEqual(metadata['x-symlink-target'],
-                         '%s/target' % self.container_name)
+                         utils.quote('%s/%s' % (
+                             self.container_name, target_name)))
         self.assertEqual(metadata['content-type'], 'application/symlink')
         self.assertEqual(body, b'')
         # ... although in the wrong policy
         object_ring = POLICIES.get_object_ring(int(wrong_policy), '/etc/swift')
         part, nodes = object_ring.get_nodes(
-            self.account, self.container_name, 'symlink')
+            self.account, self.container_name, symlink_name)
         for node in nodes:
             metadata = direct_client.direct_head_object(
-                node, part, self.account, self.container_name, 'symlink',
+                node, part, self.account, self.container_name, symlink_name,
                 headers={'X-Backend-Storage-Policy-Index': int(wrong_policy)})
             self.assertEqual(metadata['X-Object-Sysmeta-Symlink-Target'],
-                             '%s/target' % self.container_name)
+                             utils.quote('%s/%s' % (
+                                 self.container_name, target_name)))
 
         # let the reconciler run
         self.brain.start_handoff_half()
         self.get_to_final_state()
         Manager(['container-reconciler']).once()
         # clear proxy cache
-        client.post_container(self.url, self.token, self.container_name, {})
+        self.brain.client.post_container(self.container_name, {})
 
         # now the symlink works
-        metadata, body = client.get_object(self.url, self.token,
-                                           self.container_name, 'symlink')
+        metadata, body = self.brain.client.get_object(
+            self.container_name, symlink_name)
         self.assertEqual(body, b'this is the target data')
         # and it's in the correct policy
         object_ring = POLICIES.get_object_ring(int(policy), '/etc/swift')
         part, nodes = object_ring.get_nodes(
-            self.account, self.container_name, 'symlink')
+            self.account, self.container_name, symlink_name)
         for node in nodes:
             metadata = direct_client.direct_head_object(
-                node, part, self.account, self.container_name, 'symlink',
+                node, part, self.account, self.container_name, symlink_name,
                 headers={'X-Backend-Storage-Policy-Index': int(policy)})
             self.assertEqual(metadata['X-Object-Sysmeta-Symlink-Target'],
-                             '%s/target' % self.container_name)
+                             utils.quote('%s/%s' % (
+                                 self.container_name, target_name)))
 
     def test_reconciler_move_object_twice(self):
         # select some policies
@@ -552,5 +558,24 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
         self.assertEqual('custom-meta', headers['x-object-meta-test'])
 
 
+class TestReservedNamespaceMergePolicyIndex(TestContainerMergePolicyIndex):
+
+    @unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy")
+    def setUp(self):
+        super(TestReservedNamespaceMergePolicyIndex, self).setUp()
+        self.container_name = get_reserved_name('container', str(uuid.uuid4()))
+        self.object_name = get_reserved_name('object', str(uuid.uuid4()))
+        self.brain = InternalBrainSplitter('/etc/swift/internal-client.conf',
+                                           self.container_name,
+                                           self.object_name, 'container')
+
+    def get_object_name(self, name):
+        return get_reserved_name(name)
+
+    def test_reconcile_manifest(self):
+        raise unittest.SkipTest(
+            'SLO does not allow parts in the reserved namespace')
+
+
 if __name__ == "__main__":
     unittest.main()

test/unit/common/test_internal_client.py

@@ -1330,7 +1330,7 @@ class TestInternalClient(unittest.TestCase):
 
             def make_request(
                     self, method, path, headers, acceptable_statuses,
-                    body_file=None):
+                    body_file=None, params=None):
                 self.make_request_called += 1
                 self.test.assertEqual(self.path, path)
                 exp_headers = dict(self.headers)
@@ -1358,7 +1358,7 @@ class TestInternalClient(unittest.TestCase):
 
             def make_request(
                     self, method, path, headers, acceptable_statuses,
-                    body_file=None):
+                    body_file=None, params=None):
                 self.make_request_called += 1
                 self.test.assertEqual(self.path, path)
                 exp_headers = dict(self.headers)

test/unit/common/test_request_helpers.py

@@ -19,6 +19,7 @@ import unittest
 from swift.common.swob import Request, HTTPException, HeaderKeyDict
 from swift.common.storage_policy import POLICIES, EC_POLICY, REPL_POLICY
 from swift.common import request_helpers as rh
+from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX
 
 from test.unit import patch_policies
 from test.unit.common.test_utils import FakeResponse
@@ -280,6 +281,11 @@ class TestRequestHelpers(unittest.TestCase):
                     'AUTH_foo', cont, 'baz')
             self.assertEqual(raised.exception.args[0], 'Container is required')
 
+    def test_invalid_names_in_system_accounts(self):
+        self.assertIsNone(rh.validate_internal_obj(
+            AUTO_CREATE_ACCOUNT_PREFIX + 'system_account', 'foo',
+            'crazy%stown' % rh.RESERVED))
+
     def test_invalid_reserved_names(self):
         with self.assertRaises(HTTPException) as raised:
             rh.validate_internal_obj('AUTH_foo' + rh.RESERVED, 'bar', 'baz')