diff --git a/requirements.txt b/requirements.txt index 8433958..6d95218 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,10 +8,11 @@ greenlet>=0.3.1 netifaces>=0.5,!=0.10.0,!=0.10.1 pastedeploy>=1.3.3 simplejson>=2.0.9 +six>=1.9.0 xattr>=0.4 PyECLib==1.0.7 # HPSS-specific package requirements. Get these from your HPSS support # representative. hpss -hpssfs \ No newline at end of file +hpssfs diff --git a/swiftonhpss.spec b/swiftonhpss.spec index f3f6acb..eb8eb9f 100644 --- a/swiftonhpss.spec +++ b/swiftonhpss.spec @@ -17,7 +17,7 @@ BuildRequires: python-devel BuildRequires: python-setuptools Requires : python Requires : python-setuptools -Requires : openstack-swift-object = 2.3.0 +Requires : openstack-swift-object = 2.5.0 %description SwiftOnHPSS is a Swift Object Server implementation that enables users to @@ -58,6 +58,9 @@ cp -r etc/* %{buildroot}/%{_confdir}/ rm -rf %{buildroot} %changelog +* Wed Dec 23 2015 Prashanth Pai - 2.5.0-0 +- Update spec file to support Liberty release of Swift + * Thu Dec 10 2015 Phil Bridges - Fork SwiftOnFile into SwiftOnHPSS, add HPSS-specific features diff --git a/swiftonhpss/swift/__init__.py b/swiftonhpss/swift/__init__.py index ec9c426..1aa0b0b 100644 --- a/swiftonhpss/swift/__init__.py +++ b/swiftonhpss/swift/__init__.py @@ -43,7 +43,7 @@ class PkgInfo(object): # Change the Package version here -_pkginfo = PkgInfo(canonical_version='2.3.0', +_pkginfo = PkgInfo(canonical_version='2.5.0', release='0', name='swiftonhpss', final=False) diff --git a/swiftonhpss/swift/obj/diskfile.py b/swiftonhpss/swift/obj/diskfile.py index bf31fbb..baed54b 100644 --- a/swiftonhpss/swift/obj/diskfile.py +++ b/swiftonhpss/swift/obj/diskfile.py @@ -30,7 +30,7 @@ from contextlib import contextmanager from swiftonhpss.swift.common.exceptions import AlreadyExistsAsFile, \ AlreadyExistsAsDir from swift.common.utils import ThreadPool, hash_path, \ - normalize_timestamp, fallocate + normalize_timestamp, fallocate, Timestamp from swift.common.exceptions import DiskFileNotExist, DiskFileError, \ DiskFileNoSpace, DiskFileDeviceUnavailable, DiskFileNotOpen, \ DiskFileExpired @@ -53,7 +53,7 @@ from swift.obj.diskfile import get_async_dir # FIXME: Hopefully we'll be able to move to Python 2.7+ where O_CLOEXEC will # be back ported. See http://www.python.org/dev/peps/pep-0433/ -O_CLOEXEC = 02000000 +O_CLOEXEC = 0o2000000 MAX_RENAME_ATTEMPTS = 10 MAX_OPEN_ATTEMPTS = 10 @@ -583,7 +583,7 @@ class DiskFile(object): """ def __init__(self, mgr, dev_path, threadpool, partition, account=None, container=None, obj=None, - policy=None, uid=DEFAULT_UID, gid=DEFAULT_GID): + policy=None, uid=DEFAULT_UID, gid=DEFAULT_GID, **kwargs): # Variables partition and policy is currently unused. self._mgr = mgr self._device_path = dev_path @@ -617,6 +617,18 @@ class DiskFile(object): self._data_file = os.path.join(self._put_datadir, self._obj) + @property + def timestamp(self): + if self._metadata is None: + raise DiskFileNotOpen() + return Timestamp(self._metadata.get('X-Timestamp')) + + @property + def data_timestamp(self): + if self._metadata is None: + raise DiskFileNotOpen() + return Timestamp(self._metadata.get('X-Timestamp')) + def open(self): """ Open the object. @@ -733,7 +745,6 @@ class DiskFile(object): the REST API *before* the object has actually been read. It is the responsibility of the implementation to properly handle that. """ - self._metadata = None self._close_fd() def get_metadata(self): diff --git a/test-requirements.txt b/test-requirements.txt index 27953e7..ef36d60 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,7 +3,7 @@ # process, which may cause wedges in the gate later. # Hacking already pins down pep8, pyflakes and flake8 -hacking>=0.8.0,<0.9 +hacking>=0.10.0,<0.11 coverage nose nosexcover diff --git a/test/__init__.py b/test/__init__.py index 3bd25b1..c570f26 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -15,7 +15,7 @@ # See http://code.google.com/p/python-nose/issues/detail?id=373 # The code below enables nosetests to work with i18n _() blocks - +from __future__ import print_function import sys import os try: @@ -63,15 +63,12 @@ def get_config(section_name=None, defaults=None): config = readconf(config_file, section_name) except SystemExit: if not os.path.exists(config_file): - print >>sys.stderr, \ - 'Unable to read test config %s - file not found' \ - % config_file + print('Unable to read test config %s - file not found' + % config_file, file=sys.stderr) elif not os.access(config_file, os.R_OK): - print >>sys.stderr, \ - 'Unable to read test config %s - permission denied' \ - % config_file + print('Unable to read test config %s - permission denied' + % config_file, file=sys.stderr) else: - print >>sys.stderr, \ - 'Unable to read test config %s - section %s not found' \ - % (config_file, section_name) - return config + print('Unable to read test config %s - section %s not found' + % (config_file, section_name), file=sys.stderr) + return config \ No newline at end of file diff --git a/test/functional/__init__.py b/test/functional/__init__.py index 73e5006..f07d162 100644 --- a/test/functional/__init__.py +++ b/test/functional/__init__.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function import mock import os import sys @@ -23,20 +24,24 @@ import eventlet import eventlet.debug import functools import random -from ConfigParser import ConfigParser, NoSectionError + from time import time, sleep -from httplib import HTTPException from urlparse import urlparse from nose import SkipTest from contextlib import closing from gzip import GzipFile from shutil import rmtree from tempfile import mkdtemp + +from six.moves.configparser import ConfigParser, NoSectionError +from six.moves import http_client +from six.moves.http_client import HTTPException + from swift.common.middleware.memcache import MemcacheMiddleware from swift.common.storage_policy import parse_storage_policies, PolicyError from test import get_config -from test.functional.swift_test_client import Account, Connection, \ +from test.functional.swift_test_client import Account, Connection, Container, \ ResponseError # This has the side effect of mocking out the xattr module so that unit tests # (and in this case, when in-process functional tests are called for) can run @@ -46,13 +51,13 @@ from test.unit import debug_logger, FakeMemcache from swift.common import constraints, utils, ring, storage_policy from swift.common.ring import Ring from swift.common.wsgi import monkey_patch_mimetools, loadapp -from swift.common.utils import config_true_value +from swift.common.utils import config_true_value, split_path from swift.account import server as account_server from swift.container import server as container_server from swift.obj import server as object_server, mem_server as mem_object_server import swift.proxy.controllers.obj - +http_client._MAXHEADERS = constraints.MAX_HEADER_COUNT DEBUG = True # In order to get the proper blocking behavior of sockets without using @@ -105,6 +110,7 @@ orig_swift_conf_name = None in_process = False _testdir = _test_servers = _test_coros = None +policy_specified = None class FakeMemcacheMiddleware(MemcacheMiddleware): @@ -123,7 +129,7 @@ class InProcessException(BaseException): def _info(msg): - print >> sys.stderr, msg + print(msg, file=sys.stderr) def _debug(msg): @@ -209,7 +215,6 @@ def _in_process_setup_ring(swift_conf, conf_src_dir, testdir): for policy in policies: conf.remove_section(sp_prefix + str(policy.idx)) - policy_specified = os.environ.get('SWIFT_TEST_POLICY') if policy_specified: policy_to_test = policies.get_by_name(policy_specified) if policy_to_test is None: @@ -497,7 +502,7 @@ def get_cluster_info(): # Most likely the swift cluster has "expose_info = false" set # in its proxy-server.conf file, so we'll just do the best we # can. - print >>sys.stderr, "** Swift Cluster not exposing /info **" + print("** Swift Cluster not exposing /info **", file=sys.stderr) # Finally, we'll allow any constraint present in the swift-constraints # section of test.conf to override everything. Note that only those @@ -509,8 +514,8 @@ def get_cluster_info(): except KeyError: pass except ValueError: - print >>sys.stderr, "Invalid constraint value: %s = %s" % ( - k, test_constraints[k]) + print("Invalid constraint value: %s = %s" % ( + k, test_constraints[k]), file=sys.stderr) eff_constraints.update(test_constraints) # Just make it look like these constraints were loaded from a /info call, @@ -520,6 +525,9 @@ def get_cluster_info(): def setup_package(): + + global policy_specified + policy_specified = os.environ.get('SWIFT_TEST_POLICY') in_process_env = os.environ.get('SWIFT_TEST_IN_PROCESS') if in_process_env is not None: use_in_process = utils.config_true_value(in_process_env) @@ -557,8 +565,8 @@ def setup_package(): in_process_setup(the_object_server=( mem_object_server if in_mem_obj else object_server)) except InProcessException as exc: - print >> sys.stderr, ('Exception during in-process setup: %s' - % str(exc)) + print(('Exception during in-process setup: %s' + % str(exc)), file=sys.stderr) raise global web_front_end @@ -667,20 +675,19 @@ def setup_package(): global skip skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]]) if skip: - print >>sys.stderr, 'SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG' + print('SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG', file=sys.stderr) global skip2 skip2 = not all([not skip, swift_test_user[1], swift_test_key[1]]) if not skip and skip2: - print >>sys.stderr, \ - 'SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS' \ - ' DUE TO NO CONFIG FOR THEM' + print('SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS ' + 'DUE TO NO CONFIG FOR THEM', file=sys.stderr) global skip3 skip3 = not all([not skip, swift_test_user[2], swift_test_key[2]]) if not skip and skip3: - print >>sys.stderr, \ - 'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' + print('SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS' + 'DUE TO NO CONFIG FOR THEM', file=sys.stderr) global skip_if_not_v3 skip_if_not_v3 = (swift_test_auth_version != '3' @@ -688,16 +695,33 @@ def setup_package(): swift_test_user[3], swift_test_key[3]])) if not skip and skip_if_not_v3: - print >>sys.stderr, \ - 'SKIPPING FUNCTIONAL TESTS SPECIFIC TO AUTH VERSION 3' + print('SKIPPING FUNCTIONAL TESTS SPECIFIC TO AUTH VERSION 3', + file=sys.stderr) global skip_service_tokens skip_service_tokens = not all([not skip, swift_test_user[4], swift_test_key[4], swift_test_tenant[4], swift_test_service_prefix]) if not skip and skip_service_tokens: - print >>sys.stderr, \ - 'SKIPPING FUNCTIONAL TESTS SPECIFIC TO SERVICE TOKENS' + print( + 'SKIPPING FUNCTIONAL TESTS SPECIFIC TO SERVICE TOKENS', + file=sys.stderr) + + if policy_specified: + policies = FunctionalStoragePolicyCollection.from_info() + for p in policies: + # policy names are case-insensitive + if policy_specified.lower() == p['name'].lower(): + _info('Using specified policy %s' % policy_specified) + FunctionalStoragePolicyCollection.policy_specified = p + Container.policy_specified = policy_specified + break + else: + _info( + 'SKIPPING FUNCTIONAL TESTS: Failed to find specified policy %s' + % policy_specified) + raise Exception('Failed to find specified policy %s' + % policy_specified) get_cluster_info() @@ -746,8 +770,24 @@ conn = [None, None, None, None, None] def connection(url): if has_insecure: - return http_connection(url, insecure=insecure) - return http_connection(url) + parsed_url, http_conn = http_connection(url, insecure=insecure) + else: + parsed_url, http_conn = http_connection(url) + + orig_request = http_conn.request + + # Add the policy header if policy_specified is set + def request_with_policy(method, url, body=None, headers={}): + version, account, container, obj = split_path(url, 1, 4, True) + if policy_specified and method == 'PUT' and container and not obj \ + and 'X-Storage-Policy' not in headers: + headers['X-Storage-Policy'] = policy_specified + + return orig_request(method, url, body, headers) + + http_conn.request = request_with_policy + + return parsed_url, http_conn def get_url_token(user_index, os_options): @@ -898,6 +938,9 @@ def requires_acls(f): class FunctionalStoragePolicyCollection(object): + # policy_specified is set in __init__.py when tests are being set up. + policy_specified = None + def __init__(self, policies): self._all = policies self.default = None @@ -939,7 +982,12 @@ class FunctionalStoragePolicyCollection(object): p.get(k) != v for k, v in kwargs.items())]) def select(self): - return random.choice(self) + # check that a policy was specified and that it is available + # in the current list (i.e., hasn't been excluded of the current list) + if self.policy_specified and self.policy_specified in self: + return self.policy_specified + else: + return random.choice(self) def requires_policies(f): diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index f68dc03..0148ba7 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -14,25 +14,27 @@ # limitations under the License. import hashlib -import httplib import os import random import socket -import StringIO import time import urllib import simplejson as json - from nose import SkipTest from xml.dom import minidom +import six +from six.moves import http_client from swiftclient import get_auth +from swift.common import constraints from swift.common.utils import config_true_value from test import safe_repr +http_client._MAXHEADERS = constraints.MAX_HEADER_COUNT + class AuthenticationFailed(Exception): pass @@ -68,7 +70,7 @@ class ResponseError(Exception): def listing_empty(method): - for i in xrange(6): + for i in range(6): if len(method()) == 0: return True @@ -163,10 +165,10 @@ class Connection(object): x = storage_url.split('/') if x[0] == 'http:': - self.conn_class = httplib.HTTPConnection + self.conn_class = http_client.HTTPConnection self.storage_port = 80 elif x[0] == 'https:': - self.conn_class = httplib.HTTPSConnection + self.conn_class = http_client.HTTPSConnection self.storage_port = 443 else: raise ValueError('unexpected protocol %s' % (x[0])) @@ -181,7 +183,11 @@ class Connection(object): self.storage_url = str('/%s/%s' % (x[3], x[4])) self.account_name = str(x[4]) self.auth_user = auth_user - self.storage_token = storage_token + # With v2 keystone, storage_token is unicode. + # We want it to be string otherwise this would cause + # troubles when doing query with already encoded + # non ascii characters in its headers. + self.storage_token = str(storage_token) self.user_acl = '%s:%s' % (self.account, self.username) self.http_connect() @@ -202,7 +208,7 @@ class Connection(object): def http_connect(self): self.connection = self.conn_class(self.storage_host, port=self.storage_port) - #self.connection.set_debuglevel(3) + # self.connection.set_debuglevel(3) def make_path(self, path=None, cfg=None): if path is None: @@ -230,6 +236,9 @@ class Connection(object): if not cfg.get('no_auth_token'): headers['X-Auth-Token'] = self.storage_token + if cfg.get('use_token'): + headers['X-Auth-Token'] = cfg.get('use_token') + if isinstance(hdrs, dict): headers.update(hdrs) return headers @@ -276,7 +285,7 @@ class Connection(object): try: self.response = try_request() - except httplib.HTTPException as e: + except http_client.HTTPException as e: fail_messages.append(safe_repr(e)) continue @@ -328,9 +337,9 @@ class Connection(object): self.connection = self.conn_class(self.storage_host, port=self.storage_port) - #self.connection.set_debuglevel(3) + # self.connection.set_debuglevel(3) self.connection.putrequest('PUT', path) - for key, value in headers.iteritems(): + for key, value in headers.items(): self.connection.putheader(key, value) self.connection.endheaders() @@ -481,6 +490,9 @@ class Account(Base): class Container(Base): + # policy_specified is set in __init__.py when tests are being set up. + policy_specified = None + def __init__(self, conn, account, name): self.conn = conn self.account = str(account) @@ -493,9 +505,23 @@ class Container(Base): parms = {} if cfg is None: cfg = {} + if self.policy_specified and 'X-Storage-Policy' not in hdrs: + hdrs['X-Storage-Policy'] = self.policy_specified return self.conn.make_request('PUT', self.path, hdrs=hdrs, parms=parms, cfg=cfg) in (201, 202) + def update_metadata(self, hdrs=None, cfg=None): + if hdrs is None: + hdrs = {} + if cfg is None: + cfg = {} + + self.conn.make_request('POST', self.path, hdrs=hdrs, cfg=cfg) + if not 200 <= self.conn.response.status <= 299: + raise ResponseError(self.conn.response, 'POST', + self.conn.make_path(self.path)) + return True + def delete(self, hdrs=None, parms=None): if hdrs is None: hdrs = {} @@ -626,6 +652,9 @@ class File(Base): else: headers['Content-Length'] = 0 + if cfg.get('use_token'): + headers['X-Auth-Token'] = cfg.get('use_token') + if cfg.get('no_content_type'): pass elif self.content_type: @@ -643,7 +672,7 @@ class File(Base): block_size = 4096 if isinstance(data, str): - data = StringIO.StringIO(data) + data = six.StringIO(data) checksum = hashlib.md5() buff = data.read(block_size) @@ -700,13 +729,13 @@ class File(Base): return self.conn.make_request('COPY', self.path, hdrs=headers, parms=parms) == 201 - def delete(self, hdrs=None, parms=None): + def delete(self, hdrs=None, parms=None, cfg=None): if hdrs is None: hdrs = {} if parms is None: parms = {} if self.conn.make_request('DELETE', self.path, hdrs=hdrs, - parms=parms) != 204: + cfg=cfg, parms=parms) != 204: raise ResponseError(self.conn.response, 'DELETE', self.conn.make_path(self.path)) @@ -847,7 +876,7 @@ class File(Base): finally: fobj.close() - def sync_metadata(self, metadata=None, cfg=None): + def sync_metadata(self, metadata=None, cfg=None, parms=None): if metadata is None: metadata = {} if cfg is None: @@ -864,7 +893,8 @@ class File(Base): else: headers['Content-Length'] = 0 - self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg) + self.conn.make_request('POST', self.path, hdrs=headers, + parms=parms, cfg=cfg) if self.conn.response.status not in (201, 202): raise ResponseError(self.conn.response, 'POST', @@ -917,7 +947,7 @@ class File(Base): pass self.size = int(os.fstat(data.fileno())[6]) else: - data = StringIO.StringIO(data) + data = six.StringIO(data) self.size = data.len headers = self.make_headers(cfg=cfg) @@ -969,7 +999,7 @@ class File(Base): if not self.write(data, hdrs=hdrs, parms=parms, cfg=cfg): raise ResponseError(self.conn.response, 'PUT', self.conn.make_path(self.path)) - self.md5 = self.compute_md5sum(StringIO.StringIO(data)) + self.md5 = self.compute_md5sum(six.StringIO(data)) return data def write_random_return_resp(self, size=None, hdrs=None, parms=None, @@ -986,5 +1016,28 @@ class File(Base): return_resp=True) if not resp: raise ResponseError(self.conn.response) - self.md5 = self.compute_md5sum(StringIO.StringIO(data)) + self.md5 = self.compute_md5sum(six.StringIO(data)) return resp + + def post(self, hdrs=None, parms=None, cfg=None, return_resp=False): + if hdrs is None: + hdrs = {} + if parms is None: + parms = {} + if cfg is None: + cfg = {} + + headers = self.make_headers(cfg=cfg) + headers.update(hdrs) + + self.conn.make_request('POST', self.path, hdrs=headers, + parms=parms, cfg=cfg) + + if self.conn.response.status not in (201, 202): + raise ResponseError(self.conn.response, 'POST', + self.conn.make_path(self.path)) + + if return_resp: + return self.conn.response + + return True diff --git a/test/functional/test_account.py b/test/functional/test_account.py index eee42e5..032600c 100755 --- a/test/functional/test_account.py +++ b/test/functional/test_account.py @@ -21,6 +21,7 @@ from uuid import uuid4 from nose import SkipTest from string import letters +from six.moves import range from swift.common.middleware.acl import format_acl from test.functional import check_response, retry, requires_acls, \ @@ -88,22 +89,22 @@ class TestAccount(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(head) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('x-account-meta-test'), None) resp = retry(get) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('x-account-meta-test'), None) resp = retry(post, 'Value') resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('x-account-meta-test'), 'Value') resp = retry(get) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('x-account-meta-test'), 'Value') def test_invalid_acls(self): @@ -189,7 +190,7 @@ class TestAccount(unittest.TestCase): # cannot read account resp = retry(get, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # grant read access acl_user = tf.swift_test_user[2] @@ -203,7 +204,7 @@ class TestAccount(unittest.TestCase): # read-only can read account headers resp = retry(get, use_account=3) resp.read() - self.assert_(resp.status in (200, 204)) + self.assertIn(resp.status, (200, 204)) # but not acls self.assertEqual(resp.getheader('X-Account-Access-Control'), None) @@ -220,7 +221,7 @@ class TestAccount(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(get, use_account=3) resp.read() - self.assert_(resp.status in (200, 204)) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('X-Account-Meta-Test'), 'value') @requires_acls @@ -240,7 +241,7 @@ class TestAccount(unittest.TestCase): # cannot read account resp = retry(get, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # grant read-write access acl_user = tf.swift_test_user[2] @@ -254,7 +255,7 @@ class TestAccount(unittest.TestCase): # read-write can read account headers resp = retry(get, use_account=3) resp.read() - self.assert_(resp.status in (200, 204)) + self.assertIn(resp.status, (200, 204)) # but not acls self.assertEqual(resp.getheader('X-Account-Access-Control'), None) @@ -281,7 +282,7 @@ class TestAccount(unittest.TestCase): # cannot read account resp = retry(get, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # grant admin access acl_user = tf.swift_test_user[2] @@ -295,7 +296,7 @@ class TestAccount(unittest.TestCase): # admin can read account headers resp = retry(get, use_account=3) resp.read() - self.assert_(resp.status in (200, 204)) + self.assertIn(resp.status, (200, 204)) # including acls self.assertEqual(resp.getheader('X-Account-Access-Control'), acl_json_str) @@ -308,7 +309,7 @@ class TestAccount(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(get, use_account=3) resp.read() - self.assert_(resp.status in (200, 204)) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) # admin can even revoke their own access @@ -320,7 +321,7 @@ class TestAccount(unittest.TestCase): # and again, cannot read account resp = retry(get, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) @requires_acls def test_protected_tempurl(self): @@ -358,8 +359,9 @@ class TestAccount(unittest.TestCase): # read-only tester3 can read account metadata resp = retry(get, use_account=3) resp.read() - self.assert_(resp.status in (200, 204), - 'Expected status in (200, 204), got %s' % resp.status) + self.assertTrue( + resp.status in (200, 204), + 'Expected status in (200, 204), got %s' % resp.status) self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) # but not temp-url-key self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), None) @@ -376,8 +378,9 @@ class TestAccount(unittest.TestCase): # read-write tester3 can read account metadata resp = retry(get, use_account=3) resp.read() - self.assert_(resp.status in (200, 204), - 'Expected status in (200, 204), got %s' % resp.status) + self.assertTrue( + resp.status in (200, 204), + 'Expected status in (200, 204), got %s' % resp.status) self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) # but not temp-url-key self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), None) @@ -394,8 +397,9 @@ class TestAccount(unittest.TestCase): # admin tester3 can read account metadata resp = retry(get, use_account=3) resp.read() - self.assert_(resp.status in (200, 204), - 'Expected status in (200, 204), got %s' % resp.status) + self.assertTrue( + resp.status in (200, 204), + 'Expected status in (200, 204), got %s' % resp.status) self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) # including temp-url-key self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), @@ -411,8 +415,9 @@ class TestAccount(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(get, use_account=3) resp.read() - self.assert_(resp.status in (200, 204), - 'Expected status in (200, 204), got %s' % resp.status) + self.assertTrue( + resp.status in (200, 204), + 'Expected status in (200, 204), got %s' % resp.status) self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), secret) @@ -688,17 +693,17 @@ class TestAccount(unittest.TestCase): if (tf.web_front_end == 'integral'): resp = retry(post, uni_key, '1') resp.read() - self.assertTrue(resp.status in (201, 204)) + self.assertIn(resp.status, (201, 204)) resp = retry(head) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader(uni_key.encode('utf-8')), '1') resp = retry(post, 'X-Account-Meta-uni', uni_value) resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('X-Account-Meta-uni'), uni_value.encode('utf-8')) if (tf.web_front_end == 'integral'): @@ -707,7 +712,7 @@ class TestAccount(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(head) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader(uni_key.encode('utf-8')), uni_value.encode('utf-8')) @@ -729,24 +734,23 @@ class TestAccount(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(head) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('x-account-meta-one'), '1') resp = retry(post, 'X-Account-Meta-Two', '2') resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('x-account-meta-one'), '1') self.assertEqual(resp.getheader('x-account-meta-two'), '2') def test_bad_metadata(self): - - raise SkipTest('SOF constraints middleware enforces constraints.') - if tf.skip: raise SkipTest + raise SkipTest('SOF constraints middleware enforces constraints.') + def post(url, token, parsed, conn, extra_headers): headers = {'X-Auth-Token': token} headers.update(extra_headers) @@ -793,13 +797,13 @@ class TestAccount(unittest.TestCase): resp = retry(post, headers) headers = {} - for x in xrange(self.max_meta_count): + for x in range(self.max_meta_count): headers['X-Account-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 204) headers = {} - for x in xrange(self.max_meta_count + 1): + for x in range(self.max_meta_count + 1): headers['X-Account-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() @@ -830,8 +834,23 @@ class TestAccount(unittest.TestCase): resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 204) + # this POST includes metadata size that is over limit headers['X-Account-Meta-k'] = \ - 'v' * (self.max_meta_overall_size - size) + 'x' * (self.max_meta_overall_size - size) + resp = retry(post, headers) + resp.read() + self.assertEqual(resp.status, 400) + # this POST would be ok and the aggregate backend metadata + # size is on the border + headers = {'X-Account-Meta-k': + 'y' * (self.max_meta_overall_size - size - 1)} + resp = retry(post, headers) + resp.read() + self.assertEqual(resp.status, 204) + # this last POST would be ok by itself but takes the aggregate + # backend metadata size over limit + headers = {'X-Account-Meta-k': + 'z' * (self.max_meta_overall_size - size)} resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 400) @@ -862,7 +881,7 @@ class TestAccountInNonDefaultDomain(unittest.TestCase): resp = retry(head, use_account=4) resp.read() self.assertEqual(resp.status, 204) - self.assertTrue('X-Account-Project-Domain-Id' in resp.headers) + self.assertIn('X-Account-Project-Domain-Id', resp.headers) if __name__ == '__main__': diff --git a/test/functional/test_container.py b/test/functional/test_container.py index ba561b5..715051d 100755 --- a/test/functional/test_container.py +++ b/test/functional/test_container.py @@ -24,6 +24,8 @@ from test.functional import check_response, retry, requires_acls, \ load_constraint, requires_policies import test.functional as tf +from six.moves import range + class TestContainer(unittest.TestCase): @@ -70,7 +72,7 @@ class TestContainer(unittest.TestCase): body = resp.read() if resp.status == 404: break - self.assert_(resp.status // 100 == 2, resp.status) + self.assertTrue(resp.status // 100 == 2, resp.status) objs = json.loads(body) if not objs: break @@ -91,7 +93,7 @@ class TestContainer(unittest.TestCase): # container may have not been created resp = retry(delete, self.container) resp.read() - self.assert_(resp.status in (204, 404)) + self.assertIn(resp.status, (204, 404)) def test_multi_metadata(self): if tf.skip: @@ -112,14 +114,14 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(head) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('x-container-meta-one'), '1') resp = retry(post, 'X-Container-Meta-Two', '2') resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('x-container-meta-one'), '1') self.assertEqual(resp.getheader('x-container-meta-two'), '2') @@ -145,14 +147,14 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(head) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader(uni_key.encode('utf-8')), '1') resp = retry(post, 'X-Container-Meta-uni', uni_value) resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('X-Container-Meta-uni'), uni_value.encode('utf-8')) if (tf.web_front_end == 'integral'): @@ -161,7 +163,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(head) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader(uni_key.encode('utf-8')), uni_value.encode('utf-8')) @@ -196,11 +198,11 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 201) resp = retry(head, name) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') resp = retry(get, name) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') resp = retry(delete, name) resp.read() @@ -212,11 +214,11 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 201) resp = retry(head, name) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('x-container-meta-test'), None) resp = retry(get, name) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('x-container-meta-test'), None) resp = retry(delete, name) resp.read() @@ -244,22 +246,22 @@ class TestContainer(unittest.TestCase): resp = retry(head) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('x-container-meta-test'), None) resp = retry(get) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('x-container-meta-test'), None) resp = retry(post, 'Value') resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') resp = retry(get) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') def test_PUT_bad_metadata(self): @@ -319,7 +321,7 @@ class TestContainer(unittest.TestCase): name = uuid4().hex headers = {} - for x in xrange(self.max_meta_count): + for x in range(self.max_meta_count): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(put, name, headers) resp.read() @@ -329,7 +331,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 204) name = uuid4().hex headers = {} - for x in xrange(self.max_meta_count + 1): + for x in range(self.max_meta_count + 1): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(put, name, headers) resp.read() @@ -368,12 +370,11 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 404) def test_POST_bad_metadata(self): - - raise SkipTest('SOF constraints middleware enforces constraints.') - if tf.skip: raise SkipTest + raise SkipTest('SOF constraints middleware enforces constraints.') + def post(url, token, parsed, conn, extra_headers): headers = {'X-Auth-Token': token} headers.update(extra_headers) @@ -415,13 +416,13 @@ class TestContainer(unittest.TestCase): return check_response(conn) headers = {} - for x in xrange(self.max_meta_count): + for x in range(self.max_meta_count): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 204) headers = {} - for x in xrange(self.max_meta_count + 1): + for x in range(self.max_meta_count + 1): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() @@ -452,8 +453,23 @@ class TestContainer(unittest.TestCase): resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 204) + # this POST includes metadata size that is over limit headers['X-Container-Meta-k'] = \ - 'v' * (self.max_meta_overall_size - size) + 'x' * (self.max_meta_overall_size - size) + resp = retry(post, headers) + resp.read() + self.assertEqual(resp.status, 400) + # this POST would be ok and the aggregate backend metadata + # size is on the border + headers = {'X-Container-Meta-k': + 'y' * (self.max_meta_overall_size - size - 1)} + resp = retry(post, headers) + resp.read() + self.assertEqual(resp.status, 204) + # this last POST would be ok by itself but takes the aggregate + # backend metadata size over limit + headers = {'X-Container-Meta-k': + 'z' * (self.max_meta_overall_size - size)} resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 400) @@ -470,7 +486,7 @@ class TestContainer(unittest.TestCase): resp = retry(get) raise Exception('Should not have been able to GET') except Exception as err: - self.assert_(str(err).startswith('No result after '), err) + self.assertTrue(str(err).startswith('No result after '), err) def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', @@ -497,7 +513,7 @@ class TestContainer(unittest.TestCase): resp = retry(get) raise Exception('Should not have been able to GET') except Exception as err: - self.assert_(str(err).startswith('No result after '), err) + self.assertTrue(str(err).startswith('No result after '), err) def test_cross_account_container(self): if tf.skip or tf.skip2: @@ -715,7 +731,7 @@ class TestContainer(unittest.TestCase): # cannot list containers resp = retry(get, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # grant read-only access acl_user = tf.swift_test_user[2] @@ -728,23 +744,23 @@ class TestContainer(unittest.TestCase): # read-only can list containers resp = retry(get, use_account=3) listing = resp.read() - self.assertEquals(resp.status, 200) - self.assert_(self.name in listing) + self.assertEqual(resp.status, 200) + self.assertIn(self.name, listing) # read-only can not create containers new_container_name = str(uuid4()) resp = retry(put, new_container_name, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # but it can see newly created ones resp = retry(put, new_container_name, use_account=1) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(get, use_account=3) listing = resp.read() - self.assertEquals(resp.status, 200) - self.assert_(new_container_name in listing) + self.assertEqual(resp.status, 200) + self.assertIn(new_container_name, listing) @requires_acls def test_read_only_acl_metadata(self): @@ -774,13 +790,13 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=1) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # cannot see metadata resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # grant read-only access acl_user = tf.swift_test_user[2] @@ -800,7 +816,7 @@ class TestContainer(unittest.TestCase): # read-only can read container metadata resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) @requires_acls @@ -830,7 +846,7 @@ class TestContainer(unittest.TestCase): # cannot list containers resp = retry(get, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # grant read-write access acl_user = tf.swift_test_user[2] @@ -843,36 +859,36 @@ class TestContainer(unittest.TestCase): # can list containers resp = retry(get, use_account=3) listing = resp.read() - self.assertEquals(resp.status, 200) - self.assert_(self.name in listing) + self.assertEqual(resp.status, 200) + self.assertIn(self.name, listing) # can create new containers new_container_name = str(uuid4()) resp = retry(put, new_container_name, use_account=3) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(get, use_account=3) listing = resp.read() - self.assertEquals(resp.status, 200) - self.assert_(new_container_name in listing) + self.assertEqual(resp.status, 200) + self.assertIn(new_container_name, listing) # can also delete them resp = retry(delete, new_container_name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(get, use_account=3) listing = resp.read() - self.assertEquals(resp.status, 200) - self.assert_(new_container_name not in listing) + self.assertEqual(resp.status, 200) + self.assertNotIn(new_container_name, listing) # even if they didn't create them empty_container_name = str(uuid4()) resp = retry(put, empty_container_name, use_account=1) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(delete, empty_container_name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) @requires_acls def test_read_write_acl_metadata(self): @@ -902,13 +918,13 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=1) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # cannot see metadata resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # grant read-write access acl_user = tf.swift_test_user[2] @@ -921,7 +937,7 @@ class TestContainer(unittest.TestCase): # read-write can read container metadata resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # read-write can also write container metadata @@ -929,20 +945,20 @@ class TestContainer(unittest.TestCase): headers = {'x-container-meta-test': new_value} resp = retry(post, self.name, headers=headers, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) # and remove it headers = {'x-remove-container-meta-test': 'true'} resp = retry(post, self.name, headers=headers, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), None) @requires_acls @@ -972,7 +988,7 @@ class TestContainer(unittest.TestCase): # cannot list containers resp = retry(get, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # grant admin access acl_user = tf.swift_test_user[2] @@ -985,36 +1001,36 @@ class TestContainer(unittest.TestCase): # can list containers resp = retry(get, use_account=3) listing = resp.read() - self.assertEquals(resp.status, 200) - self.assert_(self.name in listing) + self.assertEqual(resp.status, 200) + self.assertIn(self.name, listing) # can create new containers new_container_name = str(uuid4()) resp = retry(put, new_container_name, use_account=3) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(get, use_account=3) listing = resp.read() - self.assertEquals(resp.status, 200) - self.assert_(new_container_name in listing) + self.assertEqual(resp.status, 200) + self.assertIn(new_container_name, listing) # can also delete them resp = retry(delete, new_container_name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(get, use_account=3) listing = resp.read() - self.assertEquals(resp.status, 200) - self.assert_(new_container_name not in listing) + self.assertEqual(resp.status, 200) + self.assertNotIn(new_container_name, listing) # even if they didn't create them empty_container_name = str(uuid4()) resp = retry(put, empty_container_name, use_account=1) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(delete, empty_container_name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) @requires_acls def test_admin_acl_metadata(self): @@ -1044,13 +1060,13 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=1) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # cannot see metadata resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # grant access acl_user = tf.swift_test_user[2] @@ -1063,7 +1079,7 @@ class TestContainer(unittest.TestCase): # can read container metadata resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # can also write container metadata @@ -1071,20 +1087,20 @@ class TestContainer(unittest.TestCase): headers = {'x-container-meta-test': new_value} resp = retry(post, self.name, headers=headers, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) # and remove it headers = {'x-remove-container-meta-test': 'true'} resp = retry(post, self.name, headers=headers, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), None) @requires_acls @@ -1118,7 +1134,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=1) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) @@ -1133,7 +1149,7 @@ class TestContainer(unittest.TestCase): # can read container metadata resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # but not sync-key self.assertEqual(resp.getheader('X-Container-Sync-Key'), None) @@ -1155,7 +1171,7 @@ class TestContainer(unittest.TestCase): # can read container metadata resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # but not sync-key self.assertEqual(resp.getheader('X-Container-Sync-Key'), None) @@ -1163,7 +1179,7 @@ class TestContainer(unittest.TestCase): # sanity check sync-key w/ account1 resp = retry(get, self.name, use_account=1) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') # and can write @@ -1177,7 +1193,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=1) # validate w/ account1 resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) # but can not write sync-key self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') @@ -1193,7 +1209,7 @@ class TestContainer(unittest.TestCase): # admin can read container metadata resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) # and ALSO sync-key self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') @@ -1206,7 +1222,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Sync-Key'), new_secret) @requires_acls @@ -1241,7 +1257,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=1) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe') self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) @@ -1257,7 +1273,7 @@ class TestContainer(unittest.TestCase): # can read container metadata resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # but not container acl self.assertEqual(resp.getheader('X-Container-Read'), None) @@ -1283,7 +1299,7 @@ class TestContainer(unittest.TestCase): # can read container metadata resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # but not container acl self.assertEqual(resp.getheader('X-Container-Read'), None) @@ -1292,7 +1308,7 @@ class TestContainer(unittest.TestCase): # sanity check container acls with account1 resp = retry(get, self.name, use_account=1) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe') @@ -1308,7 +1324,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=1) # validate w/ account1 resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) # but can not write container acls self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') @@ -1325,7 +1341,7 @@ class TestContainer(unittest.TestCase): # admin can read container metadata resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) # and ALSO container acls self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') @@ -1341,7 +1357,7 @@ class TestContainer(unittest.TestCase): self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=3) resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Read'), '.r:*') def test_long_name_content_type(self): @@ -1384,8 +1400,11 @@ class TestContainer(unittest.TestCase): raise SkipTest() def put(url, token, parsed, conn): + # using the empty storage policy header value here to ensure + # that the default policy is chosen in case policy_specified is set + # see __init__.py for details on policy_specified conn.request('PUT', parsed.path + '/' + self.container, '', - {'X-Auth-Token': token}) + {'X-Auth-Token': token, 'X-Storage-Policy': ''}) return check_response(conn) resp = retry(put) resp.read() @@ -1398,8 +1417,8 @@ class TestContainer(unittest.TestCase): resp = retry(head) resp.read() headers = dict((k.lower(), v) for k, v in resp.getheaders()) - self.assertEquals(headers.get('x-storage-policy'), - default_policy['name']) + self.assertEqual(headers.get('x-storage-policy'), + default_policy['name']) def test_error_invalid_storage_policy_name(self): def put(url, token, parsed, conn, headers): @@ -1436,8 +1455,8 @@ class TestContainer(unittest.TestCase): resp = retry(head) resp.read() headers = dict((k.lower(), v) for k, v in resp.getheaders()) - self.assertEquals(headers.get('x-storage-policy'), - policy['name']) + self.assertEqual(headers.get('x-storage-policy'), + policy['name']) # and test recreate with-out specifying Storage Policy resp = retry(put) @@ -1447,8 +1466,8 @@ class TestContainer(unittest.TestCase): resp = retry(head) resp.read() headers = dict((k.lower(), v) for k, v in resp.getheaders()) - self.assertEquals(headers.get('x-storage-policy'), - policy['name']) + self.assertEqual(headers.get('x-storage-policy'), + policy['name']) # delete it def delete(url, token, parsed, conn): @@ -1463,7 +1482,7 @@ class TestContainer(unittest.TestCase): resp = retry(head) resp.read() headers = dict((k.lower(), v) for k, v in resp.getheaders()) - self.assertEquals(headers.get('x-storage-policy'), None) + self.assertEqual(headers.get('x-storage-policy'), None) @requires_policies def test_conflict_change_storage_policy_with_put(self): @@ -1493,8 +1512,8 @@ class TestContainer(unittest.TestCase): resp = retry(head) resp.read() headers = dict((k.lower(), v) for k, v in resp.getheaders()) - self.assertEquals(headers.get('x-storage-policy'), - policy['name']) + self.assertEqual(headers.get('x-storage-policy'), + policy['name']) @requires_policies def test_noop_change_storage_policy_with_post(self): @@ -1530,8 +1549,8 @@ class TestContainer(unittest.TestCase): resp = retry(head) resp.read() headers = dict((k.lower(), v) for k, v in resp.getheaders()) - self.assertEquals(headers.get('x-storage-policy'), - policy['name']) + self.assertEqual(headers.get('x-storage-policy'), + policy['name']) class BaseTestContainerACLs(unittest.TestCase): @@ -1578,7 +1597,7 @@ class BaseTestContainerACLs(unittest.TestCase): while True: resp = retry(get, use_account=self.account) body = resp.read() - self.assert_(resp.status // 100 == 2, resp.status) + self.assertTrue(resp.status // 100 == 2, resp.status) objs = json.loads(body) if not objs: break diff --git a/test/functional/test_object.py b/test/functional/test_object.py index e74a7f6..5586809 100755 --- a/test/functional/test_object.py +++ b/test/functional/test_object.py @@ -15,11 +15,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import unittest from nose import SkipTest from uuid import uuid4 -from swift.common.utils import json +from six.moves import range from test.functional import check_response, retry, requires_acls, \ requires_policies @@ -88,7 +89,7 @@ class TestObject(unittest.TestCase): body = resp.read() if resp.status == 404: break - self.assert_(resp.status // 100 == 2, resp.status) + self.assertTrue(resp.status // 100 == 2, resp.status) objs = json.loads(body) if not objs: break @@ -106,7 +107,7 @@ class TestObject(unittest.TestCase): for container in self.containers: resp = retry(delete, container) resp.read() - self.assert_(resp.status in (204, 404)) + self.assertIn(resp.status, (204, 404)) def test_if_none_match(self): def put(url, token, parsed, conn): @@ -118,10 +119,10 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) resp = retry(put) resp.read() - self.assertEquals(resp.status, 412) + self.assertEqual(resp.status, 412) def put(url, token, parsed, conn): conn.request('PUT', '%s/%s/%s' % ( @@ -132,7 +133,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) def test_non_integer_x_delete_after(self): def put(url, token, parsed, conn): @@ -144,7 +145,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) body = resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) self.assertEqual(body, 'Non-integer X-Delete-After') def test_non_integer_x_delete_at(self): @@ -157,7 +158,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) body = resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) self.assertEqual(body, 'Non-integer X-Delete-At') def test_x_delete_at_in_the_past(self): @@ -170,7 +171,7 @@ class TestObject(unittest.TestCase): return check_response(conn) resp = retry(put) body = resp.read() - self.assertEquals(resp.status, 400) + self.assertEqual(resp.status, 400) self.assertEqual(body, 'X-Delete-At in past') def test_copy_object(self): @@ -242,6 +243,23 @@ class TestObject(unittest.TestCase): self.assertEqual(resp.status, 200) self.assertEqual(dest_contents, source_contents) + # copy source to dest with COPY and range + def copy(url, token, parsed, conn): + conn.request('COPY', '%s/%s' % (parsed.path, source), '', + {'X-Auth-Token': token, + 'Destination': dest, + 'Range': 'bytes=1-2'}) + return check_response(conn) + resp = retry(copy) + resp.read() + self.assertEqual(resp.status, 201) + + # contents of dest should be the same as source + resp = retry(get_dest) + dest_contents = resp.read() + self.assertEqual(resp.status, 200) + self.assertEqual(dest_contents, source_contents[1:3]) + # delete the copy resp = retry(delete) resp.read() @@ -369,7 +387,7 @@ class TestObject(unittest.TestCase): resp = retry(get) raise Exception('Should not have been able to GET') except Exception as err: - self.assert_(str(err).startswith('No result after ')) + self.assertTrue(str(err).startswith('No result after ')) def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.container, '', @@ -394,7 +412,7 @@ class TestObject(unittest.TestCase): resp = retry(get) raise Exception('Should not have been able to GET') except Exception as err: - self.assert_(str(err).startswith('No result after ')) + self.assertTrue(str(err).startswith('No result after ')) def test_private_object(self): if tf.skip or tf.skip3: @@ -525,12 +543,12 @@ class TestObject(unittest.TestCase): # cannot list objects resp = retry(get_listing, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # cannot get object resp = retry(get, self.obj, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # grant read-only access acl_user = tf.swift_test_user[2] @@ -543,32 +561,32 @@ class TestObject(unittest.TestCase): # can list objects resp = retry(get_listing, use_account=3) listing = resp.read() - self.assertEquals(resp.status, 200) - self.assert_(self.obj in listing) + self.assertEqual(resp.status, 200) + self.assertIn(self.obj, listing) # can get object resp = retry(get, self.obj, use_account=3) body = resp.read() - self.assertEquals(resp.status, 200) - self.assertEquals(body, 'test') + self.assertEqual(resp.status, 200) + self.assertEqual(body, 'test') # can not put an object obj_name = str(uuid4()) resp = retry(put, obj_name, use_account=3) body = resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # can not delete an object resp = retry(delete, self.obj, use_account=3) body = resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # sanity with account1 resp = retry(get_listing, use_account=3) listing = resp.read() - self.assertEquals(resp.status, 200) - self.assert_(obj_name not in listing) - self.assert_(self.obj in listing) + self.assertEqual(resp.status, 200) + self.assertNotIn(obj_name, listing) + self.assertIn(self.obj, listing) @requires_acls def test_read_write(self): @@ -606,12 +624,12 @@ class TestObject(unittest.TestCase): # cannot list objects resp = retry(get_listing, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # cannot get object resp = retry(get, self.obj, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # grant read-write access acl_user = tf.swift_test_user[2] @@ -624,32 +642,32 @@ class TestObject(unittest.TestCase): # can list objects resp = retry(get_listing, use_account=3) listing = resp.read() - self.assertEquals(resp.status, 200) - self.assert_(self.obj in listing) + self.assertEqual(resp.status, 200) + self.assertIn(self.obj, listing) # can get object resp = retry(get, self.obj, use_account=3) body = resp.read() - self.assertEquals(resp.status, 200) - self.assertEquals(body, 'test') + self.assertEqual(resp.status, 200) + self.assertEqual(body, 'test') # can put an object obj_name = str(uuid4()) resp = retry(put, obj_name, use_account=3) body = resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # can delete an object resp = retry(delete, self.obj, use_account=3) body = resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # sanity with account1 resp = retry(get_listing, use_account=3) listing = resp.read() - self.assertEquals(resp.status, 200) - self.assert_(obj_name in listing) - self.assert_(self.obj not in listing) + self.assertEqual(resp.status, 200) + self.assertIn(obj_name, listing) + self.assertNotIn(self.obj, listing) @requires_acls def test_admin(self): @@ -687,12 +705,12 @@ class TestObject(unittest.TestCase): # cannot list objects resp = retry(get_listing, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # cannot get object resp = retry(get, self.obj, use_account=3) resp.read() - self.assertEquals(resp.status, 403) + self.assertEqual(resp.status, 403) # grant admin access acl_user = tf.swift_test_user[2] @@ -705,32 +723,32 @@ class TestObject(unittest.TestCase): # can list objects resp = retry(get_listing, use_account=3) listing = resp.read() - self.assertEquals(resp.status, 200) - self.assert_(self.obj in listing) + self.assertEqual(resp.status, 200) + self.assertIn(self.obj, listing) # can get object resp = retry(get, self.obj, use_account=3) body = resp.read() - self.assertEquals(resp.status, 200) - self.assertEquals(body, 'test') + self.assertEqual(resp.status, 200) + self.assertEqual(body, 'test') # can put an object obj_name = str(uuid4()) resp = retry(put, obj_name, use_account=3) body = resp.read() - self.assertEquals(resp.status, 201) + self.assertEqual(resp.status, 201) # can delete an object resp = retry(delete, self.obj, use_account=3) body = resp.read() - self.assertEquals(resp.status, 204) + self.assertEqual(resp.status, 204) # sanity with account1 resp = retry(get_listing, use_account=3) listing = resp.read() - self.assertEquals(resp.status, 200) - self.assert_(obj_name in listing) - self.assert_(self.obj not in listing) + self.assertEqual(resp.status, 200) + self.assertIn(obj_name, listing) + self.assertNotIn(self.obj, listing) def test_manifest(self): if tf.skip: @@ -746,7 +764,7 @@ class TestObject(unittest.TestCase): parsed.path, self.container, str(objnum)), segments1[objnum], {'X-Auth-Token': token}) return check_response(conn) - for objnum in xrange(len(segments1)): + for objnum in range(len(segments1)): resp = retry(put, objnum) resp.read() self.assertEqual(resp.status, 201) @@ -809,7 +827,7 @@ class TestObject(unittest.TestCase): parsed.path, self.container, str(objnum)), segments2[objnum], {'X-Auth-Token': token}) return check_response(conn) - for objnum in xrange(len(segments2)): + for objnum in range(len(segments2)): resp = retry(put, objnum) resp.read() self.assertEqual(resp.status, 201) @@ -891,7 +909,7 @@ class TestObject(unittest.TestCase): parsed.path, acontainer, str(objnum)), segments3[objnum], {'X-Auth-Token': token}) return check_response(conn) - for objnum in xrange(len(segments3)): + for objnum in range(len(segments3)): resp = retry(put, objnum) resp.read() self.assertEqual(resp.status, 201) @@ -966,7 +984,7 @@ class TestObject(unittest.TestCase): parsed.path, acontainer, str(objnum)), '', {'X-Auth-Token': token}) return check_response(conn) - for objnum in xrange(len(segments3)): + for objnum in range(len(segments3)): resp = retry(delete, objnum) resp.read() self.assertEqual(resp.status, 204) @@ -977,7 +995,7 @@ class TestObject(unittest.TestCase): parsed.path, self.container, str(objnum)), '', {'X-Auth-Token': token}) return check_response(conn) - for objnum in xrange(len(segments2)): + for objnum in range(len(segments2)): resp = retry(delete, objnum) resp.read() self.assertEqual(resp.status, 204) @@ -988,7 +1006,7 @@ class TestObject(unittest.TestCase): parsed.path, self.container, str(objnum)), '', {'X-Auth-Token': token}) return check_response(conn) - for objnum in xrange(len(segments1)): + for objnum in range(len(segments1)): resp = retry(delete, objnum) resp.read() self.assertEqual(resp.status, 204) @@ -1095,78 +1113,78 @@ class TestObject(unittest.TestCase): resp = retry(put_cors_cont, '*') resp.read() - self.assertEquals(resp.status // 100, 2) + self.assertEqual(resp.status // 100, 2) resp = retry(put_obj, 'cat') resp.read() - self.assertEquals(resp.status // 100, 2) + self.assertEqual(resp.status // 100, 2) resp = retry(check_cors, 'OPTIONS', 'cat', {'Origin': 'http://m.com'}) - self.assertEquals(resp.status, 401) + self.assertEqual(resp.status, 401) resp = retry(check_cors, 'OPTIONS', 'cat', {'Origin': 'http://m.com', 'Access-Control-Request-Method': 'GET'}) - self.assertEquals(resp.status, 200) + self.assertEqual(resp.status, 200) resp.read() headers = dict((k.lower(), v) for k, v in resp.getheaders()) - self.assertEquals(headers.get('access-control-allow-origin'), - '*') + self.assertEqual(headers.get('access-control-allow-origin'), + '*') resp = retry(check_cors, 'GET', 'cat', {'Origin': 'http://m.com'}) - self.assertEquals(resp.status, 200) + self.assertEqual(resp.status, 200) headers = dict((k.lower(), v) for k, v in resp.getheaders()) - self.assertEquals(headers.get('access-control-allow-origin'), - '*') + self.assertEqual(headers.get('access-control-allow-origin'), + '*') resp = retry(check_cors, 'GET', 'cat', {'Origin': 'http://m.com', 'X-Web-Mode': 'True'}) - self.assertEquals(resp.status, 200) + self.assertEqual(resp.status, 200) headers = dict((k.lower(), v) for k, v in resp.getheaders()) - self.assertEquals(headers.get('access-control-allow-origin'), - '*') + self.assertEqual(headers.get('access-control-allow-origin'), + '*') #################### resp = retry(put_cors_cont, 'http://secret.com') resp.read() - self.assertEquals(resp.status // 100, 2) + self.assertEqual(resp.status // 100, 2) resp = retry(check_cors, 'OPTIONS', 'cat', {'Origin': 'http://m.com', 'Access-Control-Request-Method': 'GET'}) resp.read() - self.assertEquals(resp.status, 401) + self.assertEqual(resp.status, 401) if strict_cors: resp = retry(check_cors, 'GET', 'cat', {'Origin': 'http://m.com'}) resp.read() - self.assertEquals(resp.status, 200) + self.assertEqual(resp.status, 200) headers = dict((k.lower(), v) for k, v in resp.getheaders()) - self.assertTrue('access-control-allow-origin' not in headers) + self.assertNotIn('access-control-allow-origin', headers) resp = retry(check_cors, 'GET', 'cat', {'Origin': 'http://secret.com'}) resp.read() - self.assertEquals(resp.status, 200) + self.assertEqual(resp.status, 200) headers = dict((k.lower(), v) for k, v in resp.getheaders()) - self.assertEquals(headers.get('access-control-allow-origin'), - 'http://secret.com') + self.assertEqual(headers.get('access-control-allow-origin'), + 'http://secret.com') else: resp = retry(check_cors, 'GET', 'cat', {'Origin': 'http://m.com'}) resp.read() - self.assertEquals(resp.status, 200) + self.assertEqual(resp.status, 200) headers = dict((k.lower(), v) for k, v in resp.getheaders()) - self.assertEquals(headers.get('access-control-allow-origin'), - 'http://m.com') + self.assertEqual(headers.get('access-control-allow-origin'), + 'http://m.com') @requires_policies def test_cross_policy_copy(self): diff --git a/test/functional/tests.py b/test/functional/tests.py index 2b64e4d..c68a33b 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -17,10 +17,11 @@ from datetime import datetime import hashlib import hmac +import itertools import json import locale import random -import StringIO +import six import time import unittest import urllib @@ -31,7 +32,7 @@ from nose import SkipTest from swift.common.http import is_success, is_client_error from test.functional import normalized_urls, load_constraint, cluster_info -from test.functional import check_response, retry +from test.functional import check_response, retry, requires_acls import test.functional as tf from test.functional.swift_test_client import Account, Connection, File, \ ResponseError @@ -55,7 +56,7 @@ class Utils(object): u'\u3705\u1803\u0902\uF112\uD210\uB30E\u940C\u850B'\ u'\u5608\u3706\u1804\u0903\u03A9\u2603' return ''.join([random.choice(utf8_chars) - for x in xrange(length)]).encode('utf-8') + for x in range(length)]).encode('utf-8') create_name = create_ascii_name @@ -69,15 +70,16 @@ class Base(unittest.TestCase): def assert_body(self, body): response_body = self.env.conn.response.read() - self.assert_(response_body == body, - 'Body returned: %s' % (response_body)) + self.assertTrue(response_body == body, + 'Body returned: %s' % (response_body)) def assert_status(self, status_or_statuses): - self.assert_(self.env.conn.response.status == status_or_statuses or - (hasattr(status_or_statuses, '__iter__') and - self.env.conn.response.status in status_or_statuses), - 'Status returned: %d Expected: %s' % - (self.env.conn.response.status, status_or_statuses)) + self.assertTrue( + self.env.conn.response.status == status_or_statuses or + (hasattr(status_or_statuses, '__iter__') and + self.env.conn.response.status in status_or_statuses), + 'Status returned: %d Expected: %s' % + (self.env.conn.response.status, status_or_statuses)) class Base2(object): @@ -132,7 +134,7 @@ class TestAccount(Base): def testInvalidUTF8Path(self): invalid_utf8 = Utils.create_utf8_name()[::-1] container = self.env.account.container(invalid_utf8) - self.assert_(not container.create(None, None)) + self.assertFalse(container.create(cfg={'no_path_quote': True})) self.assert_status(412) self.assert_body('Invalid UTF8 or contains NULL') @@ -165,7 +167,7 @@ class TestAccount(Base): info = self.env.account.info() for field in ['object_count', 'container_count', 'bytes_used']: - self.assert_(info[field] >= 0) + self.assertTrue(info[field] >= 0) if info['container_count'] == len(self.env.containers): break @@ -192,8 +194,8 @@ class TestAccount(Base): for format_type in ['json', 'xml']: for a in self.env.account.containers( parms={'format': format_type}): - self.assert_(a['count'] >= 0) - self.assert_(a['bytes'] >= 0) + self.assertTrue(a['count'] >= 0) + self.assertTrue(a['bytes'] >= 0) headers = dict(self.env.conn.response.getheaders()) if format_type == 'json': @@ -209,7 +211,7 @@ class TestAccount(Base): p = {'limit': l} if l <= limit: - self.assert_(len(self.env.account.containers(parms=p)) <= l) + self.assertTrue(len(self.env.account.containers(parms=p)) <= l) self.assert_status(200) else: self.assertRaises(ResponseError, @@ -256,11 +258,11 @@ class TestAccount(Base): parms={'format': format_type, 'marker': marker, 'limit': limit}) - self.assert_(len(containers) <= limit) + self.assertTrue(len(containers) <= limit) if containers: if isinstance(containers[0], dict): containers = [x['name'] for x in containers] - self.assert_(locale.strcoll(containers[0], marker) > 0) + self.assertTrue(locale.strcoll(containers[0], marker) > 0) def testContainersOrderedByName(self): for format_type in [None, 'json', 'xml']: @@ -283,15 +285,13 @@ class TestAccount(Base): conn.connection.request('GET', '/v1/' + quoted_hax, None, {}) resp = conn.connection.getresponse() resp_headers = dict(resp.getheaders()) - self.assertTrue('www-authenticate' in resp_headers, - 'www-authenticate not found in %s' % resp_headers) + self.assertIn('www-authenticate', resp_headers) actual = resp_headers['www-authenticate'] expected = 'Swift realm="%s"' % quoted_hax # other middleware e.g. auth_token may also set www-authenticate # headers in which case actual values will be a comma separated list. # check that expected value is among the actual values - self.assertTrue(expected in actual, - '%s not found in %s' % (expected, actual)) + self.assertIn(expected, actual) class TestAccountUTF8(Base2, TestAccount): @@ -314,7 +314,7 @@ class TestAccountNoContainers(Base): def testGetRequest(self): for format_type in [None, 'json', 'xml']: - self.assert_(not self.env.account.containers( + self.assertFalse(self.env.account.containers( parms={'format': format_type})) if format_type is None: @@ -363,57 +363,56 @@ class TestContainer(Base): set_up = False def testContainerNameLimit(self): - - raise SkipTest('SOF constraints middleware enforces constraints.') - + raise SkipTest('SOF constraints middleware enforces constraints.') limit = load_constraint('max_container_name_length') for l in (limit - 100, limit - 10, limit - 1, limit, limit + 1, limit + 10, limit + 100): cont = self.env.account.container('a' * l) if l <= limit: - self.assert_(cont.create(None, None)) + self.assertTrue(cont.create()) self.assert_status(201) else: - self.assert_(not cont.create(None, None)) + self.assertFalse(cont.create()) self.assert_status(400) def testFileThenContainerDelete(self): cont = self.env.account.container(Utils.create_name()) - self.assert_(cont.create(None, None)) + self.assertTrue(cont.create()) file_item = cont.file(Utils.create_name()) - self.assert_(file_item.write_random()) + self.assertTrue(file_item.write_random()) - self.assert_(file_item.delete()) + self.assertTrue(file_item.delete()) self.assert_status(204) - self.assert_(file_item.name not in cont.files()) + self.assertNotIn(file_item.name, cont.files()) - self.assert_(cont.delete()) + self.assertTrue(cont.delete()) self.assert_status(204) - self.assert_(cont.name not in self.env.account.containers()) + self.assertNotIn(cont.name, self.env.account.containers()) def testFileListingLimitMarkerPrefix(self): cont = self.env.account.container(Utils.create_name()) - self.assert_(cont.create(None, None)) + self.assertTrue(cont.create()) - files = sorted([Utils.create_name() for x in xrange(10)]) + files = sorted([Utils.create_name() for x in range(10)]) for f in files: file_item = cont.file(f) - self.assert_(file_item.write_random()) + self.assertTrue(file_item.write_random()) - for i in xrange(len(files)): + for i in range(len(files)): f = files[i] - for j in xrange(1, len(files) - i): - self.assert_(cont.files(parms={'limit': j, 'marker': f}) == - files[i + 1: i + j + 1]) - self.assert_(cont.files(parms={'marker': f}) == files[i + 1:]) - self.assert_(cont.files(parms={'marker': f, 'prefix': f}) == []) - self.assert_(cont.files(parms={'prefix': f}) == [f]) + for j in range(1, len(files) - i): + self.assertTrue( + cont.files(parms={'limit': j, 'marker': f}) == + files[i + 1: i + j + 1]) + self.assertTrue(cont.files(parms={'marker': f}) == files[i + 1:]) + self.assertTrue(cont.files(parms={'marker': f, 'prefix': f}) == []) + self.assertTrue(cont.files(parms={'prefix': f}) == [f]) def testPrefixAndLimit(self): load_constraint('container_listing_limit') cont = self.env.account.container(Utils.create_name()) - self.assert_(cont.create(None, None)) + self.assertTrue(cont.create()) prefix_file_count = 10 limit_count = 2 @@ -440,13 +439,41 @@ class TestContainer(Base): self.assertEqual(len(files), limit_count) for file_item in files: - self.assert_(file_item.startswith(prefix)) + self.assertTrue(file_item.startswith(prefix)) + + def testListDelimiter(self): + cont = self.env.account.container(Utils.create_name()) + self.assertTrue(cont.create()) + + delimiter = '-' + files = ['test', delimiter.join(['test', 'bar']), + delimiter.join(['test', 'foo'])] + for f in files: + file_item = cont.file(f) + self.assertTrue(file_item.write_random()) + + results = cont.files() + results = cont.files(parms={'delimiter': delimiter}) + self.assertEqual(results, ['test', 'test-']) + + def testListDelimiterAndPrefix(self): + cont = self.env.account.container(Utils.create_name()) + self.assertTrue(cont.create()) + + delimiter = 'a' + files = ['bar', 'bazar'] + for f in files: + file_item = cont.file(f) + self.assertTrue(file_item.write_random()) + + results = cont.files(parms={'delimiter': delimiter, 'prefix': 'ba'}) + self.assertEqual(results, ['bar', 'baza']) def testCreate(self): cont = self.env.account.container(Utils.create_name()) - self.assert_(cont.create(None, None)) + self.assertTrue(cont.create()) self.assert_status(201) - self.assert_(cont.name in self.env.account.containers()) + self.assertIn(cont.name, self.env.account.containers()) def testContainerFileListOnContainerThatDoesNotExist(self): for format_type in [None, 'json', 'xml']: @@ -459,13 +486,13 @@ class TestContainer(Base): valid_utf8 = Utils.create_utf8_name() invalid_utf8 = valid_utf8[::-1] container = self.env.account.container(valid_utf8) - self.assert_(container.create(None, None)) - self.assert_(container.name in self.env.account.containers()) + self.assertTrue(container.create(cfg={'no_path_quote': True})) + self.assertIn(container.name, self.env.account.containers()) self.assertEqual(container.files(), []) - self.assert_(container.delete()) + self.assertTrue(container.delete()) container = self.env.account.container(invalid_utf8) - self.assert_(not container.create(None, None)) + self.assertFalse(container.create(cfg={'no_path_quote': True})) self.assert_status(412) self.assertRaises(ResponseError, container.files, cfg={'no_path_quote': True}) @@ -473,9 +500,9 @@ class TestContainer(Base): def testCreateOnExisting(self): cont = self.env.account.container(Utils.create_name()) - self.assert_(cont.create(None, None)) + self.assertTrue(cont.create()) self.assert_status(201) - self.assert_(cont.create(None, None)) + self.assertTrue(cont.create()) self.assert_status(202) def testSlashInName(self): @@ -491,31 +518,31 @@ class TestContainer(Base): cont_name = cont_name.encode('utf-8') cont = self.env.account.container(cont_name) - self.assert_(not cont.create(None, None), - 'created container with name %s' % (cont_name)) + self.assertFalse(cont.create(cfg={'no_path_quote': True}), + 'created container with name %s' % (cont_name)) self.assert_status(404) - self.assert_(cont.name not in self.env.account.containers()) + self.assertNotIn(cont.name, self.env.account.containers()) def testDelete(self): cont = self.env.account.container(Utils.create_name()) - self.assert_(cont.create(None, None)) + self.assertTrue(cont.create()) self.assert_status(201) - self.assert_(cont.delete()) + self.assertTrue(cont.delete()) self.assert_status(204) - self.assert_(cont.name not in self.env.account.containers()) + self.assertNotIn(cont.name, self.env.account.containers()) def testDeleteOnContainerThatDoesNotExist(self): cont = self.env.account.container(Utils.create_name()) - self.assert_(not cont.delete()) + self.assertFalse(cont.delete()) self.assert_status(404) def testDeleteOnContainerWithFiles(self): cont = self.env.account.container(Utils.create_name()) - self.assert_(cont.create(None, None)) + self.assertTrue(cont.create()) file_item = cont.file(Utils.create_name()) file_item.write_random(self.env.file_size) - self.assert_(file_item.name in cont.files()) - self.assert_(not cont.delete()) + self.assertIn(file_item.name, cont.files()) + self.assertFalse(cont.delete()) self.assert_status(409) def testFileCreateInContainerThatDoesNotExist(self): @@ -547,10 +574,10 @@ class TestContainer(Base): files = [x['name'] for x in files] for file_item in self.env.files: - self.assert_(file_item in files) + self.assertIn(file_item, files) for file_item in files: - self.assert_(file_item in self.env.files) + self.assertIn(file_item, self.env.files) def testMarkerLimitFileList(self): for format_type in [None, 'json', 'xml']: @@ -567,11 +594,11 @@ class TestContainer(Base): if isinstance(files[0], dict): files = [x['name'] for x in files] - self.assert_(len(files) <= limit) + self.assertTrue(len(files) <= limit) if files: if isinstance(files[0], dict): files = [x['name'] for x in files] - self.assert_(locale.strcoll(files[0], marker) > 0) + self.assertTrue(locale.strcoll(files[0], marker) > 0) def testFileOrder(self): for format_type in [None, 'json', 'xml']: @@ -600,19 +627,19 @@ class TestContainer(Base): def testTooLongName(self): cont = self.env.account.container('x' * 257) - self.assert_(not cont.create(None, None), - 'created container with name %s' % (cont.name)) + self.assertFalse(cont.create(), + 'created container with name %s' % (cont.name)) self.assert_status(400) def testContainerExistenceCachingProblem(self): cont = self.env.account.container(Utils.create_name()) self.assertRaises(ResponseError, cont.files) - self.assert_(cont.create(None, None)) + self.assertTrue(cont.create()) cont.files() cont = self.env.account.container(Utils.create_name()) self.assertRaises(ResponseError, cont.files) - self.assert_(cont.create(None, None)) + self.assertTrue(cont.create()) file_item = cont.file(Utils.create_name()) file_item.write_random() @@ -624,7 +651,7 @@ class TestContainerUTF8(Base2, TestContainer): class TestContainerPathsEnv(object): @classmethod def setUp(cls): - raise SkipTest('Objects ending in / are not supported') + raise SkipTest('Objects ending in / are not supported') cls.conn = Connection(tf.config) cls.conn.authenticate() cls.account = Account(cls.conn, tf.config.get('account', @@ -711,7 +738,7 @@ class TestContainerPaths(Base): raise ValueError('too deep recursion') for file_item in self.env.container.files(parms={'path': path}): - self.assert_(file_item.startswith(path)) + self.assertTrue(file_item.startswith(path)) if file_item.endswith('/'): recurse_path(file_item, count + 1) found_dirs.append(file_item) @@ -721,28 +748,28 @@ class TestContainerPaths(Base): recurse_path('') for file_item in self.env.stored_files: if file_item.startswith('/'): - self.assert_(file_item not in found_dirs) - self.assert_(file_item not in found_files) + self.assertNotIn(file_item, found_dirs) + self.assertNotIn(file_item, found_files) elif file_item.endswith('/'): - self.assert_(file_item in found_dirs) - self.assert_(file_item not in found_files) + self.assertIn(file_item, found_dirs) + self.assertNotIn(file_item, found_files) else: - self.assert_(file_item in found_files) - self.assert_(file_item not in found_dirs) + self.assertIn(file_item, found_files) + self.assertNotIn(file_item, found_dirs) found_files = [] found_dirs = [] recurse_path('/') for file_item in self.env.stored_files: if not file_item.startswith('/'): - self.assert_(file_item not in found_dirs) - self.assert_(file_item not in found_files) + self.assertNotIn(file_item, found_dirs) + self.assertNotIn(file_item, found_files) elif file_item.endswith('/'): - self.assert_(file_item in found_dirs) - self.assert_(file_item not in found_files) + self.assertIn(file_item, found_dirs) + self.assertNotIn(file_item, found_files) else: - self.assert_(file_item in found_files) - self.assert_(file_item not in found_dirs) + self.assertIn(file_item, found_files) + self.assertNotIn(file_item, found_dirs) def testContainerListing(self): for format_type in (None, 'json', 'xml'): @@ -756,8 +783,8 @@ class TestContainerPaths(Base): for format_type in ('json', 'xml'): for file_item in self.env.container.files(parms={'format': format_type}): - self.assert_(int(file_item['bytes']) >= 0) - self.assert_('last_modified' in file_item) + self.assertTrue(int(file_item['bytes']) >= 0) + self.assertIn('last_modified', file_item) if file_item['name'].endswith('/'): self.assertEqual(file_item['content_type'], 'application/directory') @@ -856,7 +883,7 @@ class TestFile(Base): file_item.sync_metadata(metadata) dest_cont = self.env.account.container(Utils.create_name()) - self.assert_(dest_cont.create(None, None)) + self.assertTrue(dest_cont.create()) # copy both from within and across containers for cont in (self.env.container, dest_cont): @@ -867,13 +894,13 @@ class TestFile(Base): file_item = self.env.container.file(source_filename) file_item.copy('%s%s' % (prefix, cont), dest_filename) - self.assert_(dest_filename in cont.files()) + self.assertIn(dest_filename, cont.files()) file_item = cont.file(dest_filename) - self.assert_(data == file_item.read()) - self.assert_(file_item.initialize()) - self.assert_(metadata == file_item.metadata) + self.assertTrue(data == file_item.read()) + self.assertTrue(file_item.initialize()) + self.assertTrue(metadata == file_item.metadata) def testCopyAccount(self): # makes sure to test encoded characters @@ -886,7 +913,7 @@ class TestFile(Base): file_item.sync_metadata(metadata) dest_cont = self.env.account.container(Utils.create_name()) - self.assert_(dest_cont.create(None, None)) + self.assertTrue(dest_cont.create()) acct = self.env.conn.account_name # copy both from within and across containers @@ -900,16 +927,18 @@ class TestFile(Base): '%s%s' % (prefix, cont), dest_filename) - self.assert_(dest_filename in cont.files()) + self.assertIn(dest_filename, cont.files()) file_item = cont.file(dest_filename) - self.assert_(data == file_item.read()) - self.assert_(file_item.initialize()) - self.assert_(metadata == file_item.metadata) + self.assertTrue(data == file_item.read()) + self.assertTrue(file_item.initialize()) + self.assertTrue(metadata == file_item.metadata) dest_cont = self.env.account2.container(Utils.create_name()) - self.assert_(dest_cont.create(None, None)) + self.assertTrue(dest_cont.create(hdrs={ + 'X-Container-Write': self.env.conn.user_acl + })) acct = self.env.conn2.account_name # copy both with and without initial slash @@ -921,13 +950,13 @@ class TestFile(Base): '%s%s' % (prefix, dest_cont), dest_filename) - self.assert_(dest_filename in dest_cont.files()) + self.assertIn(dest_filename, dest_cont.files()) file_item = dest_cont.file(dest_filename) - self.assert_(data == file_item.read()) - self.assert_(file_item.initialize()) - self.assert_(metadata == file_item.metadata) + self.assertTrue(data == file_item.read()) + self.assertTrue(file_item.initialize()) + self.assertTrue(metadata == file_item.metadata) def testCopy404s(self): source_filename = Utils.create_name() @@ -935,37 +964,38 @@ class TestFile(Base): file_item.write_random() dest_cont = self.env.account.container(Utils.create_name()) - self.assert_(dest_cont.create(None, None)) + self.assertTrue(dest_cont.create()) for prefix in ('', '/'): # invalid source container source_cont = self.env.account.container(Utils.create_name()) file_item = source_cont.file(source_filename) - self.assert_(not file_item.copy( + self.assertFalse(file_item.copy( '%s%s' % (prefix, self.env.container), Utils.create_name())) self.assert_status(404) - self.assert_(not file_item.copy('%s%s' % (prefix, dest_cont), - Utils.create_name())) + self.assertFalse(file_item.copy('%s%s' % (prefix, dest_cont), + Utils.create_name())) self.assert_status(404) # invalid source object file_item = self.env.container.file(Utils.create_name()) - self.assert_(not file_item.copy( + self.assertFalse(file_item.copy( '%s%s' % (prefix, self.env.container), Utils.create_name())) self.assert_status(404) - self.assert_(not file_item.copy('%s%s' % (prefix, dest_cont), + self.assertFalse(file_item.copy('%s%s' % (prefix, dest_cont), Utils.create_name())) self.assert_status(404) # invalid destination container file_item = self.env.container.file(source_filename) - self.assert_(not file_item.copy( - '%s%s' % (prefix, Utils.create_name()), - Utils.create_name())) + self.assertTrue( + not file_item.copy( + '%s%s' % (prefix, Utils.create_name()), + Utils.create_name())) def testCopyAccount404s(self): acct = self.env.conn.account_name @@ -975,16 +1005,21 @@ class TestFile(Base): file_item.write_random() dest_cont = self.env.account.container(Utils.create_name()) - self.assert_(dest_cont.create(None, None)) + self.assertTrue(dest_cont.create(hdrs={ + 'X-Container-Read': self.env.conn2.user_acl + })) dest_cont2 = self.env.account2.container(Utils.create_name()) - self.assert_(dest_cont2.create(None, None)) + self.assertTrue(dest_cont2.create(hdrs={ + 'X-Container-Write': self.env.conn.user_acl, + 'X-Container-Read': self.env.conn.user_acl + })) for acct, cont in ((acct, dest_cont), (acct2, dest_cont2)): for prefix in ('', '/'): # invalid source container source_cont = self.env.account.container(Utils.create_name()) file_item = source_cont.file(source_filename) - self.assert_(not file_item.copy_account( + self.assertFalse(file_item.copy_account( acct, '%s%s' % (prefix, self.env.container), Utils.create_name())) @@ -995,7 +1030,7 @@ class TestFile(Base): else: self.assert_status(404) - self.assert_(not file_item.copy_account( + self.assertFalse(file_item.copy_account( acct, '%s%s' % (prefix, cont), Utils.create_name())) @@ -1003,7 +1038,7 @@ class TestFile(Base): # invalid source object file_item = self.env.container.file(Utils.create_name()) - self.assert_(not file_item.copy_account( + self.assertFalse(file_item.copy_account( acct, '%s%s' % (prefix, self.env.container), Utils.create_name())) @@ -1014,7 +1049,7 @@ class TestFile(Base): else: self.assert_status(404) - self.assert_(not file_item.copy_account( + self.assertFalse(file_item.copy_account( acct, '%s%s' % (prefix, cont), Utils.create_name())) @@ -1022,7 +1057,7 @@ class TestFile(Base): # invalid destination container file_item = self.env.container.file(source_filename) - self.assert_(not file_item.copy_account( + self.assertFalse(file_item.copy_account( acct, '%s%s' % (prefix, Utils.create_name()), Utils.create_name())) @@ -1039,9 +1074,9 @@ class TestFile(Base): file_item.write_random() file_item = self.env.container.file(source_filename) - self.assert_(not file_item.copy(Utils.create_name(), - Utils.create_name(), - cfg={'no_destination': True})) + self.assertFalse(file_item.copy(Utils.create_name(), + Utils.create_name(), + cfg={'no_destination': True})) self.assert_status(412) def testCopyDestinationSlashProblems(self): @@ -1050,9 +1085,9 @@ class TestFile(Base): file_item.write_random() # no slash - self.assert_(not file_item.copy(Utils.create_name(), - Utils.create_name(), - cfg={'destination': Utils.create_name()})) + self.assertFalse(file_item.copy(Utils.create_name(), + Utils.create_name(), + cfg={'destination': Utils.create_name()})) self.assert_status(412) def testCopyFromHeader(self): @@ -1067,7 +1102,7 @@ class TestFile(Base): data = file_item.write_random() dest_cont = self.env.account.container(Utils.create_name()) - self.assert_(dest_cont.create(None, None)) + self.assertTrue(dest_cont.create()) # copy both from within and across containers for cont in (self.env.container, dest_cont): @@ -1079,18 +1114,20 @@ class TestFile(Base): file_item.write(hdrs={'X-Copy-From': '%s%s/%s' % ( prefix, self.env.container.name, source_filename)}) - self.assert_(dest_filename in cont.files()) + self.assertIn(dest_filename, cont.files()) file_item = cont.file(dest_filename) - self.assert_(data == file_item.read()) - self.assert_(file_item.initialize()) - self.assert_(metadata == file_item.metadata) + self.assertTrue(data == file_item.read()) + self.assertTrue(file_item.initialize()) + self.assertTrue(metadata == file_item.metadata) def testCopyFromAccountHeader(self): acct = self.env.conn.account_name src_cont = self.env.account.container(Utils.create_name()) - self.assert_(src_cont.create(None, None)) + self.assertTrue(src_cont.create(hdrs={ + 'X-Container-Read': self.env.conn2.user_acl + })) source_filename = Utils.create_name() file_item = src_cont.file(source_filename) @@ -1102,9 +1139,11 @@ class TestFile(Base): data = file_item.write_random() dest_cont = self.env.account.container(Utils.create_name()) - self.assert_(dest_cont.create(None, None)) + self.assertTrue(dest_cont.create()) dest_cont2 = self.env.account2.container(Utils.create_name()) - self.assert_(dest_cont2.create(None, None)) + self.assertTrue(dest_cont2.create(hdrs={ + 'X-Container-Write': self.env.conn.user_acl + })) for cont in (src_cont, dest_cont, dest_cont2): # copy both with and without initial slash @@ -1118,13 +1157,13 @@ class TestFile(Base): src_cont.name, source_filename)}) - self.assert_(dest_filename in cont.files()) + self.assertIn(dest_filename, cont.files()) file_item = cont.file(dest_filename) - self.assert_(data == file_item.read()) - self.assert_(file_item.initialize()) - self.assert_(metadata == file_item.metadata) + self.assertTrue(data == file_item.read()) + self.assertTrue(file_item.initialize()) + self.assertTrue(metadata == file_item.metadata) def testCopyFromHeader404s(self): source_filename = Utils.create_name() @@ -1134,38 +1173,41 @@ class TestFile(Base): for prefix in ('', '/'): # invalid source container file_item = self.env.container.file(Utils.create_name()) + copy_from = ('%s%s/%s' + % (prefix, Utils.create_name(), source_filename)) self.assertRaises(ResponseError, file_item.write, - hdrs={'X-Copy-From': '%s%s/%s' % - (prefix, - Utils.create_name(), source_filename)}) + hdrs={'X-Copy-From': copy_from}) self.assert_status(404) # invalid source object + copy_from = ('%s%s/%s' + % (prefix, self.env.container.name, + Utils.create_name())) file_item = self.env.container.file(Utils.create_name()) self.assertRaises(ResponseError, file_item.write, - hdrs={'X-Copy-From': '%s%s/%s' % - (prefix, - self.env.container.name, Utils.create_name())}) + hdrs={'X-Copy-From': copy_from}) self.assert_status(404) # invalid destination container dest_cont = self.env.account.container(Utils.create_name()) file_item = dest_cont.file(Utils.create_name()) + copy_from = ('%s%s/%s' + % (prefix, self.env.container.name, source_filename)) self.assertRaises(ResponseError, file_item.write, - hdrs={'X-Copy-From': '%s%s/%s' % - (prefix, - self.env.container.name, source_filename)}) + hdrs={'X-Copy-From': copy_from}) self.assert_status(404) def testCopyFromAccountHeader404s(self): acct = self.env.conn2.account_name src_cont = self.env.account2.container(Utils.create_name()) - self.assert_(src_cont.create(None, None)) + self.assertTrue(src_cont.create(hdrs={ + 'X-Container-Read': self.env.conn.user_acl + })) source_filename = Utils.create_name() file_item = src_cont.file(source_filename) file_item.write_random() dest_cont = self.env.account.container(Utils.create_name()) - self.assert_(dest_cont.create(None, None)) + self.assertTrue(dest_cont.create()) for prefix in ('', '/'): # invalid source container @@ -1202,16 +1244,14 @@ class TestFile(Base): self.assert_status(404) def testNameLimit(self): - - raise SkipTest('SOF constraints middleware enforces constraints.') - + raise SkipTest('SOF constraints middleware enforces constraints.') limit = load_constraint('max_object_name_length') for l in (1, 10, limit / 2, limit - 1, limit, limit + 1, limit * 2): file_item = self.env.container.file('a' * l) if l <= limit: - self.assert_(file_item.write()) + self.assertTrue(file_item.write()) self.assert_status(201) else: self.assertRaises(ResponseError, file_item.write) @@ -1226,16 +1266,16 @@ class TestFile(Base): file_name = Utils.create_name(6) + '?' + Utils.create_name(6) file_item = self.env.container.file(file_name) - self.assert_(file_item.write(cfg={'no_path_quote': True})) - self.assert_(file_name not in self.env.container.files()) - self.assert_(file_name.split('?')[0] in self.env.container.files()) + self.assertTrue(file_item.write(cfg={'no_path_quote': True})) + self.assertNotIn(file_name, self.env.container.files()) + self.assertIn(file_name.split('?')[0], self.env.container.files()) def testDeleteThen404s(self): file_item = self.env.container.file(Utils.create_name()) - self.assert_(file_item.write_random()) + self.assertTrue(file_item.write_random()) self.assert_status(201) - self.assert_(file_item.delete()) + self.assertTrue(file_item.delete()) self.assert_status(204) file_item.metadata = {Utils.create_ascii_name(): Utils.create_name()} @@ -1254,7 +1294,7 @@ class TestFile(Base): self.assert_status(400) def testMetadataNumberLimit(self): - raise SkipTest("Bad test") + raise SkipTest("Bad test") # TODO(ppai): Fix it in upstream swift first # Refer to comments below number_limit = load_constraint('max_meta_count') @@ -1269,12 +1309,12 @@ class TestFile(Base): metadata = {} while len(metadata.keys()) < i: key = Utils.create_ascii_name() - # The following line returns a valid utf8 byte sequence + # The following line returns a valid utf8 byte sequence val = Utils.create_name() if len(key) > j: key = key[:j] - # This slicing done below can make the 'utf8' byte + # This slicing done below can make the 'utf8' byte # sequence invalid and hence it cannot be decoded. val = val[:j] @@ -1285,15 +1325,15 @@ class TestFile(Base): file_item.metadata = metadata if i <= number_limit: - self.assert_(file_item.write()) + self.assertTrue(file_item.write()) self.assert_status(201) - self.assert_(file_item.sync_metadata()) + self.assertTrue(file_item.sync_metadata()) self.assert_status((201, 202)) else: self.assertRaises(ResponseError, file_item.write) self.assert_status(400) file_item.metadata = {} - self.assert_(file_item.write()) + self.assertTrue(file_item.write()) self.assert_status(201) file_item.metadata = metadata self.assertRaises(ResponseError, file_item.sync_metadata) @@ -1304,7 +1344,7 @@ class TestFile(Base): 'zip': 'application/zip'} container = self.env.account.container(Utils.create_name()) - self.assert_(container.create(None, None)) + self.assertTrue(container.create()) for i in file_types.keys(): file_item = container.file(Utils.create_name() + '.' + i) @@ -1330,8 +1370,9 @@ class TestFile(Base): for i in range(0, file_length, range_size): range_string = 'bytes=%d-%d' % (i, i + range_size - 1) hdrs = {'Range': range_string} - self.assert_(data[i: i + range_size] == file_item.read(hdrs=hdrs), - range_string) + self.assertTrue( + data[i: i + range_size] == file_item.read(hdrs=hdrs), + range_string) range_string = 'bytes=-%d' % (i) hdrs = {'Range': range_string} @@ -1348,8 +1389,9 @@ class TestFile(Base): range_string = 'bytes=%d-' % (i) hdrs = {'Range': range_string} - self.assert_(file_item.read(hdrs=hdrs) == data[i - file_length:], - range_string) + self.assertTrue( + file_item.read(hdrs=hdrs) == data[i - file_length:], + range_string) range_string = 'bytes=%d-%d' % (file_length + 1000, file_length + 2000) hdrs = {'Range': range_string} @@ -1358,20 +1400,21 @@ class TestFile(Base): range_string = 'bytes=%d-%d' % (file_length - 1000, file_length + 2000) hdrs = {'Range': range_string} - self.assert_(file_item.read(hdrs=hdrs) == data[-1000:], range_string) + self.assertTrue( + file_item.read(hdrs=hdrs) == data[-1000:], range_string) hdrs = {'Range': '0-4'} - self.assert_(file_item.read(hdrs=hdrs) == data, range_string) + self.assertTrue(file_item.read(hdrs=hdrs) == data, range_string) # RFC 2616 14.35.1 # "If the entity is shorter than the specified suffix-length, the # entire entity-body is used." range_string = 'bytes=-%d' % (file_length + 10) hdrs = {'Range': range_string} - self.assert_(file_item.read(hdrs=hdrs) == data, range_string) + self.assertTrue(file_item.read(hdrs=hdrs) == data, range_string) def testRangedGetsWithLWSinHeader(self): - #Skip this test until webob 1.2 can tolerate LWS in Range header. + # Skip this test until webob 1.2 can tolerate LWS in Range header. file_length = 10000 file_item = self.env.container.file(Utils.create_name()) data = file_item.write_random(file_length) @@ -1379,7 +1422,7 @@ class TestFile(Base): for r in ('BYTES=0-999', 'bytes = 0-999', 'BYTES = 0 - 999', 'bytes = 0 - 999', 'bytes=0 - 999', 'bytes=0-999 '): - self.assert_(file_item.read(hdrs={'Range': r}) == data[0:1000]) + self.assertTrue(file_item.read(hdrs={'Range': r}) == data[0:1000]) def testFileSizeLimit(self): limit = load_constraint('max_file_size') @@ -1400,8 +1443,8 @@ class TestFile(Base): file_item = self.env.container.file(Utils.create_name()) if i <= limit: - self.assert_(timeout(tsecs, file_item.write, - cfg={'set_content_length': i})) + self.assertTrue(timeout(tsecs, file_item.write, + cfg={'set_content_length': i})) else: self.assertRaises(ResponseError, timeout, tsecs, file_item.write, @@ -1417,9 +1460,9 @@ class TestFile(Base): file_item = self.env.container.file(Utils.create_name()) file_item.write_random(self.env.file_size) - self.assert_(file_item.name in self.env.container.files()) - self.assert_(file_item.delete()) - self.assert_(file_item.name not in self.env.container.files()) + self.assertIn(file_item.name, self.env.container.files()) + self.assertTrue(file_item.delete()) + self.assertNotIn(file_item.name, self.env.container.files()) def testBadHeaders(self): file_length = 100 @@ -1446,15 +1489,16 @@ class TestFile(Base): self.assert_status(501) # bad request types - #for req in ('LICK', 'GETorHEAD_base', 'container_info', - # 'best_response'): + # for req in ('LICK', 'GETorHEAD_base', 'container_info', + # 'best_response'): for req in ('LICK', 'GETorHEAD_base'): self.env.account.conn.make_request(req) self.assert_status(405) # bad range headers - self.assert_(len(file_item.read(hdrs={'Range': 'parsecs=8-12'})) == - file_length) + self.assertTrue( + len(file_item.read(hdrs={'Range': 'parsecs=8-12'})) == + file_length) self.assert_status(200) def testMetadataLengthLimits(self): @@ -1471,14 +1515,14 @@ class TestFile(Base): file_item.metadata = metadata if l[0] <= key_limit and l[1] <= value_limit: - self.assert_(file_item.write()) + self.assertTrue(file_item.write()) self.assert_status(201) - self.assert_(file_item.sync_metadata()) + self.assertTrue(file_item.sync_metadata()) else: self.assertRaises(ResponseError, file_item.write) self.assert_status(400) file_item.metadata = {} - self.assert_(file_item.write()) + self.assertTrue(file_item.write()) self.assert_status(201) file_item.metadata = metadata self.assertRaises(ResponseError, file_item.sync_metadata) @@ -1495,7 +1539,7 @@ class TestFile(Base): file_item = self.env.container.file(Utils.create_name()) data = file_item.write_random() self.assert_status(201) - self.assert_(data == file_item.read()) + self.assertTrue(data == file_item.read()) self.assert_status(200) def testHead(self): @@ -1515,7 +1559,7 @@ class TestFile(Base): self.assertEqual(info['content_length'], self.env.file_size) self.assertEqual(info['etag'], md5) self.assertEqual(info['content_type'], content_type) - self.assert_('last_modified' in info) + self.assertIn('last_modified', info) def testDeleteOfFileThatDoesNotExist(self): # in container that exists @@ -1551,11 +1595,11 @@ class TestFile(Base): metadata[Utils.create_ascii_name()] = Utils.create_name() file_item.metadata = metadata - self.assert_(file_item.sync_metadata()) + self.assertTrue(file_item.sync_metadata()) self.assert_status((201, 202)) file_item = self.env.container.file(file_item.name) - self.assert_(file_item.initialize()) + self.assertTrue(file_item.initialize()) self.assert_status(200) self.assertEqual(file_item.metadata, metadata) @@ -1609,13 +1653,13 @@ class TestFile(Base): file_item.write_random(self.env.file_size) file_item = self.env.container.file(file_item.name) - self.assert_(file_item.initialize()) + self.assertTrue(file_item.initialize()) self.assert_status(200) self.assertEqual(file_item.metadata, metadata) def testSerialization(self): container = self.env.account.container(Utils.create_name()) - self.assert_(container.create(None, None)) + self.assertTrue(container.create()) files = [] for i in (0, 1, 10, 100, 1000, 10000): @@ -1657,8 +1701,9 @@ class TestFile(Base): f[format_type] = True found = True - self.assert_(found, 'Unexpected file %s found in ' - '%s listing' % (file_item['name'], format_type)) + self.assertTrue( + found, 'Unexpected file %s found in ' + '%s listing' % (file_item['name'], format_type)) headers = dict(self.env.conn.response.getheaders()) if format_type == 'json': @@ -1670,13 +1715,15 @@ class TestFile(Base): lm_diff = max([f['last_modified'] for f in files]) -\ min([f['last_modified'] for f in files]) - self.assert_(lm_diff < write_time + 1, 'Diff in last ' - 'modified times should be less than time to write files') + self.assertTrue( + lm_diff < write_time + 1, 'Diff in last ' + 'modified times should be less than time to write files') for f in files: for format_type in ['json', 'xml']: - self.assert_(f[format_type], 'File %s not found in %s listing' - % (f['name'], format_type)) + self.assertTrue( + f[format_type], 'File %s not found in %s listing' + % (f['name'], format_type)) def testStackedOverwrite(self): file_item = self.env.container.file(Utils.create_name()) @@ -1685,7 +1732,7 @@ class TestFile(Base): data = file_item.write_random(512) file_item.write(data) - self.assert_(file_item.read() == data) + self.assertTrue(file_item.read() == data) def testTooLongName(self): file_item = self.env.container.file('x' * 1025) @@ -1695,18 +1742,18 @@ class TestFile(Base): def testZeroByteFile(self): file_item = self.env.container.file(Utils.create_name()) - self.assert_(file_item.write('')) - self.assert_(file_item.name in self.env.container.files()) - self.assert_(file_item.read() == '') + self.assertTrue(file_item.write('')) + self.assertIn(file_item.name, self.env.container.files()) + self.assertTrue(file_item.read() == '') def testEtagResponse(self): file_item = self.env.container.file(Utils.create_name()) - data = StringIO.StringIO(file_item.write_random(512)) + data = six.StringIO(file_item.write_random(512)) etag = File.compute_md5sum(data) headers = dict(self.env.conn.response.getheaders()) - self.assert_('etag' in headers.keys()) + self.assertIn('etag', headers.keys()) header_etag = headers['etag'].strip('"') self.assertEqual(etag, header_etag) @@ -1731,8 +1778,8 @@ class TestFile(Base): for j in chunks(data, i): file_item.chunked_write(j) - self.assert_(file_item.chunked_write()) - self.assert_(data == file_item.read()) + self.assertTrue(file_item.chunked_write()) + self.assertTrue(data == file_item.read()) info = file_item.info() self.assertEqual(etag, info['etag']) @@ -1850,7 +1897,7 @@ class TestDlo(Base): file_contents, "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff") # The copied object must not have X-Object-Manifest - self.assertTrue("x_object_manifest" not in file_item.info()) + self.assertNotIn("x_object_manifest", file_item.info()) def test_copy_account(self): # dlo use same account and same container only @@ -1876,7 +1923,7 @@ class TestDlo(Base): file_contents, "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff") # The copied object must not have X-Object-Manifest - self.assertTrue("x_object_manifest" not in file_item.info()) + self.assertNotIn("x_object_manifest", file_item.info()) def test_copy_manifest(self): # Copying the manifest with multipart-manifest=get query string @@ -1988,7 +2035,7 @@ class TestFileComparison(Base): def testIfMatch(self): for file_item in self.env.files: hdrs = {'If-Match': file_item.md5} - self.assert_(file_item.read(hdrs=hdrs)) + self.assertTrue(file_item.read(hdrs=hdrs)) hdrs = {'If-Match': 'bogus'} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) @@ -1997,7 +2044,7 @@ class TestFileComparison(Base): def testIfNoneMatch(self): for file_item in self.env.files: hdrs = {'If-None-Match': 'bogus'} - self.assert_(file_item.read(hdrs=hdrs)) + self.assertTrue(file_item.read(hdrs=hdrs)) hdrs = {'If-None-Match': file_item.md5} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) @@ -2006,8 +2053,8 @@ class TestFileComparison(Base): def testIfModifiedSince(self): for file_item in self.env.files: hdrs = {'If-Modified-Since': self.env.time_old_f1} - self.assert_(file_item.read(hdrs=hdrs)) - self.assert_(file_item.info(hdrs=hdrs)) + self.assertTrue(file_item.read(hdrs=hdrs)) + self.assertTrue(file_item.info(hdrs=hdrs)) hdrs = {'If-Modified-Since': self.env.time_new} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) @@ -2018,8 +2065,8 @@ class TestFileComparison(Base): def testIfUnmodifiedSince(self): for file_item in self.env.files: hdrs = {'If-Unmodified-Since': self.env.time_new} - self.assert_(file_item.read(hdrs=hdrs)) - self.assert_(file_item.info(hdrs=hdrs)) + self.assertTrue(file_item.read(hdrs=hdrs)) + self.assertTrue(file_item.info(hdrs=hdrs)) hdrs = {'If-Unmodified-Since': self.env.time_old_f2} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) @@ -2031,7 +2078,7 @@ class TestFileComparison(Base): for file_item in self.env.files: hdrs = {'If-Match': file_item.md5, 'If-Unmodified-Since': self.env.time_new} - self.assert_(file_item.read(hdrs=hdrs)) + self.assertTrue(file_item.read(hdrs=hdrs)) hdrs = {'If-Match': 'bogus', 'If-Unmodified-Since': self.env.time_new} @@ -2054,7 +2101,7 @@ class TestFileComparison(Base): file = self.env.container.file(file_name) info = file.info() - self.assert_('last_modified' in info) + self.assertIn('last_modified', info) last_modified = info['last_modified'] self.assertEqual(put_last_modified, info['last_modified']) @@ -2063,7 +2110,7 @@ class TestFileComparison(Base): self.assert_status(304) hdrs = {'If-Unmodified-Since': last_modified} - self.assert_(file.read(hdrs=hdrs)) + self.assertTrue(file.read(hdrs=hdrs)) class TestFileComparisonUTF8(Base2, TestFileComparison): @@ -2100,7 +2147,7 @@ class TestSloEnv(object): if not cls.container.create(None, None): raise ResponseError(cls.conn.response) - seg_info = {} + cls.seg_info = seg_info = {} for letter, size in (('a', 1024 * 1024), ('b', 1024 * 1024), ('c', 1024 * 1024), @@ -2151,6 +2198,68 @@ class TestSloEnv(object): 'manifest-bcd-submanifest')}, seg_info['seg_e']]), parms={'multipart-manifest': 'put'}) + abcde_submanifest_etag = hashlib.md5( + seg_info['seg_a']['etag'] + bcd_submanifest_etag + + seg_info['seg_e']['etag']).hexdigest() + abcde_submanifest_size = (seg_info['seg_a']['size_bytes'] + + seg_info['seg_b']['size_bytes'] + + seg_info['seg_c']['size_bytes'] + + seg_info['seg_d']['size_bytes'] + + seg_info['seg_e']['size_bytes']) + + file_item = cls.container.file("ranged-manifest") + file_item.write( + json.dumps([ + {'etag': abcde_submanifest_etag, + 'size_bytes': abcde_submanifest_size, + 'path': '/%s/%s' % (cls.container.name, + 'manifest-abcde-submanifest'), + 'range': '-1048578'}, # 'c' + ('d' * 2**20) + 'e' + {'etag': abcde_submanifest_etag, + 'size_bytes': abcde_submanifest_size, + 'path': '/%s/%s' % (cls.container.name, + 'manifest-abcde-submanifest'), + 'range': '524288-1572863'}, # 'a' * 2**19 + 'b' * 2**19 + {'etag': abcde_submanifest_etag, + 'size_bytes': abcde_submanifest_size, + 'path': '/%s/%s' % (cls.container.name, + 'manifest-abcde-submanifest'), + 'range': '3145727-3145728'}]), # 'cd' + parms={'multipart-manifest': 'put'}) + ranged_manifest_etag = hashlib.md5( + abcde_submanifest_etag + ':3145727-4194304;' + + abcde_submanifest_etag + ':524288-1572863;' + + abcde_submanifest_etag + ':3145727-3145728;').hexdigest() + ranged_manifest_size = 2 * 1024 * 1024 + 4 + + file_item = cls.container.file("ranged-submanifest") + file_item.write( + json.dumps([ + seg_info['seg_c'], + {'etag': ranged_manifest_etag, + 'size_bytes': ranged_manifest_size, + 'path': '/%s/%s' % (cls.container.name, + 'ranged-manifest')}, + {'etag': ranged_manifest_etag, + 'size_bytes': ranged_manifest_size, + 'path': '/%s/%s' % (cls.container.name, + 'ranged-manifest'), + 'range': '524289-1572865'}, + {'etag': ranged_manifest_etag, + 'size_bytes': ranged_manifest_size, + 'path': '/%s/%s' % (cls.container.name, + 'ranged-manifest'), + 'range': '-3'}]), + parms={'multipart-manifest': 'put'}) + + file_item = cls.container.file("manifest-db") + file_item.write( + json.dumps([ + {'path': seg_info['seg_d']['path'], 'etag': None, + 'size_bytes': None}, + {'path': seg_info['seg_b']['path'], 'etag': None, + 'size_bytes': None}, + ]), parms={'multipart-manifest': 'put'}) class TestSlo(Base): @@ -2187,6 +2296,39 @@ class TestSlo(Base): self.assertEqual('d', file_contents[-2]) self.assertEqual('e', file_contents[-1]) + def test_slo_get_ranged_manifest(self): + file_item = self.env.container.file('ranged-manifest') + grouped_file_contents = [ + (char, sum(1 for _char in grp)) + for char, grp in itertools.groupby(file_item.read())] + self.assertEqual([ + ('c', 1), + ('d', 1024 * 1024), + ('e', 1), + ('a', 512 * 1024), + ('b', 512 * 1024), + ('c', 1), + ('d', 1)], grouped_file_contents) + + def test_slo_get_ranged_submanifest(self): + file_item = self.env.container.file('ranged-submanifest') + grouped_file_contents = [ + (char, sum(1 for _char in grp)) + for char, grp in itertools.groupby(file_item.read())] + self.assertEqual([ + ('c', 1024 * 1024 + 1), + ('d', 1024 * 1024), + ('e', 1), + ('a', 512 * 1024), + ('b', 512 * 1024), + ('c', 1), + ('d', 512 * 1024 + 1), + ('e', 1), + ('a', 512 * 1024), + ('b', 1), + ('c', 1), + ('d', 1)], grouped_file_contents) + def test_slo_ranged_get(self): file_item = self.env.container.file('manifest-abcde') file_contents = file_item.read(size=1024 * 1024 + 2, @@ -2259,6 +2401,72 @@ class TestSlo(Base): else: self.fail("Expected ResponseError but didn't get it") + def test_slo_unspecified_etag(self): + file_item = self.env.container.file("manifest-a-unspecified-etag") + file_item.write( + json.dumps([{ + 'size_bytes': 1024 * 1024, + 'etag': None, + 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), + parms={'multipart-manifest': 'put'}) + self.assert_status(201) + + def test_slo_unspecified_size(self): + file_item = self.env.container.file("manifest-a-unspecified-size") + file_item.write( + json.dumps([{ + 'size_bytes': None, + 'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(), + 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), + parms={'multipart-manifest': 'put'}) + self.assert_status(201) + + def test_slo_missing_etag(self): + file_item = self.env.container.file("manifest-a-missing-etag") + try: + file_item.write( + json.dumps([{ + 'size_bytes': 1024 * 1024, + 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), + parms={'multipart-manifest': 'put'}) + except ResponseError as err: + self.assertEqual(400, err.status) + else: + self.fail("Expected ResponseError but didn't get it") + + def test_slo_missing_size(self): + file_item = self.env.container.file("manifest-a-missing-size") + try: + file_item.write( + json.dumps([{ + 'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(), + 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), + parms={'multipart-manifest': 'put'}) + except ResponseError as err: + self.assertEqual(400, err.status) + else: + self.fail("Expected ResponseError but didn't get it") + + def test_slo_overwrite_segment_with_manifest(self): + file_item = self.env.container.file("seg_b") + try: + file_item.write( + json.dumps([ + {'size_bytes': 1024 * 1024, + 'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(), + 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}, + {'size_bytes': 1024 * 1024, + 'etag': hashlib.md5('b' * 1024 * 1024).hexdigest(), + 'path': '/%s/%s' % (self.env.container.name, 'seg_b')}, + {'size_bytes': 1024 * 1024, + 'etag': hashlib.md5('c' * 1024 * 1024).hexdigest(), + 'path': '/%s/%s' % (self.env.container.name, 'seg_c')}]), + parms={'multipart-manifest': 'put'}) + except ResponseError as err: + self.assertEqual(409, err.status) + else: + self.fail("Expected ResponseError but didn't get it") + def test_slo_copy(self): file_item = self.env.container.file("manifest-abcde") file_item.copy(self.env.container.name, "copied-abcde") @@ -2280,7 +2488,9 @@ class TestSlo(Base): # copy to different account acct = self.env.conn2.account_name dest_cont = self.env.account2.container(Utils.create_name()) - self.assert_(dest_cont.create(None, None)) + self.assertTrue(dest_cont.create(hdrs={ + 'X-Container-Write': self.env.conn.user_acl + })) file_item = self.env.container.file("manifest-abcde") file_item.copy_account(acct, dest_cont, "copied-abcde") @@ -2319,7 +2529,9 @@ class TestSlo(Base): # different account acct = self.env.conn2.account_name dest_cont = self.env.account2.container(Utils.create_name()) - self.assert_(dest_cont.create(None, None)) + self.assertTrue(dest_cont.create(hdrs={ + 'X-Container-Write': self.env.conn.user_acl + })) file_item.copy_account(acct, dest_cont, "copied-abcde-manifest-only", @@ -2332,6 +2544,58 @@ class TestSlo(Base): except ValueError: self.fail("COPY didn't copy the manifest (invalid json on GET)") + def _make_manifest(self): + file_item = self.env.container.file("manifest-post") + seg_info = self.env.seg_info + file_item.write( + json.dumps([seg_info['seg_a'], seg_info['seg_b'], + seg_info['seg_c'], seg_info['seg_d'], + seg_info['seg_e']]), + parms={'multipart-manifest': 'put'}) + return file_item + + def test_slo_post_the_manifest_metadata_update(self): + file_item = self._make_manifest() + # sanity check, check the object is an SLO manifest + file_item.info() + file_item.header_fields([('slo', 'x-static-large-object')]) + + # POST a user metadata (i.e. x-object-meta-post) + file_item.sync_metadata({'post': 'update'}) + + updated = self.env.container.file("manifest-post") + updated.info() + updated.header_fields([('user-meta', 'x-object-meta-post')]) # sanity + updated.header_fields([('slo', 'x-static-large-object')]) + updated_contents = updated.read(parms={'multipart-manifest': 'get'}) + try: + json.loads(updated_contents) + except ValueError: + self.fail("Unexpected content on GET, expected a json body") + + def test_slo_post_the_manifest_metadata_update_with_qs(self): + # multipart-manifest query should be ignored on post + for verb in ('put', 'get', 'delete'): + file_item = self._make_manifest() + # sanity check, check the object is an SLO manifest + file_item.info() + file_item.header_fields([('slo', 'x-static-large-object')]) + # POST a user metadata (i.e. x-object-meta-post) + file_item.sync_metadata(metadata={'post': 'update'}, + parms={'multipart-manifest': verb}) + updated = self.env.container.file("manifest-post") + updated.info() + updated.header_fields( + [('user-meta', 'x-object-meta-post')]) # sanity + updated.header_fields([('slo', 'x-static-large-object')]) + updated_contents = updated.read( + parms={'multipart-manifest': 'get'}) + try: + json.loads(updated_contents) + except ValueError: + self.fail( + "Unexpected content on GET, expected a json body") + def test_slo_get_the_manifest(self): manifest = self.env.container.file("manifest-abcde") got_body = manifest.read(parms={'multipart-manifest': 'get'}) @@ -2343,6 +2607,30 @@ class TestSlo(Base): except ValueError: self.fail("GET with multipart-manifest=get got invalid json") + def test_slo_get_the_manifest_with_details_from_server(self): + manifest = self.env.container.file("manifest-db") + got_body = manifest.read(parms={'multipart-manifest': 'get'}) + + self.assertEqual('application/json; charset=utf-8', + manifest.content_type) + try: + value = json.loads(got_body) + except ValueError: + self.fail("GET with multipart-manifest=get got invalid json") + + self.assertEqual(len(value), 2) + self.assertEqual(value[0]['bytes'], 1024 * 1024) + self.assertEqual(value[0]['hash'], + hashlib.md5('d' * 1024 * 1024).hexdigest()) + self.assertEqual(value[0]['name'], + '/%s/seg_d' % self.env.container.name.decode("utf-8")) + + self.assertEqual(value[1]['bytes'], 1024 * 1024) + self.assertEqual(value[1]['hash'], + hashlib.md5('b' * 1024 * 1024).hexdigest()) + self.assertEqual(value[1]['name'], + '/%s/seg_b' % self.env.container.name.decode("utf-8")) + def test_slo_head_the_manifest(self): manifest = self.env.container.file("manifest-abcde") got_info = manifest.info(parms={'multipart-manifest': 'get'}) @@ -2405,7 +2693,7 @@ class TestObjectVersioningEnv(object): @classmethod def setUp(cls): cls.conn = Connection(tf.config) - cls.conn.authenticate() + cls.storage_url, cls.storage_token = cls.conn.authenticate() cls.account = Account(cls.conn, tf.config.get('account', tf.config['username'])) @@ -2434,6 +2722,30 @@ class TestObjectVersioningEnv(object): # if versioning is off, then X-Versions-Location won't persist cls.versioning_enabled = 'versions' in container_info + # setup another account to test ACLs + config2 = deepcopy(tf.config) + config2['account'] = tf.config['account2'] + config2['username'] = tf.config['username2'] + config2['password'] = tf.config['password2'] + cls.conn2 = Connection(config2) + cls.storage_url2, cls.storage_token2 = cls.conn2.authenticate() + cls.account2 = cls.conn2.get_account() + cls.account2.delete_containers() + + # setup another account with no access to anything to test ACLs + config3 = deepcopy(tf.config) + config3['account'] = tf.config['account'] + config3['username'] = tf.config['username3'] + config3['password'] = tf.config['password3'] + cls.conn3 = Connection(config3) + cls.storage_url3, cls.storage_token3 = cls.conn3.authenticate() + cls.account3 = cls.conn3.get_account() + + @classmethod + def tearDown(cls): + cls.account.delete_containers() + cls.account2.delete_containers() + class TestCrossPolicyObjectVersioningEnv(object): # tri-state: None initially, then True/False @@ -2456,14 +2768,14 @@ class TestCrossPolicyObjectVersioningEnv(object): cls.multiple_policies_enabled = True else: cls.multiple_policies_enabled = False - # We have to lie here that versioning is enabled. We actually - # don't know, but it does not matter. We know these tests cannot - # run without multiple policies present. If multiple policies are - # present, we won't be setting this field to any value, so it - # should all still work. - cls.versioning_enabled = True + cls.versioning_enabled = False return + if cls.versioning_enabled is None: + cls.versioning_enabled = 'versioned_writes' in cluster_info + if not cls.versioning_enabled: + return + policy = cls.policies.select() version_policy = cls.policies.exclude(name=policy['name']).select() @@ -2495,6 +2807,25 @@ class TestCrossPolicyObjectVersioningEnv(object): # if versioning is off, then X-Versions-Location won't persist cls.versioning_enabled = 'versions' in container_info + # setup another account to test ACLs + config2 = deepcopy(tf.config) + config2['account'] = tf.config['account2'] + config2['username'] = tf.config['username2'] + config2['password'] = tf.config['password2'] + cls.conn2 = Connection(config2) + cls.storage_url2, cls.storage_token2 = cls.conn2.authenticate() + cls.account2 = cls.conn2.get_account() + cls.account2.delete_containers() + + # setup another account with no access to anything to test ACLs + config3 = deepcopy(tf.config) + config3['account'] = tf.config['account'] + config3['username'] = tf.config['username3'] + config3['password'] = tf.config['password3'] + cls.conn3 = Connection(config3) + cls.storage_url3, cls.storage_token3 = cls.conn3.authenticate() + cls.account3 = cls.conn3.get_account() + class TestObjectVersioning(Base): env = TestObjectVersioningEnv @@ -2513,40 +2844,103 @@ class TestObjectVersioning(Base): def tearDown(self): super(TestObjectVersioning, self).tearDown() try: - # delete versions first! + # only delete files and not container + # as they were configured in self.env self.env.versions_container.delete_files() self.env.container.delete_files() except ResponseError: pass + def test_clear_version_option(self): + # sanity + self.assertEqual(self.env.container.info()['versions'], + self.env.versions_container.name) + self.env.container.update_metadata( + hdrs={'X-Versions-Location': ''}) + self.assertEqual(self.env.container.info().get('versions'), None) + + # set location back to the way it was + self.env.container.update_metadata( + hdrs={'X-Versions-Location': self.env.versions_container.name}) + self.assertEqual(self.env.container.info()['versions'], + self.env.versions_container.name) + def test_overwriting(self): container = self.env.container versions_container = self.env.versions_container + cont_info = container.info() + self.assertEquals(cont_info['versions'], versions_container.name) + obj_name = Utils.create_name() versioned_obj = container.file(obj_name) - versioned_obj.write("aaaaa") + versioned_obj.write("aaaaa", hdrs={'Content-Type': 'text/jibberish01'}) + obj_info = versioned_obj.info() + self.assertEqual('text/jibberish01', obj_info['content_type']) self.assertEqual(0, versions_container.info()['object_count']) - - versioned_obj.write("bbbbb") + versioned_obj.write("bbbbb", hdrs={'Content-Type': 'text/jibberish02', + 'X-Object-Meta-Foo': 'Bar'}) + versioned_obj.initialize() + self.assertEqual(versioned_obj.content_type, 'text/jibberish02') + self.assertEqual(versioned_obj.metadata['foo'], 'Bar') # the old version got saved off self.assertEqual(1, versions_container.info()['object_count']) versioned_obj_name = versions_container.files()[0] - self.assertEqual( - "aaaaa", versions_container.file(versioned_obj_name).read()) + prev_version = versions_container.file(versioned_obj_name) + prev_version.initialize() + self.assertEqual("aaaaa", prev_version.read()) + self.assertEqual(prev_version.content_type, 'text/jibberish01') + + # make sure the new obj metadata did not leak to the prev. version + self.assertTrue('foo' not in prev_version.metadata) + + # check that POST does not create a new version + versioned_obj.sync_metadata(metadata={'fu': 'baz'}) + self.assertEqual(1, versions_container.info()['object_count']) # if we overwrite it again, there are two versions versioned_obj.write("ccccc") self.assertEqual(2, versions_container.info()['object_count']) + versioned_obj_name = versions_container.files()[1] + prev_version = versions_container.file(versioned_obj_name) + prev_version.initialize() + self.assertEqual("bbbbb", prev_version.read()) + self.assertEqual(prev_version.content_type, 'text/jibberish02') + self.assertTrue('foo' in prev_version.metadata) + self.assertTrue('fu' in prev_version.metadata) # as we delete things, the old contents return self.assertEqual("ccccc", versioned_obj.read()) + + # test copy from a different container + src_container = self.env.account.container(Utils.create_name()) + self.assertTrue(src_container.create()) + src_name = Utils.create_name() + src_obj = src_container.file(src_name) + src_obj.write("ddddd", hdrs={'Content-Type': 'text/jibberish04'}) + src_obj.copy(container.name, obj_name) + + self.assertEqual("ddddd", versioned_obj.read()) + versioned_obj.initialize() + self.assertEqual(versioned_obj.content_type, 'text/jibberish04') + + # make sure versions container has the previous version + self.assertEqual(3, versions_container.info()['object_count']) + versioned_obj_name = versions_container.files()[2] + prev_version = versions_container.file(versioned_obj_name) + prev_version.initialize() + self.assertEqual("ccccc", prev_version.read()) + + # test delete + versioned_obj.delete() + self.assertEqual("ccccc", versioned_obj.read()) versioned_obj.delete() self.assertEqual("bbbbb", versioned_obj.read()) versioned_obj.delete() self.assertEqual("aaaaa", versioned_obj.read()) + self.assertEqual(0, versions_container.info()['object_count']) versioned_obj.delete() self.assertRaises(ResponseError, versioned_obj.read) @@ -2598,6 +2992,87 @@ class TestObjectVersioning(Base): self.assertEqual(3, versions_container.info()['object_count']) self.assertEqual("112233", man_file.read()) + def test_versioning_container_acl(self): + # create versions container and DO NOT give write access to account2 + versions_container = self.env.account.container(Utils.create_name()) + self.assertTrue(versions_container.create(hdrs={ + 'X-Container-Write': '' + })) + + # check account2 cannot write to versions container + fail_obj_name = Utils.create_name() + fail_obj = versions_container.file(fail_obj_name) + self.assertRaises(ResponseError, fail_obj.write, "should fail", + cfg={'use_token': self.env.storage_token2}) + + # create container and give write access to account2 + # don't set X-Versions-Location just yet + container = self.env.account.container(Utils.create_name()) + self.assertTrue(container.create(hdrs={ + 'X-Container-Write': self.env.conn2.user_acl})) + + # check account2 cannot set X-Versions-Location on container + self.assertRaises(ResponseError, container.update_metadata, hdrs={ + 'X-Versions-Location': versions_container}, + cfg={'use_token': self.env.storage_token2}) + + # good! now let admin set the X-Versions-Location + # p.s.: sticking a 'x-remove' header here to test precedence + # of both headers. Setting the location should succeed. + self.assertTrue(container.update_metadata(hdrs={ + 'X-Remove-Versions-Location': versions_container, + 'X-Versions-Location': versions_container})) + + # write object twice to container and check version + obj_name = Utils.create_name() + versioned_obj = container.file(obj_name) + self.assertTrue(versioned_obj.write("never argue with the data", + cfg={'use_token': self.env.storage_token2})) + self.assertEqual(versioned_obj.read(), "never argue with the data") + + self.assertTrue( + versioned_obj.write("we don't have no beer, just tequila", + cfg={'use_token': self.env.storage_token2})) + self.assertEqual(versioned_obj.read(), + "we don't have no beer, just tequila") + self.assertEqual(1, versions_container.info()['object_count']) + + # read the original uploaded object + for filename in versions_container.files(): + backup_file = versions_container.file(filename) + break + self.assertEqual(backup_file.read(), "never argue with the data") + + # user3 (some random user with no access to anything) + # tries to read from versioned container + self.assertRaises(ResponseError, backup_file.read, + cfg={'use_token': self.env.storage_token3}) + + # user3 cannot write or delete from source container either + self.assertRaises(ResponseError, versioned_obj.write, + "some random user trying to write data", + cfg={'use_token': self.env.storage_token3}) + self.assertRaises(ResponseError, versioned_obj.delete, + cfg={'use_token': self.env.storage_token3}) + + # user2 can't read or delete from versions-location + self.assertRaises(ResponseError, backup_file.read, + cfg={'use_token': self.env.storage_token2}) + self.assertRaises(ResponseError, backup_file.delete, + cfg={'use_token': self.env.storage_token2}) + + # but is able to delete from the source container + # this could be a helpful scenario for dev ops that want to setup + # just one container to hold object versions of multiple containers + # and each one of those containers are owned by different users + self.assertTrue(versioned_obj.delete( + cfg={'use_token': self.env.storage_token2})) + + # tear-down since we create these containers here + # and not in self.env + versions_container.delete_recursive() + container.delete_recursive() + def test_versioning_check_acl(self): container = self.env.container versions_container = self.env.versions_container @@ -2713,8 +3188,8 @@ class TestTempurl(Base): self.assertEqual(contents, "obj contents") # GET tempurls also allow HEAD requests - self.assert_(self.env.obj.info(parms=self.obj_tempurl_parms, - cfg={'no_auth_token': True})) + self.assertTrue(self.env.obj.info(parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True})) def test_GET_with_key_2(self): expires = int(time.time()) + 86400 @@ -2727,6 +3202,59 @@ class TestTempurl(Base): contents = self.env.obj.read(parms=parms, cfg={'no_auth_token': True}) self.assertEqual(contents, "obj contents") + def test_GET_DLO_inside_container(self): + seg1 = self.env.container.file( + "get-dlo-inside-seg1" + Utils.create_name()) + seg2 = self.env.container.file( + "get-dlo-inside-seg2" + Utils.create_name()) + seg1.write("one fish two fish ") + seg2.write("red fish blue fish") + + manifest = self.env.container.file("manifest" + Utils.create_name()) + manifest.write( + '', + hdrs={"X-Object-Manifest": "%s/get-dlo-inside-seg" % + (self.env.container.name,)}) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(manifest.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + contents = manifest.read(parms=parms, cfg={'no_auth_token': True}) + self.assertEqual(contents, "one fish two fish red fish blue fish") + + def test_GET_DLO_outside_container(self): + seg1 = self.env.container.file( + "get-dlo-outside-seg1" + Utils.create_name()) + seg2 = self.env.container.file( + "get-dlo-outside-seg2" + Utils.create_name()) + seg1.write("one fish two fish ") + seg2.write("red fish blue fish") + + container2 = self.env.account.container(Utils.create_name()) + container2.create() + + manifest = container2.file("manifest" + Utils.create_name()) + manifest.write( + '', + hdrs={"X-Object-Manifest": "%s/get-dlo-outside-seg" % + (self.env.container.name,)}) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(manifest.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + # cross container tempurl works fine for account tempurl key + contents = manifest.read(parms=parms, cfg={'no_auth_token': True}) + self.assertEqual(contents, "one fish two fish red fish blue fish") + self.assert_status([200]) + def test_PUT(self): new_obj = self.env.container.file(Utils.create_name()) @@ -2742,8 +3270,60 @@ class TestTempurl(Base): self.assertEqual(new_obj.read(), "new obj contents") # PUT tempurls also allow HEAD requests - self.assert_(new_obj.info(parms=put_parms, - cfg={'no_auth_token': True})) + self.assertTrue(new_obj.info(parms=put_parms, + cfg={'no_auth_token': True})) + + def test_PUT_manifest_access(self): + new_obj = self.env.container.file(Utils.create_name()) + + # give out a signature which allows a PUT to new_obj + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'PUT', expires, self.env.conn.make_path(new_obj.path), + self.env.tempurl_key) + put_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + # try to create manifest pointing to some random container + try: + new_obj.write('', { + 'x-object-manifest': '%s/foo' % 'some_random_container' + }, parms=put_parms, cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 400) + else: + self.fail('request did not error') + + # create some other container + other_container = self.env.account.container(Utils.create_name()) + if not other_container.create(): + raise ResponseError(self.conn.response) + + # try to create manifest pointing to new container + try: + new_obj.write('', { + 'x-object-manifest': '%s/foo' % other_container + }, parms=put_parms, cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 400) + else: + self.fail('request did not error') + + # try again using a tempurl POST to an already created object + new_obj.write('', {}, parms=put_parms, cfg={'no_auth_token': True}) + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'POST', expires, self.env.conn.make_path(new_obj.path), + self.env.tempurl_key) + post_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + try: + new_obj.post({'x-object-manifest': '%s/foo' % other_container}, + parms=post_parms, cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 400) + else: + self.fail('request did not error') def test_HEAD(self): expires = int(time.time()) + 86400 @@ -2753,8 +3333,8 @@ class TestTempurl(Base): head_parms = {'temp_url_sig': sig, 'temp_url_expires': str(expires)} - self.assert_(self.env.obj.info(parms=head_parms, - cfg={'no_auth_token': True})) + self.assertTrue(self.env.obj.info(parms=head_parms, + cfg={'no_auth_token': True})) # HEAD tempurls don't allow PUT or GET requests, despite the fact that # PUT and GET tempurls both allow HEAD requests self.assertRaises(ResponseError, self.env.other_obj.read, @@ -2897,8 +3477,8 @@ class TestContainerTempurl(Base): self.assertEqual(contents, "obj contents") # GET tempurls also allow HEAD requests - self.assert_(self.env.obj.info(parms=self.obj_tempurl_parms, - cfg={'no_auth_token': True})) + self.assertTrue(self.env.obj.info(parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True})) def test_GET_with_key_2(self): expires = int(time.time()) + 86400 @@ -2926,8 +3506,8 @@ class TestContainerTempurl(Base): self.assertEqual(new_obj.read(), "new obj contents") # PUT tempurls also allow HEAD requests - self.assert_(new_obj.info(parms=put_parms, - cfg={'no_auth_token': True})) + self.assertTrue(new_obj.info(parms=put_parms, + cfg={'no_auth_token': True})) def test_HEAD(self): expires = int(time.time()) + 86400 @@ -2937,8 +3517,8 @@ class TestContainerTempurl(Base): head_parms = {'temp_url_sig': sig, 'temp_url_expires': str(expires)} - self.assert_(self.env.obj.info(parms=head_parms, - cfg={'no_auth_token': True})) + self.assertTrue(self.env.obj.info(parms=head_parms, + cfg={'no_auth_token': True})) # HEAD tempurls don't allow PUT or GET requests, despite the fact that # PUT and GET tempurls both allow HEAD requests self.assertRaises(ResponseError, self.env.other_obj.read, @@ -2997,6 +3577,7 @@ class TestContainerTempurl(Base): parms=parms) self.assert_status([401]) + @requires_acls def test_tempurl_keys_visible_to_account_owner(self): if not tf.cluster_info.get('tempauth'): raise SkipTest('TEMP AUTH SPECIFIC TEST') @@ -3004,6 +3585,7 @@ class TestContainerTempurl(Base): self.assertEqual(metadata.get('tempurl_key'), self.env.tempurl_key) self.assertEqual(metadata.get('tempurl_key2'), self.env.tempurl_key2) + @requires_acls def test_tempurl_keys_hidden_from_acl_readonly(self): if not tf.cluster_info.get('tempauth'): raise SkipTest('TEMP AUTH SPECIFIC TEST') @@ -3012,12 +3594,75 @@ class TestContainerTempurl(Base): metadata = self.env.container.info() self.env.container.conn.storage_token = original_token - self.assertTrue('tempurl_key' not in metadata, - 'Container TempURL key found, should not be visible ' - 'to readonly ACLs') - self.assertTrue('tempurl_key2' not in metadata, - 'Container TempURL key-2 found, should not be visible ' - 'to readonly ACLs') + self.assertNotIn( + 'tempurl_key', metadata, + 'Container TempURL key found, should not be visible ' + 'to readonly ACLs') + self.assertNotIn( + 'tempurl_key2', metadata, + 'Container TempURL key-2 found, should not be visible ' + 'to readonly ACLs') + + def test_GET_DLO_inside_container(self): + seg1 = self.env.container.file( + "get-dlo-inside-seg1" + Utils.create_name()) + seg2 = self.env.container.file( + "get-dlo-inside-seg2" + Utils.create_name()) + seg1.write("one fish two fish ") + seg2.write("red fish blue fish") + + manifest = self.env.container.file("manifest" + Utils.create_name()) + manifest.write( + '', + hdrs={"X-Object-Manifest": "%s/get-dlo-inside-seg" % + (self.env.container.name,)}) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(manifest.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + contents = manifest.read(parms=parms, cfg={'no_auth_token': True}) + self.assertEqual(contents, "one fish two fish red fish blue fish") + + def test_GET_DLO_outside_container(self): + container2 = self.env.account.container(Utils.create_name()) + container2.create() + seg1 = container2.file( + "get-dlo-outside-seg1" + Utils.create_name()) + seg2 = container2.file( + "get-dlo-outside-seg2" + Utils.create_name()) + seg1.write("one fish two fish ") + seg2.write("red fish blue fish") + + manifest = self.env.container.file("manifest" + Utils.create_name()) + manifest.write( + '', + hdrs={"X-Object-Manifest": "%s/get-dlo-outside-seg" % + (container2.name,)}) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(manifest.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + # cross container tempurl does not work for container tempurl key + try: + manifest.read(parms=parms, cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 401) + else: + self.fail('request did not error') + try: + manifest.info(parms=parms, cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 401) + else: + self.fail('request did not error') class TestContainerTempurlUTF8(Base2, TestContainerTempurl): @@ -3103,7 +3748,7 @@ class TestSloTempurl(Base): self.assertEqual(len(contents), 2 * 1024 * 1024) # GET tempurls also allow HEAD requests - self.assert_(self.env.manifest.info( + self.assertTrue(self.env.manifest.info( parms=parms, cfg={'no_auth_token': True})) @@ -3210,8 +3855,6 @@ class TestServiceToken(unittest.TestCase): headers = {} if self.body: headers.update({'Content-Length': len(self.body)}) - if self.headers: - headers.update(self.headers) if self.x_auth_token == self.SET_TO_USERS_TOKEN: headers.update({'X-Auth-Token': token}) elif self.x_auth_token == self.SET_TO_SERVICE_TOKEN: @@ -3244,7 +3887,7 @@ class TestServiceToken(unittest.TestCase): self.prepare_request('HEAD') resp = retry(self.do_request) resp.read() - self.assert_(resp.status in (200, 204), resp.status) + self.assertIn(resp.status, (200, 204)) def test_user_cannot_access_service_account(self): for method, container, obj in self._scenario_generator(): diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 372fb58..4a07e17 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -15,10 +15,12 @@ """ Swift tests """ +from __future__ import print_function import os import copy import logging import errno +from six.moves import range import sys from contextlib import contextmanager, closing from collections import defaultdict, Iterable @@ -30,17 +32,18 @@ import eventlet from eventlet.green import socket from tempfile import mkdtemp from shutil import rmtree -from swift.common.utils import Timestamp +from swift.common.utils import Timestamp, NOTICE from test import get_config from swift.common import swob, utils from swift.common.ring import Ring, RingData from hashlib import md5 import logging.handlers -from httplib import HTTPException + +from six.moves.http_client import HTTPException from swift.common import storage_policy from swift.common.storage_policy import StoragePolicy, ECStoragePolicy import functools -import cPickle as pickle +import six.moves.cPickle as pickle from gzip import GzipFile import mock as mocklib import inspect @@ -226,9 +229,7 @@ class FakeRing(Ring): return [dict(node, index=i) for i, node in enumerate(list(self._devs))] def get_more_nodes(self, part): - # replicas^2 is the true cap - for x in xrange(self.replicas, min(self.replicas + self.max_more_nodes, - self.replicas * self.replicas)): + for x in range(self.replicas, (self.replicas + self.max_more_nodes)): yield {'ip': '10.0.0.%s' % x, 'replication_ip': '10.0.0.%s' % x, 'port': self._base_port + x, @@ -478,8 +479,18 @@ class FakeLogger(logging.Logger, object): logging.INFO: 'info', logging.DEBUG: 'debug', logging.CRITICAL: 'critical', + NOTICE: 'notice', } + def notice(self, msg, *args, **kwargs): + """ + Convenience function for syslog priority LOG_NOTICE. The python + logging lvl is set to 25, just above info. SysLogHandler is + monkey patched to map this log lvl to the LOG_NOTICE syslog + priority. + """ + self.log(NOTICE, msg, *args, **kwargs) + def _log(self, level, msg, *args, **kwargs): store_name = self.store_in[level] cargs = [msg] @@ -495,7 +506,9 @@ class FakeLogger(logging.Logger, object): def _clear(self): self.log_dict = defaultdict(list) self.lines_dict = {'critical': [], 'error': [], 'info': [], - 'warning': [], 'debug': []} + 'warning': [], 'debug': [], 'notice': []} + + clear = _clear # this is a public interface def get_lines_for_level(self, level): if level not in self.lines_dict: @@ -560,8 +573,8 @@ class FakeLogger(logging.Logger, object): try: line = record.getMessage() except TypeError: - print 'WARNING: unable to format log message %r %% %r' % ( - record.msg, record.args) + print('WARNING: unable to format log message %r %% %r' % ( + record.msg, record.args)) raise self.lines_dict[record.levelname.lower()].append(line) @@ -585,7 +598,7 @@ class DebugLogger(FakeLogger): def handle(self, record): self._handle(record) - print self.formatter.format(record) + print(self.formatter.format(record)) class DebugLogAdapter(utils.LogAdapter): @@ -704,6 +717,74 @@ def mock(update): delattr(module, attr) +class FakeStatus(object): + """ + This will work with our fake_http_connect, if you hand in one of these + instead of a status int or status int tuple to the "codes" iter you can + add some eventlet sleep to the expect and response stages of the + connection. + """ + + def __init__(self, status, expect_sleep=None, response_sleep=None): + """ + :param status: the response status int, or a tuple of + ([expect_status, ...], response_status) + :param expect_sleep: float, time to eventlet sleep during expect, can + be a iter of floats + :param response_sleep: float, time to eventlet sleep during response + """ + # connect exception + if isinstance(status, (Exception, eventlet.Timeout)): + raise status + if isinstance(status, tuple): + self.expect_status = list(status[:-1]) + self.status = status[-1] + self.explicit_expect_list = True + else: + self.expect_status, self.status = ([], status) + self.explicit_expect_list = False + if not self.expect_status: + # when a swift backend service returns a status before reading + # from the body (mostly an error response) eventlet.wsgi will + # respond with that status line immediately instead of 100 + # Continue, even if the client sent the Expect 100 header. + # BufferedHttp and the proxy both see these error statuses + # when they call getexpect, so our FakeConn tries to act like + # our backend services and return certain types of responses + # as expect statuses just like a real backend server would do. + if self.status in (507, 412, 409): + self.expect_status = [status] + else: + self.expect_status = [100, 100] + + # setup sleep attributes + if not isinstance(expect_sleep, (list, tuple)): + expect_sleep = [expect_sleep] * len(self.expect_status) + self.expect_sleep_list = list(expect_sleep) + while len(self.expect_sleep_list) < len(self.expect_status): + self.expect_sleep_list.append(None) + self.response_sleep = response_sleep + + def get_response_status(self): + if self.response_sleep is not None: + eventlet.sleep(self.response_sleep) + if self.expect_status and self.explicit_expect_list: + raise Exception('Test did not consume all fake ' + 'expect status: %r' % (self.expect_status,)) + if isinstance(self.status, (Exception, eventlet.Timeout)): + raise self.status + return self.status + + def get_expect_status(self): + expect_sleep = self.expect_sleep_list.pop(0) + if expect_sleep is not None: + eventlet.sleep(expect_sleep) + expect_status = self.expect_status.pop(0) + if isinstance(expect_status, (Exception, eventlet.Timeout)): + raise expect_status + return expect_status + + class SlowBody(object): """ This will work with our fake_http_connect, if you hand in these @@ -741,29 +822,9 @@ def fake_http_connect(*code_iter, **kwargs): def __init__(self, status, etag=None, body='', timestamp='1', headers=None, expect_headers=None, connection_id=None, give_send=None): - # connect exception - if isinstance(status, (Exception, eventlet.Timeout)): - raise status - if isinstance(status, tuple): - self.expect_status = list(status[:-1]) - self.status = status[-1] - self.explicit_expect_list = True - else: - self.expect_status, self.status = ([], status) - self.explicit_expect_list = False - if not self.expect_status: - # when a swift backend service returns a status before reading - # from the body (mostly an error response) eventlet.wsgi will - # respond with that status line immediately instead of 100 - # Continue, even if the client sent the Expect 100 header. - # BufferedHttp and the proxy both see these error statuses - # when they call getexpect, so our FakeConn tries to act like - # our backend services and return certain types of responses - # as expect statuses just like a real backend server would do. - if self.status in (507, 412, 409): - self.expect_status = [status] - else: - self.expect_status = [100, 100] + if not isinstance(status, FakeStatus): + status = FakeStatus(status) + self._status = status self.reason = 'Fake' self.host = '1.2.3.4' self.port = '1234' @@ -785,11 +846,6 @@ def fake_http_connect(*code_iter, **kwargs): eventlet.sleep() def getresponse(self): - if self.expect_status and self.explicit_expect_list: - raise Exception('Test did not consume all fake ' - 'expect status: %r' % (self.expect_status,)) - if isinstance(self.status, (Exception, eventlet.Timeout)): - raise self.status exc = kwargs.get('raise_exc') if exc: if isinstance(exc, (Exception, eventlet.Timeout)): @@ -797,16 +853,19 @@ def fake_http_connect(*code_iter, **kwargs): raise Exception('test') if kwargs.get('raise_timeout_exc'): raise eventlet.Timeout() + self.status = self._status.get_response_status() return self def getexpect(self): - expect_status = self.expect_status.pop(0) - if isinstance(self.expect_status, (Exception, eventlet.Timeout)): - raise self.expect_status + expect_status = self._status.get_expect_status() headers = dict(self.expect_headers) if expect_status == 409: headers['X-Backend-Timestamp'] = self.timestamp - return FakeConn(expect_status, headers=headers) + response = FakeConn(expect_status, + timestamp=self.timestamp, + headers=headers) + response.status = expect_status + return response def getheaders(self): etag = self.etag @@ -834,7 +893,7 @@ def fake_http_connect(*code_iter, **kwargs): # when timestamp is None, HeaderKeyDict raises KeyError headers.pop('x-timestamp', None) try: - if container_ts_iter.next() is False: + if next(container_ts_iter) is False: headers['x-container-timestamp'] = '1' except StopIteration: pass @@ -911,24 +970,24 @@ def fake_http_connect(*code_iter, **kwargs): kwargs['give_content_type'](args[6]['Content-Type']) else: kwargs['give_content_type']('') - i, status = conn_id_and_code_iter.next() + i, status = next(conn_id_and_code_iter) if 'give_connect' in kwargs: give_conn_fn = kwargs['give_connect'] argspec = inspect.getargspec(give_conn_fn) if argspec.keywords or 'connection_id' in argspec.args: ckwargs['connection_id'] = i give_conn_fn(*args, **ckwargs) - etag = etag_iter.next() - headers = headers_iter.next() - expect_headers = expect_headers_iter.next() - timestamp = timestamps_iter.next() + etag = next(etag_iter) + headers = next(headers_iter) + expect_headers = next(expect_headers_iter) + timestamp = next(timestamps_iter) if status <= 0: raise HTTPException() if body_iter is None: body = static_body or '' else: - body = body_iter.next() + body = next(body_iter) return FakeConn(status, etag, body=body, timestamp=timestamp, headers=headers, expect_headers=expect_headers, connection_id=i, give_send=kwargs.get('give_send')) diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index af0d43b..a77bec4 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -207,9 +207,6 @@ class TestDiskFile(unittest.TestCase): assert not gdf._is_dir assert gdf._fd is not None assert gdf._metadata == exp_md - self.assertRaises(DiskFileNotOpen, gdf.get_metadata) - self.assertRaises(DiskFileNotOpen, gdf.reader) - self.assertRaises(DiskFileNotOpen, gdf.__enter__) def test_open_and_close(self): mock_close = Mock() @@ -480,7 +477,6 @@ class TestDiskFile(unittest.TestCase): gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") md = {'Content-Type': 'application/octet-stream', 'a': 'b'} gdf.write_metadata(md.copy()) - self.assertEqual(None, gdf._metadata) fmd = _metadata[_mapit(the_dir)] md.update({'X-Object-Type': 'file', 'X-Type': 'Object'}) self.assertTrue(fmd['a'], md['a']) diff --git a/tox.ini b/tox.ini index befbcf3..2deccdf 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,pep8,functest,functest-ci +envlist = py27,pep8,functest minversion = 1.6 skipsdist = True @@ -15,11 +15,12 @@ deps = # Note: pip supports installing from git repos. # https://pip.pypa.io/en/latest/reference/pip_install.html#git # Example: git+https://github.com/openstack/swift.git@2.0.0 - https://launchpad.net/swift/kilo/2.3.0/+download/swift-2.3.0.tar.gz + https://launchpad.net/swift/liberty/2.5.0/+download/swift-2.5.0.tar.gz PyECLib==1.0.7 -r{toxinidir}/test-requirements.txt changedir = {toxinidir}/test/unit commands = nosetests -v {posargs} +passenv = SWIFT_* *_proxy [testenv:cover] setenv = VIRTUAL_ENV={envdir} @@ -28,9 +29,6 @@ setenv = VIRTUAL_ENV={envdir} NOSE_COVER_HTML=1 NOSE_COVER_HTML_DIR={toxinidir}/cover -[tox:jenkins] -downloadcache = ~/cache/pip - [testenv:functest] changedir = {toxinidir} commands = bash ./.functests -q @@ -46,14 +44,20 @@ changedir = {toxinidir} commands = {posargs} [flake8] -# it's not a bug that we aren't using all of hacking -# H102 -> apache2 license exists -# H103 -> license is apache -# H201 -> no bare excepts (unless marked with " # noqa") -# H231 -> Check for except statements to be Python 3.x compatible -# H501 -> don't use locals() for str formatting -# H903 -> \n not \r\n -ignore = H -select = F,E,W,H102,H103,H201,H231,H501,H903 +# it's not a bug that we aren't using all of hacking, ignore: +# F812: list comprehension redefines ... +# H101: Use TODO(NAME) +# H202: assertRaises Exception too broad +# H233: Python 3.x incompatible use of print operator +# H234: assertEquals is deprecated, use assertEqual +# H301: one import per line +# H306: imports not in alphabetical order (time, os) +# H401: docstring should not start with a space +# H403: multi line docstrings should end on a new line +# H404: multi line docstring should start without a leading new line +# H405: multi line docstring summary not separated with an empty line +# H501: Do not use self.__dict__ for string formatting +# H703: Multiple positional placeholders +ignore = F812,H101,H202,H233,H234,H301,H306,H401,H403,H404,H405,H501,H703 exclude = .venv,.tox,dist,doc,*egg,test show-source = True