diff --git a/README.md b/README.md index c505be9..2546198 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,13 @@ setup.py install. $ virtualenv $HOME/venv $ . $HOME/venv/bin/activate + $ pip install -r tools/pip-requires $ python setup.py install +Note, without the manual pip install, the installation might failed with +this error: 'TypeError: dist must be a Distribution instance' +ref: https://bugs.launchpad.net/swift/+bug/1217288 + swfiller usage -------------- diff --git a/bin/swfiller b/bin/swfiller index e598714..af92ffb 100755 --- a/bin/swfiller +++ b/bin/swfiller @@ -53,9 +53,10 @@ def main(): if args.config and os.path.isfile(args.config): try: conf = utils.parse_ini(args.config) + except Exception, exc: logging.info('Unable to parse provided conf file') - except Exception: - pass + logging.error(exc) + sys.exit(1) else: try: conf = utils.parse_ini() diff --git a/swsync/containers.py b/swsync/containers.py index 659ef11..e9103e4 100644 --- a/swsync/containers.py +++ b/swsync/containers.py @@ -96,7 +96,11 @@ class Containers(object): container_headers = orig_container_headers.copy() for h in ('x-container-object-count', 'x-trans-id', 'x-container-bytes-used'): - del container_headers[h] + try: + del container_headers[h] + except KeyError: + # Nov2013: swift server does not set x-trans-id header + pass p = dest_storage_cnx[0] url = "%s://%s%s" % (p.scheme, p.netloc, p.path) try: diff --git a/swsync/filler.py b/swsync/filler.py index 69c826d..7f5c15a 100644 --- a/swsync/filler.py +++ b/swsync/filler.py @@ -86,6 +86,10 @@ def create_swift_user(client, account_name, account_id, user_amount): # Get swift_operator_role id roleid = [role.id for role in client.roles.list() if role.name == get_config('filler', 'swift_operator_role')] + if not roleid: + logging.error('Could not find swift_operator_role %s in keystone' % + get_config('filler', 'swift_operator_role')) + sys.exit(1) roleid = roleid[0] # Add tenant/user in swift operator role/group client.roles.add_user_role(uid.id, roleid, account_id) @@ -235,7 +239,7 @@ def create_containers(cnx, acc, c_amount, index_containers=None): try: cnx.put_container(container_name, headers=copy.copy(meta)) containers_d[container_name] = {'meta': meta, 'objects': []} - except ClientException, e: + except(ClientException), e: logging.warning("Unable to create container %s due to %s" % (container_name.encode('ascii', 'ignore'), e)) diff --git a/swsync/objects.py b/swsync/objects.py index 60bcfe1..40730b2 100644 --- a/swsync/objects.py +++ b/swsync/objects.py @@ -19,7 +19,12 @@ import logging import eventlet import swift.common.bufferedhttp import swift.common.http -import swift.container.sync +try: + from swift.container.sync import _Iter2FileLikeObject as FileLikeIter +except ImportError: + # Nov2013: swift.common.utils now include a more generic object + from swift.common.utils import FileLikeIter + from swiftclient import client as swiftclient import urllib import urllib2 @@ -107,11 +112,10 @@ def sync_object(orig_storage_url, orig_token, dest_storage_url, post_headers = orig_headers post_headers['x-auth-token'] = dest_token sync_to = dest_storage_url + "/" + container_name - iterlike = swift.container.sync._Iter2FileLikeObject try: swiftclient.put_object(sync_to, name=object_name, headers=post_headers, - contents=iterlike(orig_body)) + contents=FileLikeIter(orig_body)) except(swiftclient.ClientException), e: logging.info("error sync object: %s, %s" % ( object_name, e.http_reason)) diff --git a/tests/units/test_accounts.py b/tests/units/test_accounts.py index 6f014ff..8d9264f 100644 --- a/tests/units/test_accounts.py +++ b/tests/units/test_accounts.py @@ -101,8 +101,8 @@ class TestAccountSyncMetadata(TestAccountBase): post_account_called=post_account_called, get_account_called=get_account_called) - self.assertEquals(len(sync_container_called), 1) - self.assertEquals(len(get_account_called), 2) + self.assertEqual(len(sync_container_called), 1) + self.assertEqual(len(get_account_called), 2) self.assertTrue(info_called) self.assertIn('x-account-meta-life', @@ -134,8 +134,8 @@ class TestAccountSyncMetadata(TestAccountBase): post_account_called=post_account_called, get_account_called=get_account_called) - self.assertEquals(len(sync_container_called), 1) - self.assertEquals(len(get_account_called), 2) + self.assertEqual(len(sync_container_called), 1) + self.assertEqual(len(get_account_called), 2) self.assertTrue(info_called) self.assertIn('x-account-meta-life', @@ -167,8 +167,8 @@ class TestAccountSyncMetadata(TestAccountBase): post_account_called=post_account_called, get_account_called=get_account_called) - self.assertEquals(len(sync_container_called), 1) - self.assertEquals(len(get_account_called), 2) + self.assertEqual(len(sync_container_called), 1) + self.assertEqual(len(get_account_called), 2) self.assertTrue(info_called) self.assertIn('x-account-meta-life', @@ -212,7 +212,7 @@ class TestAccountSync(TestAccountBase): ret = self.accounts_cls.get_swift_auth( "http://test.com", tenant_name, "user", "password") tenant_id = fakes.TENANTS_LIST[tenant_name]['id'] - self.assertEquals(ret[0], "%s/v1/AUTH_%s" % (fakes.STORAGE_DEST, + self.assertEqual(ret[0], "%s/v1/AUTH_%s" % (fakes.STORAGE_DEST, tenant_id)) def test_get_ks_auth_orig(self): @@ -220,11 +220,11 @@ class TestAccountSync(TestAccountBase): k = fakes.CONFIGDICT['auth']['keystone_origin_admin_credentials'] tenant_name, username, password = k.split(':') - self.assertEquals(kwargs['tenant_name'], tenant_name) - self.assertEquals(kwargs['username'], username) - self.assertEquals(kwargs['password'], password) + self.assertEqual(kwargs['tenant_name'], tenant_name) + self.assertEqual(kwargs['username'], username) + self.assertEqual(kwargs['password'], password) k = fakes.CONFIGDICT['auth']['keystone_origin'] - self.assertEquals(k, kwargs['auth_url']) + self.assertEqual(k, kwargs['auth_url']) def test_process(self): ret = [] @@ -240,7 +240,7 @@ class TestAccountSync(TestAccountBase): for x in fakes.TENANTS_LIST) ret_orig_storage_id = sorted( x[0][x[0].find('AUTH_') + 5:] for x in ret) - self.assertEquals(tenant_list_ids, ret_orig_storage_id) + self.assertEqual(tenant_list_ids, ret_orig_storage_id) [self.assertTrue(y[1].startswith(fakes.STORAGE_DEST)) for y in ret] def test_sync_account(self): @@ -270,7 +270,7 @@ class TestAccountSync(TestAccountBase): ret_container_list = sorted(x[7] for x in ret) default_container_list = sorted(x[0]['name'] for x in fakes.CONTAINERS_LIST) - self.assertEquals(ret_container_list, default_container_list) + self.assertEqual(ret_container_list, default_container_list) def test_sync_exception_get_account(self): called = [] diff --git a/tests/units/test_containers.py b/tests/units/test_containers.py index 0445102..c881033 100644 --- a/tests/units/test_containers.py +++ b/tests/units/test_containers.py @@ -104,8 +104,8 @@ class TestContainersSyncMetadata(TestContainersBase): post_called, info_called) self.assertEqual(len(get_called), 2) self.assertEqual(len(post_called), 1) - self.assertEquals(post_called[0]['x-container-meta-psg'], '') - self.assertEquals(post_called[0]['x-container-meta-om'], 'enkl') + self.assertEqual(post_called[0]['x-container-meta-psg'], '') + self.assertEqual(post_called[0]['x-container-meta-om'], 'enkl') self.assertIn('HEADER: sync headers: cont1', info_called) def test_sync_containers_metada_added_on_orig(self): @@ -130,7 +130,7 @@ class TestContainersSyncMetadata(TestContainersBase): self.assertIn('HEADER: sync headers: cont1', info_called) self.assertEqual(len(get_called), 2) self.assertEqual(len(post_called), 1) - self.assertEquals(post_called[0]['x-container-meta-om'], 'enkl') + self.assertEqual(post_called[0]['x-container-meta-om'], 'enkl') def test_sync_containers_metada_changed(self): get_called = [] @@ -153,7 +153,7 @@ class TestContainersSyncMetadata(TestContainersBase): post_called, info_called) self.assertEqual(len(get_called), 2) self.assertEqual(len(post_called), 1) - self.assertEquals(post_called[0]['x-container-meta-psg'], 'magic') + self.assertEqual(post_called[0]['x-container-meta-psg'], 'magic') self.assertIn('HEADER: sync headers: cont1', info_called) def test_sync_containers_metadata_raise_client(self): diff --git a/tests/units/test_middleware_lm.py b/tests/units/test_middleware_lm.py index e692b22..e353f18 100644 --- a/tests/units/test_middleware_lm.py +++ b/tests/units/test_middleware_lm.py @@ -53,11 +53,11 @@ class TestLastModifiedMiddleware(unittest.TestCase): def test_denied_method_conf(self): app = FakeApp() test = middleware.filter_factory({})(app) - self.assertEquals(test.key_name, 'Last-Modified') + self.assertEqual(test.key_name, 'Last-Modified') test = middleware.filter_factory({'key_name': "Last Modified"})(app) - self.assertEquals(test.key_name, 'Last-Modified') + self.assertEqual(test.key_name, 'Last-Modified') test = middleware.filter_factory({'key_name': "Custom Key"})(app) - self.assertEquals(test.key_name, 'Custom-Key') + self.assertEqual(test.key_name, 'Custom-Key') def test_PUT_on_container(self): self.called = False diff --git a/tests/units/test_objects.py b/tests/units/test_objects.py index 5d14d2e..be11126 100644 --- a/tests/units/test_objects.py +++ b/tests/units/test_objects.py @@ -16,6 +16,12 @@ # under the License. import eventlet import swift +try: + from swift.container.sync import _Iter2FileLikeObject as FileLikeIter +except ImportError: + # Nov2013: swift.common.utils now include a more generic object + from swift.common.utils import FileLikeIter + import swiftclient import swsync.objects as swobjects @@ -86,8 +92,7 @@ class TestObject(test_base.TestCase): def put_object(url, name=None, headers=None, contents=None): self.assertEqual('obj1', name) self.assertIn('x-auth-token', headers) - self.assertIsInstance(contents, - swift.container.sync._Iter2FileLikeObject) + self.assertIsInstance(contents, FileLikeIter) contents_read = contents.read() self.assertEqual(len(contents_read), len(body)) @@ -153,7 +158,7 @@ class TestObject(test_base.TestCase): "cont1", "obj1") self.assertIn('x-foo', headers) - self.assertEquals(headers['x-foo'], 'BaR') + self.assertEqual(headers['x-foo'], 'BaR') def test_get_object_over_conn_timeout(self): new_connect = fake_http_connect(200, connect_waitfor=2)