diff --git a/swsync/accounts.py b/swsync/accounts.py index 95a3ef6..4bf2c3a 100644 --- a/swsync/accounts.py +++ b/swsync/accounts.py @@ -16,6 +16,7 @@ # under the License. import datetime import logging +import os import time import dateutil.relativedelta @@ -51,35 +52,80 @@ class Accounts(object): password=password, tenant_name=tenant_name) + def account_headers_clean(self, account_headers, to_null=False): + ret = {} + for key, value in account_headers.iteritems(): + if key.startswith('x-account-meta'): + if to_null: + value = '' + ret[key] = value + return ret + def sync_account(self, orig_storage_url, orig_token, dest_storage_url, dest_token): """Sync a single account with url/tok to dest_url/dest_tok.""" orig_storage_cnx = swiftclient.http_connection(orig_storage_url) dest_storage_cnx = swiftclient.http_connection(dest_storage_url) + account_id = os.path.basename(orig_storage_url.replace("AUTH_", '')) try: - orig_stats, orig_containers = ( + orig_account_headers, orig_containers = ( swiftclient.get_account(None, orig_token, http_conn=orig_storage_cnx, full_listing=True)) - dest_stats, dest_containers = ( + dest_account_headers, dest_containers = ( swiftclient.get_account(None, dest_token, http_conn=dest_storage_cnx, full_listing=True)) except(swiftclient.client.ClientException), e: - logging.info("error getting containeaccount: %s, %s" % ( - orig_storage_url, e.http_reason)) + logging.info("error getting account: %s, %s" % ( + account_id, e.http_reason)) return - if int(dest_stats['x-account-container-count']) > \ - int(orig_stats['x-account-container-count']): + + if int(dest_account_headers['x-account-container-count']) > \ + int(orig_account_headers['x-account-container-count']): self.container_cls.delete_container(dest_storage_cnx, dest_token, orig_containers, dest_containers) + do_headers = False + if len(dest_account_headers) != len(orig_account_headers): + do_headers = True + else: + for k, v in orig_account_headers.iteritems(): + if not k.startswith('x-account-meta'): + continue + if k not in dest_account_headers: + do_headers = True + elif dest_account_headers[k] != v: + do_headers = True + + if do_headers: + orig_metadata_headers = self.account_headers_clean( + orig_account_headers) + dest_metadata_headers = self.account_headers_clean( + dest_account_headers, to_null=True) + + new_headers = dict(dest_metadata_headers.items() + + orig_metadata_headers.items()) + try: + swiftclient.post_account( + "", dest_token, new_headers, + http_conn=dest_storage_cnx, + ) + logging.info("HEADER: sync headers: %s" % (account_id)) + except(swiftclient.client.ClientException), e: + logging.info("ERROR: updating container metadata: %s, %s" % ( + account_id, e.http_reason)) + # We don't pass on because since the server was busy + # let's pass it on for the next pass + return + for container in orig_containers: - logging.info("Syncronizing %s: %s", container['name'], container) + logging.info("Syncronizing container %s: %s", + container['name'], container) dt1 = datetime.datetime.fromtimestamp(time.time()) self.container_cls.sync(orig_storage_cnx, orig_storage_url, diff --git a/tests/units/test_accounts.py b/tests/units/test_accounts.py index ebc80ec..962ec80 100644 --- a/tests/units/test_accounts.py +++ b/tests/units/test_accounts.py @@ -24,25 +24,183 @@ import tests.units.base import tests.units.fakes as fakes -class TestAccount(tests.units.base.TestCase): +class TestAccountBase(tests.units.base.TestCase): def setUp(self): - super(TestAccount, self).setUp() + super(TestAccountBase, self).setUp() self.accounts_cls = swsync.accounts.Accounts() self._stubs() - def get_account(self, *args, **kwargs): - return ({'x-account-container-count': len(fakes.CONTAINERS_LIST)}, - [x[0] for x in fakes.CONTAINERS_LIST]) - def _stubs(self): self.stubs.Set(keystoneclient.v2_0, 'client', fakes.FakeKS) self.stubs.Set(swiftclient.client, 'Connection', fakes.FakeSWConnection) self.stubs.Set(swsync.accounts, 'get_config', fakes.fake_get_config) - self.stubs.Set(swiftclient, 'get_account', self.get_account) self.stubs.Set(swiftclient, 'http_connection', fakes.FakeSWClient.http_connection) + +class TestAccountSyncMetadata(TestAccountBase): + def _base_sync_metadata(self, orig_dict={}, + dest_dict={}, + get_account_called=[], + post_account_called=[], + info_called=[], + sync_container_called=[], + raise_post_account=False): + + def fake_info(msg, *args): + info_called.append(msg) + self.stubs.Set(logging, 'info', fake_info) + + def get_account(self, *args, **kwargs): + if len(get_account_called) == 0: + get_account_called.append(args) + return orig_dict + else: + get_account_called.append(args) + return dest_dict + self.stubs.Set(swiftclient, 'get_account', get_account) + + def post_account(url, token, headers, **kwargs): + post_account_called.append(headers) + + if raise_post_account: + raise swiftclient.client.ClientException("Error in testing") + self.stubs.Set(swiftclient, 'post_account', post_account) + + class Containers(object): + def sync(*args, **kwargs): + sync_container_called.append(args) + self.accounts_cls.container_cls = Containers() + self.accounts_cls.sync_account("http://orig", "otoken", + "http://dest", "dtoken") + + def test_sync_metadata_delete_dest(self): + get_account_called = [] + sync_container_called = [] + post_account_called = [] + info_called = [] + + orig_dict = ({'x-account-meta-life': 'beautiful', + 'x-account-container-count': 1}, + [{'name': 'cont1'}]) + + dest_dict = ({'x-account-meta-vita': 'bella', + 'x-account-container-count': 1}, + [{'name': 'cont1'}]) + self._base_sync_metadata(orig_dict, + dest_dict, + info_called=info_called, + sync_container_called=sync_container_called, + post_account_called=post_account_called, + get_account_called=get_account_called) + + self.assertEquals(len(sync_container_called), 1) + self.assertEquals(len(get_account_called), 2) + self.assertTrue(info_called) + + self.assertIn('x-account-meta-life', + post_account_called[0]) + self.assertEqual(post_account_called[0]['x-account-meta-life'], + 'beautiful') + self.assertIn('x-account-meta-vita', + post_account_called[0]) + self.assertEqual(post_account_called[0]['x-account-meta-vita'], + '') + + def test_sync_metadata_update_dest(self): + get_account_called = [] + sync_container_called = [] + post_account_called = [] + info_called = [] + + orig_dict = ({'x-account-meta-life': 'beautiful', + 'x-account-container-count': 1}, + [{'name': 'cont1'}]) + + dest_dict = ({'x-account-meta-life': 'bella', + 'x-account-container-count': 1}, + [{'name': 'cont1'}]) + self._base_sync_metadata(orig_dict, + dest_dict, + info_called=info_called, + sync_container_called=sync_container_called, + post_account_called=post_account_called, + get_account_called=get_account_called) + + self.assertEquals(len(sync_container_called), 1) + self.assertEquals(len(get_account_called), 2) + self.assertTrue(info_called) + + self.assertIn('x-account-meta-life', + post_account_called[0]) + self.assertEqual(post_account_called[0]['x-account-meta-life'], + 'beautiful') + + self.assertIn('x-account-meta-life', + post_account_called[0]) + self.assertEqual(post_account_called[0]['x-account-meta-life'], + 'beautiful') + + def test_sync_metadata_add_to_dest(self): + info_called = [] + get_account_called = [] + sync_container_called = [] + post_account_called = [] + + orig_dict = ({'x-account-meta-life': 'beautiful', + 'x-account-container-count': 1}, + [{'name': 'cont1'}]) + + dest_dict = ({'x-account-container-count': 1}, + [{'name': 'cont1'}]) + self._base_sync_metadata(orig_dict, + dest_dict, + info_called=info_called, + sync_container_called=sync_container_called, + post_account_called=post_account_called, + get_account_called=get_account_called) + + self.assertEquals(len(sync_container_called), 1) + self.assertEquals(len(get_account_called), 2) + self.assertTrue(info_called) + + self.assertIn('x-account-meta-life', + post_account_called[0]) + self.assertEqual(post_account_called[0]['x-account-meta-life'], + 'beautiful') + + self.assertIn('x-account-meta-life', + post_account_called[0]) + self.assertEqual(post_account_called[0]['x-account-meta-life'], + 'beautiful') + + def test_sync_metadata_raise(self): + info_called = [] + get_account_called = [] + sync_container_called = [] + post_account_called = [] + + orig_dict = ({'x-account-meta-life': 'beautiful', + 'x-account-container-count': 1}, + [{'name': 'cont1'}]) + + dest_dict = ({'x-account-container-count': 1}, + [{'name': 'cont1'}]) + self._base_sync_metadata(orig_dict, + dest_dict, + info_called=info_called, + sync_container_called=sync_container_called, + post_account_called=post_account_called, + get_account_called=get_account_called, + raise_post_account=True) + self.assertTrue(info_called) + self.assertIn('ERROR: updating container metadata: orig, ', + info_called) + self.assertFalse(sync_container_called) + + +class TestAccountSync(TestAccountBase): def test_get_swift_auth(self): tenant_name = 'foo1' ret = self.accounts_cls.get_swift_auth( @@ -82,6 +240,11 @@ class TestAccount(tests.units.base.TestCase): def test_sync_account(self): ret = [] + def get_account(*args, **kwargs): + return ({'x-account-container-count': len(fakes.CONTAINERS_LIST)}, + [x[0] for x in fakes.CONTAINERS_LIST]) + self.stubs.Set(swiftclient, 'get_account', get_account) + class Containers(object): def sync(*args, **kwargs): ret.append(args)