stackforge
/
swiftsync
Code Issues Proposed changes

Merge pull request #53 from chmouel/account-metadata-sync 

Add account metadata sync.
This commit is contained in:
Fabien Boucher 2013-04-11 05:28:55 -07:00
parent 7601f73634 2ae84b0f96
commit 85490fbbe5
2 changed files with 223 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
swsync/accounts.py
View File

@ -16,6 +16,7 @@
# under the License.
import datetime
import logging
import os
import time
import dateutil.relativedelta
@ -51,35 +52,80 @@ class Accounts(object):
password=password,
tenant_name=tenant_name)
def account_headers_clean(self, account_headers, to_null=False):
ret = {}
for key, value in account_headers.iteritems():
if key.startswith('x-account-meta'):
if to_null:
value = ''
ret[key] = value
return ret
def sync_account(self, orig_storage_url, orig_token,
dest_storage_url, dest_token):
"""Sync a single account with url/tok to dest_url/dest_tok."""
orig_storage_cnx = swiftclient.http_connection(orig_storage_url)
dest_storage_cnx = swiftclient.http_connection(dest_storage_url)
account_id = os.path.basename(orig_storage_url.replace("AUTH_", ''))
try:
orig_stats, orig_containers = (
orig_account_headers, orig_containers = (
swiftclient.get_account(None, orig_token,
http_conn=orig_storage_cnx,
full_listing=True))
dest_stats, dest_containers = (
dest_account_headers, dest_containers = (
swiftclient.get_account(None, dest_token,
http_conn=dest_storage_cnx,
full_listing=True))
except(swiftclient.client.ClientException), e:
logging.info("error getting containeaccount: %s, %s" % (
orig_storage_url, e.http_reason))
logging.info("error getting account: %s, %s" % (
account_id, e.http_reason))
return
if int(dest_stats['x-account-container-count']) > \
int(orig_stats['x-account-container-count']):
if int(dest_account_headers['x-account-container-count']) > \
int(orig_account_headers['x-account-container-count']):
self.container_cls.delete_container(dest_storage_cnx,
dest_token,
orig_containers,
dest_containers)
do_headers = False
if len(dest_account_headers) != len(orig_account_headers):
do_headers = True
else:
for k, v in orig_account_headers.iteritems():
if not k.startswith('x-account-meta'):
continue
if k not in dest_account_headers:
do_headers = True
elif dest_account_headers[k] != v:
do_headers = True
if do_headers:
orig_metadata_headers = self.account_headers_clean(
orig_account_headers)
dest_metadata_headers = self.account_headers_clean(
dest_account_headers, to_null=True)
new_headers = dict(dest_metadata_headers.items() +
orig_metadata_headers.items())
try:
swiftclient.post_account(
"", dest_token, new_headers,
http_conn=dest_storage_cnx,
)
logging.info("HEADER: sync headers: %s" % (account_id))
except(swiftclient.client.ClientException), e:
logging.info("ERROR: updating container metadata: %s, %s" % (
account_id, e.http_reason))
# We don't pass on because since the server was busy
# let's pass it on for the next pass
return
for container in orig_containers:
logging.info("Syncronizing %s: %s", container['name'], container)
logging.info("Syncronizing container %s: %s",
container['name'], container)
dt1 = datetime.datetime.fromtimestamp(time.time())
self.container_cls.sync(orig_storage_cnx,
orig_storage_url,

177
tests/units/test_accounts.py
View File

@ -24,25 +24,183 @@ import tests.units.base
import tests.units.fakes as fakes
class TestAccount(tests.units.base.TestCase):
class TestAccountBase(tests.units.base.TestCase):
def setUp(self):
super(TestAccount, self).setUp()
super(TestAccountBase, self).setUp()
self.accounts_cls = swsync.accounts.Accounts()
self._stubs()
def get_account(self, *args, **kwargs):
return ({'x-account-container-count': len(fakes.CONTAINERS_LIST)},
[x[0] for x in fakes.CONTAINERS_LIST])
def _stubs(self):
self.stubs.Set(keystoneclient.v2_0, 'client', fakes.FakeKS)
self.stubs.Set(swiftclient.client, 'Connection',
fakes.FakeSWConnection)
self.stubs.Set(swsync.accounts, 'get_config', fakes.fake_get_config)
self.stubs.Set(swiftclient, 'get_account', self.get_account)
self.stubs.Set(swiftclient, 'http_connection',
fakes.FakeSWClient.http_connection)
class TestAccountSyncMetadata(TestAccountBase):
def _base_sync_metadata(self, orig_dict={},
dest_dict={},
get_account_called=[],
post_account_called=[],
info_called=[],
sync_container_called=[],
raise_post_account=False):
def fake_info(msg, *args):
info_called.append(msg)
self.stubs.Set(logging, 'info', fake_info)
def get_account(self, *args, **kwargs):
if len(get_account_called) == 0:
get_account_called.append(args)
return orig_dict
else:
get_account_called.append(args)
return dest_dict
self.stubs.Set(swiftclient, 'get_account', get_account)
def post_account(url, token, headers, **kwargs):
post_account_called.append(headers)
if raise_post_account:
raise swiftclient.client.ClientException("Error in testing")
self.stubs.Set(swiftclient, 'post_account', post_account)
class Containers(object):
def sync(*args, **kwargs):
sync_container_called.append(args)
self.accounts_cls.container_cls = Containers()
self.accounts_cls.sync_account("http://orig", "otoken",
"http://dest", "dtoken")
def test_sync_metadata_delete_dest(self):
get_account_called = []
sync_container_called = []
post_account_called = []
info_called = []
orig_dict = ({'x-account-meta-life': 'beautiful',
'x-account-container-count': 1},
[{'name': 'cont1'}])
dest_dict = ({'x-account-meta-vita': 'bella',
'x-account-container-count': 1},
[{'name': 'cont1'}])
self._base_sync_metadata(orig_dict,
dest_dict,
info_called=info_called,
sync_container_called=sync_container_called,
post_account_called=post_account_called,
get_account_called=get_account_called)
self.assertEquals(len(sync_container_called), 1)
self.assertEquals(len(get_account_called), 2)
self.assertTrue(info_called)
self.assertIn('x-account-meta-life',
post_account_called[0])
self.assertEqual(post_account_called[0]['x-account-meta-life'],
'beautiful')
self.assertIn('x-account-meta-vita',
post_account_called[0])
self.assertEqual(post_account_called[0]['x-account-meta-vita'],
'')
def test_sync_metadata_update_dest(self):
get_account_called = []
sync_container_called = []
post_account_called = []
info_called = []
orig_dict = ({'x-account-meta-life': 'beautiful',
'x-account-container-count': 1},
[{'name': 'cont1'}])
dest_dict = ({'x-account-meta-life': 'bella',
'x-account-container-count': 1},
[{'name': 'cont1'}])
self._base_sync_metadata(orig_dict,
dest_dict,
info_called=info_called,
sync_container_called=sync_container_called,
post_account_called=post_account_called,
get_account_called=get_account_called)
self.assertEquals(len(sync_container_called), 1)
self.assertEquals(len(get_account_called), 2)
self.assertTrue(info_called)
self.assertIn('x-account-meta-life',
post_account_called[0])
self.assertEqual(post_account_called[0]['x-account-meta-life'],
'beautiful')
self.assertIn('x-account-meta-life',
post_account_called[0])
self.assertEqual(post_account_called[0]['x-account-meta-life'],
'beautiful')
def test_sync_metadata_add_to_dest(self):
info_called = []
get_account_called = []
sync_container_called = []
post_account_called = []
orig_dict = ({'x-account-meta-life': 'beautiful',
'x-account-container-count': 1},
[{'name': 'cont1'}])
dest_dict = ({'x-account-container-count': 1},
[{'name': 'cont1'}])
self._base_sync_metadata(orig_dict,
dest_dict,
info_called=info_called,
sync_container_called=sync_container_called,
post_account_called=post_account_called,
get_account_called=get_account_called)
self.assertEquals(len(sync_container_called), 1)
self.assertEquals(len(get_account_called), 2)
self.assertTrue(info_called)
self.assertIn('x-account-meta-life',
post_account_called[0])
self.assertEqual(post_account_called[0]['x-account-meta-life'],
'beautiful')
self.assertIn('x-account-meta-life',
post_account_called[0])
self.assertEqual(post_account_called[0]['x-account-meta-life'],
'beautiful')
def test_sync_metadata_raise(self):
info_called = []
get_account_called = []
sync_container_called = []
post_account_called = []
orig_dict = ({'x-account-meta-life': 'beautiful',
'x-account-container-count': 1},
[{'name': 'cont1'}])
dest_dict = ({'x-account-container-count': 1},
[{'name': 'cont1'}])
self._base_sync_metadata(orig_dict,
dest_dict,
info_called=info_called,
sync_container_called=sync_container_called,
post_account_called=post_account_called,
get_account_called=get_account_called,
raise_post_account=True)
self.assertTrue(info_called)
self.assertIn('ERROR: updating container metadata: orig, ',
info_called)
self.assertFalse(sync_container_called)
class TestAccountSync(TestAccountBase):
def test_get_swift_auth(self):
tenant_name = 'foo1'
ret = self.accounts_cls.get_swift_auth(
@ -82,6 +240,11 @@ class TestAccount(tests.units.base.TestCase):
def test_sync_account(self):
ret = []
def get_account(*args, **kwargs):
return ({'x-account-container-count': len(fakes.CONTAINERS_LIST)},
[x[0] for x in fakes.CONTAINERS_LIST])
self.stubs.Set(swiftclient, 'get_account', get_account)
class Containers(object):
def sync(*args, **kwargs):
ret.append(args)