stackforge
/
swiftsync
Code Issues Proposed changes

Add containers class tests.

This commit is contained in:
Chmouel Boudjnah 2013-03-27 16:55:16 +01:00
parent d20ac73b40
commit 369f02ee98
5 changed files with 143 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
sync/accounts.py
View File

@ -29,6 +29,7 @@ class Accounts(object):
"""Process Keystone Accounts."""
def __init__(self):
self.keystone_cnx = None
self.container_cls = containers.Containers()
def get_swift_auth(self, auth_url, tenant, user, password):
"""Get swift connexion from args"""
@ -60,16 +61,15 @@ class Accounts(object):
http_conn=orig_storage_cnx,
full_listing=True))
container_cls = containers.Containers()
for container in orig_containers:
print container
dt1 = datetime.datetime.fromtimestamp(time.time())
container_cls.sync(orig_storage_cnx,
orig_storage_url,
orig_token,
dest_storage_cnx,
dest_storage_url, dest_token,
container['name'])
self.container_cls.sync(orig_storage_cnx,
orig_storage_url,
orig_token,
dest_storage_cnx,
dest_storage_url, dest_token,
container['name'])
dt2 = datetime.datetime.fromtimestamp(time.time())
rd = dateutil.relativedelta.relativedelta(dt2, dt1)

6
sync/containers.py
View File

@ -25,10 +25,12 @@ class Containers(object):
"""Containers syncornization."""
def __init__(self):
self.max_gthreads = int(get_config("sync", "max_gthreads"))
self.objects_cls = sync_object
def sync(self, orig_storage_cnx, orig_storage_url,
orig_token, dest_storage_cnx, dest_storage_url, dest_token,
container_name):
orig_container_stats, orig_objects = swiftclient.get_container(
None, orig_token, container_name, http_conn=orig_storage_cnx,
)
@ -53,8 +55,8 @@ class Containers(object):
set1 = set((x['last_modified'], x['name']) for x in orig_objects)
set2 = set((x['last_modified'], x['name']) for x in dest_objects)
diff = set1 - set2
if not diff:
return
@ -62,7 +64,7 @@ class Containers(object):
pile = eventlet.GreenPile(pool)
for obj in diff:
print obj
pile.spawn(sync_object,
pile.spawn(self.objects_cls,
orig_storage_url,
orig_token,
dest_storage_url,

48
tests/fakes.py
View File

@ -26,28 +26,38 @@ TENANTS_LIST = {'foo1': {'id': uuid.uuid4().hex},
'foo2': {'id': uuid.uuid4().hex},
'foo3': {'id': uuid.uuid4().hex}}
get_container = lambda x: {'name': x,
'bytes': random.randint(1, 5000),
'count': random.randint(1, 50),
'bytes': random.randint(1, 5000)}
get_object = lambda x: {'bytes': random.randint(1, 5000),
'last_modified': str(datetime.datetime.now()),
'name': x}
def gen_random_lastmodified():
delta = datetime.timedelta(seconds=random.randint(1, 60))
return str(datetime.datetime.now() + delta)
def gen_container(x):
return {'name': x,
'bytes': random.randint(1, 5000),
'count': random.randint(1, 50),
'bytes': random.randint(1, 5000)}
def gen_object(x):
return {'bytes': random.randint(1, 5000),
'last_modified': gen_random_lastmodified(),
'name': x}
CONTAINERS_LIST = [
(get_container('cont1'),
[get_object('obj%s' % (x)) for x in xrange(random.randint(1, 100))]),
(get_container('cont2'),
[get_object('obj%s' % (x)) for x in xrange(random.randint(1, 100))]),
(get_container('cont3'),
[get_object('obj%s' % (x)) for x in xrange(random.randint(1, 100))]),
(gen_container('cont1'),
[gen_object('obj%s' % (x)) for x in xrange(random.randint(1, 10))]),
(gen_container('cont2'),
[gen_object('obj%s' % (x)) for x in xrange(random.randint(1, 10))]),
(gen_container('cont3'),
[gen_object('obj%s' % (x)) for x in xrange(random.randint(1, 10))]),
]
CONTAINER_HEADERS = {
'x-foo': 'true', 'x-bar': 'bar',
'x-container-object-count': _,
'x-container-bytes-used': _,
'x-trans-id': _,
'x-container-object-count': '10',
'x-container-bytes-used': '1000000',
'x-trans-id': 'transid',
}
CONFIGDICT = {'auth':
@ -81,12 +91,6 @@ class FakeSWClient(object):
def http_connection(url):
return (urlparse.urlparse(url), None)
@staticmethod
def get_container(_, token, name, **kwargs):
for clist in CONTAINERS_LIST:
if clist[0]['name'] == name:
return (CONTAINER_HEADERS, clist[0])
@staticmethod
def get_account(*args, **kwargs):
return (('x-foo', 'x-bar'),

3
tests/test_accounts.py
View File

@ -79,8 +79,7 @@ class TestAccount(test_base.TestCase):
class Containers(object):
def sync(*args, **kwargs):
ret.append(args)
sync.accounts.containers.Containers = Containers
self.accounts_cls.container_cls = Containers()
tenant_name = TENANTS_LIST.keys()[0]
orig_storage_url = "%s/AUTH_%s" % (STORAGE_ORIG,

105
tests/test_containers.py Normal file
View File

@ -0,0 +1,105 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
#
# Author: Chmouel Boudjnah <chmouel@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import urlparse
import swiftclient
import sync.containers
import sync.objects
import base as test_base
from fakes import STORAGE_ORIG, STORAGE_DEST, TENANTS_LIST, \
CONTAINERS_LIST, CONTAINER_HEADERS, gen_object
class TestContainers(test_base.TestCase):
def setUp(self):
super(TestContainers, self).setUp()
self.container_cls = sync.containers.Containers()
self.tenant_name = 'foo1'
self.tenant_id = TENANTS_LIST[self.tenant_name]['id']
self.orig_storage_url = "%s/AUTH_%s" % (STORAGE_ORIG, self.tenant_id)
self.orig_storage_cnx = (urlparse.urlparse(self.orig_storage_url),
None)
self.dest_storage_url = "%s/AUTH_%s" % (STORAGE_DEST, self.tenant_id)
self.dest_storage_cnx = (urlparse.urlparse(self.dest_storage_url),
None)
def test_sync_when_container_nothere(self):
get_cnt_called = []
def put_container(*args, **kwargs):
get_cnt_called.append(args)
def head_container(*args, **kwargs):
raise swiftclient.client.ClientException("Not Here")
def get_container(_, token, name, **kwargs):
for clist in CONTAINERS_LIST:
if clist[0]['name'] == name:
return (CONTAINER_HEADERS, clist[1])
self.stubs.Set(swiftclient, 'get_container', get_container)
self.stubs.Set(swiftclient, 'put_container', put_container)
self.stubs.Set(swiftclient, 'head_container', head_container)
self.container_cls.sync(
self.orig_storage_cnx, self.orig_storage_url, "token",
self.dest_storage_cnx, self.dest_storage_url, "token",
"cont1"
)
self.assertEqual(len(get_cnt_called), 1)
def test_sync(self):
get_cnt_called = []
sync_object_called = []
def head_container(*args, **kwargs):
pass
def get_container(*args, **kwargs):
# MASTER
if not get_cnt_called:
cont = CONTAINERS_LIST[0][0]
objects = list(CONTAINERS_LIST[0][1])
objects.append(gen_object('NEWOBJ'))
get_cnt_called.append((cont, objects))
# TARGET
else:
cont = CONTAINERS_LIST[0][0]
objects = list(CONTAINERS_LIST[0][1])
return (cont, objects)
def sync_object(*args, **kwargs):
sync_object_called.append(args)
self.stubs.Set(swiftclient, 'head_container', head_container)
self.stubs.Set(swiftclient, 'get_container', get_container)
self.container_cls.objects_cls = sync_object
self.container_cls.sync(
self.orig_storage_cnx,
self.orig_storage_url,
"token",
self.dest_storage_cnx,
self.dest_storage_url,
"token",
"cont1")
self.assertEqual(sync_object_called[0][-1][1], 'NEWOBJ')