stackforge
/
swiftsync
Code Issues Proposed changes

Add reverse option and fix pep8 errors 

Add the reverse option and fix some pep8 errors along the way to have the gate passing.

Change-Id: I6d23ce883123fd66ebb030e373ac011a9e1baa46
This commit is contained in:
James Downs 2015-01-01 15:06:35 -08:00 committed by Chmouel Boudjnah
parent ea4c2c2903
commit 89c639e38a
14 changed files with 132 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
bin/swsync
View File

@ -34,7 +34,15 @@ class Main(object):
dest='log_level', dest='log_level',
default='info', default='info',
help='Number of containers to distribute objects among') help='Number of containers to distribute objects among')
parser.add_option(
'-R', '--reverse',
action="store_true",
dest='reverse',
default=False,
help='Traverse container list forward or reversed')
self.options, args = parser.parse_args() self.options, args = parser.parse_args()
if self.options:
swsync.utils.REVERSE = self.options.reverse
if args: if args:
conf = swsync.utils.parse_ini(args[0]) conf = swsync.utils.parse_ini(args[0])
else: else:

9
swsync/__init__.py
View File

@ -1 +1,10 @@
# -*- encoding: utf-8 -*- # -*- encoding: utf-8 -*-
"""
:mod:`swsync` -- A massive Swift syncer
===================================
.. automodule:: swsync
:platform: Unix
:synopsis: A massive Swift syncer
"""

14
swsync/accounts.py
View File

@ -24,8 +24,8 @@ import keystoneclient.v2_0.client
import swiftclient import swiftclient
import swsync.containers import swsync.containers
from utils import ConfigurationError from swsync.utils import ConfigurationError
from utils import get_config from swsync.utils import get_config
class Accounts(object): class Accounts(object):
@ -55,10 +55,12 @@ class Accounts(object):
def get_target_tenant_filter(self): def get_target_tenant_filter(self):
"""Returns a set of target tenants from the tenant_list_file. """Returns a set of target tenants from the tenant_list_file.
tenant_list_file is defined in the config file or given as a command tenant_list_file is defined in the config file or given as a command
line argument. line argument.
If tenant_list_file is not defined, returns None (an empty filter). If tenant_list_file is not defined, returns None (an empty filter).
""" """
try: try:
tenant_filter_filename = get_config('sync', 'tenant_filter_file') tenant_filter_filename = get_config('sync', 'tenant_filter_file')
@ -137,7 +139,11 @@ class Accounts(object):
# let's pass it on for the next pass # let's pass it on for the next pass
return return
for container in orig_containers: container_list = iter(orig_containers)
if swsync.utils.REVERSE:
container_list = reversed(orig_containers)
for container in container_list:
logging.info("Syncronizing container %s: %s", logging.info("Syncronizing container %s: %s",
container['name'], container) container['name'], container)
dt1 = datetime.datetime.fromtimestamp(time.time()) dt1 = datetime.datetime.fromtimestamp(time.time())
@ -150,7 +156,7 @@ class Accounts(object):
dt2 = datetime.datetime.fromtimestamp(time.time()) dt2 = datetime.datetime.fromtimestamp(time.time())
rd = dateutil.relativedelta.relativedelta(dt2, dt1) rd = dateutil.relativedelta.relativedelta(dt2, dt1)
#TODO(chmou): use logging # TODO(chmou): use logging
logging.info("%s done: %d hours, %d minutes and %d seconds", logging.info("%s done: %d hours, %d minutes and %d seconds",
container['name'], container['name'],
rd.hours, rd.hours,

24
swsync/filler.py
View File

@ -26,26 +26,23 @@
# of objects/containers store in swift for each account then delete # of objects/containers store in swift for each account then delete
# accounts. # accounts.
import os
import sys
import copy import copy
import logging import logging
import os
import pickle import pickle
import random import random
import string import string
import StringIO import StringIO
import sys
import eventlet
from keystoneclient.exceptions import ClientException as KSClientException
from swiftclient import client as sclient from swiftclient import client as sclient
from swiftclient.client import ClientException from swiftclient.client import ClientException
from keystoneclient.exceptions import ClientException as KSClientException from swsync.utils import get_config
import eventlet
sys.path.append("../") sys.path.append("../")
from utils import get_config
eventlet.patcher.monkey_patch() eventlet.patcher.monkey_patch()
# Some unicode codepoint # Some unicode codepoint
@ -66,8 +63,8 @@ def customize(bstr, mdl):
elif mdl == 1: elif mdl == 1:
return bstr + " s" return bstr + " s"
elif mdl == 2: elif mdl == 2:
return unicode(bstr, 'utf8') + u'_' + u"".\ return (unicode(bstr, 'utf8') + u'_' +
join([random.choice(ucodes) for i in range(3)]) u"".join([random.choice(ucodes) for i in range(3)]))
else: else:
return bstr return bstr
@ -147,8 +144,7 @@ def delete_account_content(acc, user):
in container_infos[1]] in container_infos[1]]
# Delete objects # Delete objects
for obj in object_names: for obj in object_names:
logging.info("\ logging.info("Deleting object %s in container %s for account %s" %
Deleting object %s in container %s for account %s" %
(obj, container, str(acc))) (obj, container, str(acc)))
cnx.delete_object(container, obj) cnx.delete_object(container, obj)
@ -251,7 +247,7 @@ def create_account_meta(cnx):
for i in range(3): for i in range(3):
# python-swiftclient does not quote correctly meta ... need # python-swiftclient does not quote correctly meta ... need
# to investigate why it does not work when key are utf8 # to investigate why it does not work when key are utf8
#meta_keys.extend([customize(m, (i + 1) % 3) for m in # meta_keys.extend([customize(m, (i + 1) % 3) for m in
# map(get_rand_str, ('X-Account-Meta-',) * 1)]) # map(get_rand_str, ('X-Account-Meta-',) * 1)])
meta_keys.extend(map(get_rand_str, ('X-Account-Meta-',) * 3)) meta_keys.extend(map(get_rand_str, ('X-Account-Meta-',) * 3))
meta_values.extend([customize(m, (i + 1) % 3) for m in meta_values.extend([customize(m, (i + 1) % 3) for m in
@ -266,7 +262,7 @@ def fill_swift(pool, created_account, c_amount,
o_amount, fmax, index_containers): o_amount, fmax, index_containers):
cnx = swift_cnx(acc, users[0][0]) cnx = swift_cnx(acc, users[0][0])
# Use the first user we find for fill in the swift account # Use the first user we find for fill in the swift account
#TODO(fbo) must keep track of the account meta # TODO(fbo) must keep track of the account meta
create_account_meta(cnx) create_account_meta(cnx)
create_containers(cnx, acc, c_amount, index_containers) create_containers(cnx, acc, c_amount, index_containers)
create_objects(cnx, acc, o_amount, fmax, index_containers) create_objects(cnx, acc, o_amount, fmax, index_containers)

5
swsync/objects.py
View File

@ -15,6 +15,8 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import logging import logging
import urllib
import urllib2
import eventlet import eventlet
import swift.common.bufferedhttp import swift.common.bufferedhttp
@ -24,10 +26,7 @@ try:
except ImportError: except ImportError:
# Nov2013: swift.common.utils now include a more generic object # Nov2013: swift.common.utils now include a more generic object
from swift.common.utils import FileLikeIter from swift.common.utils import FileLikeIter
from swiftclient import client as swiftclient from swiftclient import client as swiftclient
import urllib
import urllib2
def quote(value, safe='/'): def quote(value, safe='/'):

5
swsync/utils.py
View File

@ -24,6 +24,7 @@ curdir = os.path.abspath(os.path.dirname(__file__))
INIFILE = os.path.abspath(os.path.join(curdir, '..', 'etc', "config.ini")) INIFILE = os.path.abspath(os.path.join(curdir, '..', 'etc', "config.ini"))
SAMPLE_INIFILE = os.path.abspath(os.path.join(curdir, '..', SAMPLE_INIFILE = os.path.abspath(os.path.join(curdir, '..',
'etc', "config.ini-sample")) 'etc', "config.ini-sample"))
REVERSE = False
class ConfigurationError(Exception): class ConfigurationError(Exception):
@ -55,7 +56,7 @@ def parse_ini(inicfg=None):
elif inicfg is None and os.path.exists(INIFILE): elif inicfg is None and os.path.exists(INIFILE):
fp = open(INIFILE) fp = open(INIFILE)
else: else:
raise ConfigurationError("Cannot found inicfg") raise ConfigurationError("Cannot find inicfg")
config = ConfigParser.RawConfigParser() config = ConfigParser.RawConfigParser()
config.readfp(fp) config.readfp(fp)
@ -75,7 +76,7 @@ def get_config(section, option, default=None, _config=None):
section) section)
if CONFIG.has_option(section, option): if CONFIG.has_option(section, option):
return CONFIG.get(section, option) return CONFIG.get(section, option)
elif not default is None: elif default is not None:
return default return default
else: else:
raise ConfigurationError("Invalid configuration, missing " raise ConfigurationError("Invalid configuration, missing "

3
tests/functional/test_middleware_lm.py
View File

@ -20,9 +20,10 @@
# Last-Modified middleware must be installed in the proxy-server # Last-Modified middleware must be installed in the proxy-server
# pipeline. # pipeline.
import swiftclient
import unittest import unittest
import swiftclient
CONF = { CONF = {
'user': ('demo:demo', 'wxcvbn'), 'user': ('demo:demo', 'wxcvbn'),
'auth_url': 'http://192.168.56.101:5000/v2.0', 'auth_url': 'http://192.168.56.101:5000/v2.0',

119
tests/functional/test_syncer.py
View File

@ -24,11 +24,12 @@
# to synchronize the destination swift must own the ResellerAdmin role in # to synchronize the destination swift must own the ResellerAdmin role in
# keystone. # keystone.
import eventlet
import unittest import unittest
import eventlet
from keystoneclient.v2_0 import client as ksclient from keystoneclient.v2_0 import client as ksclient
from swiftclient import client as sclient from swiftclient import client as sclient
from swsync import accounts from swsync import accounts
from swsync import filler from swsync import filler
from swsync.utils import get_config from swsync.utils import get_config
@ -63,19 +64,19 @@ class TestSyncer(unittest.TestCase):
password=self.o_admin_password, password=self.o_admin_password,
tenant_name=self.o_admin_tenant) tenant_name=self.o_admin_tenant)
# Retreive admin (ResellerAdmin) token # Retreive admin (ResellerAdmin) token
(self.o_admin_auth_url, self.o_admin_token) = \ (self.o_admin_auth_url, self.o_admin_token) = (
sclient.Connection(self.o_st, sclient.Connection(self.o_st,
"%s:%s" % (self.o_admin_tenant, "%s:%s" % (self.o_admin_tenant,
self.o_admin_user), self.o_admin_user),
self.o_admin_password, self.o_admin_password,
auth_version=2).get_auth() auth_version=2).get_auth())
# Retreive admin (ResellerAdmin) token # Retreive admin (ResellerAdmin) token
(self.d_admin_auth_url, self.d_admin_token) = \ (self.d_admin_auth_url, self.d_admin_token) = (
sclient.Connection(self.d_st, sclient.Connection(self.d_st,
"%s:%s" % (self.o_admin_tenant, "%s:%s" % (self.o_admin_tenant,
self.o_admin_user), self.o_admin_user),
self.o_admin_password, self.o_admin_password,
auth_version=2).get_auth() auth_version=2).get_auth())
# Instanciate syncer # Instanciate syncer
self.swsync = accounts.Accounts() self.swsync = accounts.Accounts()
@ -88,10 +89,10 @@ class TestSyncer(unittest.TestCase):
yield account, account_id, username yield account, account_id, username
def create_st_account_url(self, account_id): def create_st_account_url(self, account_id):
o_account_url = \ o_account_url = (
self.o_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id self.o_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id)
d_account_url = \ d_account_url = (
self.d_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id self.d_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id)
return o_account_url, d_account_url return o_account_url, d_account_url
def verify_aco_diff(self, alo, ald): def verify_aco_diff(self, alo, ald):
@ -203,16 +204,15 @@ class TestSyncer(unittest.TestCase):
http_conn=cnx) http_conn=cnx)
def test_01_sync_one_empty_account(self): def test_01_sync_one_empty_account(self):
"""one empty account with meta data """One empty account with meta data."""
"""
index = {} index = {}
# create account # create account
self.created = filler.create_swift_account(self.o_ks_client, self.created = filler.create_swift_account(self.o_ks_client,
self.pile, self.pile,
1, 1, index) 1, 1, index)
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
# post meta data on account # post meta data on account
tenant_cnx = sclient.Connection(self.o_st, tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username), "%s:%s" % (account, username),
@ -224,8 +224,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process() self.swsync.process()
# Now verify dest # Now verify dest
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
alo = self.get_account_detail(account_id, alo = self.get_account_detail(account_id,
self.o_admin_token, 'orig') self.o_admin_token, 'orig')
ald = self.get_account_detail(account_id, ald = self.get_account_detail(account_id,
@ -233,16 +233,15 @@ class TestSyncer(unittest.TestCase):
self.verify_aco_diff(alo, ald) self.verify_aco_diff(alo, ald)
def test_02_sync_many_empty_account(self): def test_02_sync_many_empty_account(self):
"""Many empty account with meta data """Many empty account with meta data."""
"""
index = {} index = {}
# Create account # Create account
self.created = filler.create_swift_account(self.o_ks_client, self.created = filler.create_swift_account(self.o_ks_client,
self.pile, self.pile,
3, 1, index) 3, 1, index)
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
# Post meta data on account # Post meta data on account
tenant_cnx = sclient.Connection(self.o_st, tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username), "%s:%s" % (account, username),
@ -254,8 +253,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process() self.swsync.process()
# Now verify dest # Now verify dest
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
alo = self.get_account_detail(account_id, alo = self.get_account_detail(account_id,
self.o_admin_token, 'orig') self.o_admin_token, 'orig')
ald = self.get_account_detail(account_id, ald = self.get_account_detail(account_id,
@ -263,8 +262,7 @@ class TestSyncer(unittest.TestCase):
self.verify_aco_diff(alo, ald) self.verify_aco_diff(alo, ald)
def test_03_sync_many_accounts_with_many_containers_meta(self): def test_03_sync_many_accounts_with_many_containers_meta(self):
"""Many accounts with many containers and container meta data """Many accounts with many containers and container meta data."""
"""
index = {} index = {}
index_container = {} index_container = {}
# Create account # Create account
@ -272,8 +270,8 @@ class TestSyncer(unittest.TestCase):
self.pile, self.pile,
3, 1, index) 3, 1, index)
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st, tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username), "%s:%s" % (account, username),
self.default_user_password, self.default_user_password,
@ -285,8 +283,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process() self.swsync.process()
# Now verify dest # Now verify dest
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
# Verify container listing # Verify container listing
clo = self.list_containers(account_id, clo = self.list_containers(account_id,
self.o_admin_token, 'orig') self.o_admin_token, 'orig')
@ -307,8 +305,7 @@ class TestSyncer(unittest.TestCase):
self.verify_aco_diff(cdo, cdd) self.verify_aco_diff(cdo, cdd)
def test_04_sync_many_accounts_many_containers_and_obj_meta(self): def test_04_sync_many_accounts_many_containers_and_obj_meta(self):
"""Many accounts with many containers and some object """Many accounts with many containers and some object."""
"""
index = {} index = {}
index_container = {} index_container = {}
# Create account # Create account
@ -316,8 +313,8 @@ class TestSyncer(unittest.TestCase):
self.pile, self.pile,
1, 1, index) 1, 1, index)
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st, tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username), "%s:%s" % (account, username),
self.default_user_password, self.default_user_password,
@ -330,8 +327,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process() self.swsync.process()
# Now verify dest # Now verify dest
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
# Verify container listing # Verify container listing
olo = self.list_objects_in_containers(account_id, olo = self.list_objects_in_containers(account_id,
self.o_admin_token, 'orig') self.o_admin_token, 'orig')
@ -363,16 +360,15 @@ class TestSyncer(unittest.TestCase):
self.assertEqual(objd_o[1], objd_d[1]) self.assertEqual(objd_o[1], objd_d[1])
def test_05_account_two_passes(self): def test_05_account_two_passes(self):
"""Account modified two sync passes """Account modified two sync passes."""
"""
index = {} index = {}
# create account # create account
self.created = filler.create_swift_account(self.o_ks_client, self.created = filler.create_swift_account(self.o_ks_client,
self.pile, self.pile,
3, 1, index) 3, 1, index)
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
# post meta data on account # post meta data on account
tenant_cnx = sclient.Connection(self.o_st, tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username), "%s:%s" % (account, username),
@ -384,8 +380,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process() self.swsync.process()
# Add more meta to account # Add more meta to account
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
# Modify meta data on account # Modify meta data on account
tenant_cnx = sclient.Connection(self.o_st, tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username), "%s:%s" % (account, username),
@ -408,8 +404,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process() self.swsync.process()
# Now verify dest # Now verify dest
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
alo = self.get_account_detail(account_id, alo = self.get_account_detail(account_id,
self.o_admin_token, 'orig') self.o_admin_token, 'orig')
ald = self.get_account_detail(account_id, ald = self.get_account_detail(account_id,
@ -417,8 +413,7 @@ class TestSyncer(unittest.TestCase):
self.verify_aco_diff(alo, ald) self.verify_aco_diff(alo, ald)
def test_06_container_two_passes(self): def test_06_container_two_passes(self):
"""Containers modified two sync passes """Containers modified two sync passes."""
"""
index = {} index = {}
index_container = {} index_container = {}
# Create account # Create account
@ -426,8 +421,8 @@ class TestSyncer(unittest.TestCase):
self.pile, self.pile,
3, 1, index) 3, 1, index)
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st, tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username), "%s:%s" % (account, username),
self.default_user_password, self.default_user_password,
@ -439,8 +434,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process() self.swsync.process()
# Modify container in account # Modify container in account
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st, tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username), "%s:%s" % (account, username),
self.default_user_password, self.default_user_password,
@ -475,8 +470,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process() self.swsync.process()
# Now verify dest # Now verify dest
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
# Verify container listing # Verify container listing
clo = self.list_containers(account_id, clo = self.list_containers(account_id,
self.o_admin_token, 'orig') self.o_admin_token, 'orig')
@ -497,8 +492,7 @@ class TestSyncer(unittest.TestCase):
self.verify_aco_diff(cdo, cdd) self.verify_aco_diff(cdo, cdd)
def test_07_object_two_passes(self): def test_07_object_two_passes(self):
"""Objects modified two passes """Objects modified two passes."""
"""
index = {} index = {}
index_container = {} index_container = {}
# Create account # Create account
@ -506,8 +500,8 @@ class TestSyncer(unittest.TestCase):
self.pile, self.pile,
1, 1, index) 1, 1, index)
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st, tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username), "%s:%s" % (account, username),
self.default_user_password, self.default_user_password,
@ -520,8 +514,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process() self.swsync.process()
# Modify objects in containers # Modify objects in containers
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st, tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username), "%s:%s" % (account, username),
self.default_user_password, self.default_user_password,
@ -565,8 +559,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process() self.swsync.process()
# Now verify dest # Now verify dest
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
# Verify container listing # Verify container listing
olo = self.list_objects_in_containers(account_id, olo = self.list_objects_in_containers(account_id,
self.o_admin_token, 'orig') self.o_admin_token, 'orig')
@ -601,8 +595,7 @@ class TestSyncer(unittest.TestCase):
self.assertEqual(objd_o[1], objd_d[1]) self.assertEqual(objd_o[1], objd_d[1])
def test_08_sync_containers_with_last_modified(self): def test_08_sync_containers_with_last_modified(self):
"""Containers with last-modified middleware """Containers with last-modified middleware."""
"""
index = {} index = {}
index_container = {} index_container = {}
# Create account # Create account
@ -612,8 +605,8 @@ class TestSyncer(unittest.TestCase):
# Create container and store new account && container # Create container and store new account && container
account_dest, container_dest = None, None account_dest, container_dest = None, None
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st, tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username), "%s:%s" % (account, username),
self.default_user_password, self.default_user_password,
@ -661,8 +654,8 @@ class TestSyncer(unittest.TestCase):
for k, v in self.created.items(): for k, v in self.created.items():
user_info_list = [user[1] for user in v] user_info_list = [user[1] for user in v]
account_id = k[1] account_id = k[1]
o_account_url, d_account_url = \ o_account_url, d_account_url = (
self.create_st_account_url(account_id) self.create_st_account_url(account_id))
# Remove account content on origin and destination # Remove account content on origin and destination
self.delete_account_cont(o_account_url, self.o_admin_token) self.delete_account_cont(o_account_url, self.o_admin_token)
self.delete_account_cont(d_account_url, self.d_admin_token) self.delete_account_cont(d_account_url, self.d_admin_token)

45
tests/functional/test_syncer_filter.py
View File

@ -24,12 +24,13 @@
# In your config.ini file, you should uncomment the field tenant_filter_file # In your config.ini file, you should uncomment the field tenant_filter_file
# and specify a path to a file where you're allowed to read and write. # and specify a path to a file where you're allowed to read and write.
import eventlet
import random import random
import unittest import unittest
import eventlet
from keystoneclient.v2_0 import client as ksclient from keystoneclient.v2_0 import client as ksclient
from swiftclient import client as sclient from swiftclient import client as sclient
from swsync import accounts from swsync import accounts
from swsync import filler from swsync import filler
from swsync.utils import get_config from swsync.utils import get_config
@ -65,19 +66,19 @@ class TestSyncer(unittest.TestCase):
password=self.o_admin_password, password=self.o_admin_password,
tenant_name=self.o_admin_tenant) tenant_name=self.o_admin_tenant)
# Retreive admin (ResellerAdmin) token # Retreive admin (ResellerAdmin) token
(self.o_admin_auth_url, self.o_admin_token) = \ (self.o_admin_auth_url, self.o_admin_token) = (
sclient.Connection(self.o_st, sclient.Connection(self.o_st,
"%s:%s" % (self.o_admin_tenant, "%s:%s" % (self.o_admin_tenant,
self.o_admin_user), self.o_admin_user),
self.o_admin_password, self.o_admin_password,
auth_version=2).get_auth() auth_version=2).get_auth())
# Retreive admin (ResellerAdmin) token # Retreive admin (ResellerAdmin) token
(self.d_admin_auth_url, self.d_admin_token) = \ (self.d_admin_auth_url, self.d_admin_token) = (
sclient.Connection(self.d_st, sclient.Connection(self.d_st,
"%s:%s" % (self.o_admin_tenant, "%s:%s" % (self.o_admin_tenant,
self.o_admin_user), self.o_admin_user),
self.o_admin_password, self.o_admin_password,
auth_version=2).get_auth() auth_version=2).get_auth())
# Instanciate syncer # Instanciate syncer
self.swsync = accounts.Accounts() self.swsync = accounts.Accounts()
@ -90,15 +91,14 @@ class TestSyncer(unittest.TestCase):
yield account, account_id, username yield account, account_id, username
def create_st_account_url(self, account_id): def create_st_account_url(self, account_id):
o_account_url = \ o_account_url = (
self.o_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id self.o_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id)
d_account_url = \ d_account_url = (
self.d_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id self.d_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id)
return o_account_url, d_account_url return o_account_url, d_account_url
def verify_aco_diff(self, alo, ald): def verify_aco_diff(self, alo, ald):
"""Verify that 2 accounts are similar to validate migration """Verify that 2 accounts are similar to validate migration."""
"""
for k, v in alo[0].items(): for k, v in alo[0].items():
if k not in ('x-timestamp', 'x-trans-id', if k not in ('x-timestamp', 'x-trans-id',
'date', 'last-modified'): 'date', 'last-modified'):
@ -206,8 +206,7 @@ class TestSyncer(unittest.TestCase):
http_conn=cnx) http_conn=cnx)
def test_01_sync_one_of_two_empty_accounts(self): def test_01_sync_one_of_two_empty_accounts(self):
"""create two empty accounts, Sync only one """Create two empty accounts, Sync only one."""
"""
index = {} index = {}
# create account # create account
@ -215,8 +214,8 @@ class TestSyncer(unittest.TestCase):
self.pile, self.pile,
2, 1, index) 2, 1, index)
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
# post meta data on account # post meta data on account
tenant_cnx = sclient.Connection(self.o_st, tenant_cnx = sclient.Connection(self.o_st,
@ -235,22 +234,22 @@ class TestSyncer(unittest.TestCase):
self.swsync.process() self.swsync.process()
# Now verify dest # Now verify dest
for account, account_id, username in \ for account, account_id, username in (
self.extract_created_a_u_iter(self.created): self.extract_created_a_u_iter(self.created)):
alo = self.get_account_detail(account_id, alo = self.get_account_detail(account_id,
self.o_admin_token, 'orig') self.o_admin_token, 'orig')
ald = self.get_account_detail(account_id, ald = self.get_account_detail(account_id,
self.d_admin_token, 'dest') self.d_admin_token, 'dest')
if account == t_account: if account == t_account:
self.verify_aco_diff(alo, ald) self.verify_aco_diff(alo, ald)
def tearDown(self): def tearDown(self):
if self.created: if self.created:
for k, v in self.created.items(): for k, v in self.created.items():
user_info_list = [user[1] for user in v] user_info_list = [user[1] for user in v]
account_id = k[1] account_id = k[1]
o_account_url, d_account_url = \ o_account_url, d_account_url = (
self.create_st_account_url(account_id) self.create_st_account_url(account_id))
# Remove account content on origin and destination # Remove account content on origin and destination
self.delete_account_cont(o_account_url, self.o_admin_token) self.delete_account_cont(o_account_url, self.o_admin_token)
self.delete_account_cont(d_account_url, self.d_admin_token) self.delete_account_cont(d_account_url, self.d_admin_token)

4
tests/units/base.py
View File

@ -18,10 +18,10 @@
"""Test base classes imported from ceilometer. """Test base classes imported from ceilometer.
""" """
import unittest2
import mox import mox
import stubout import stubout
import unittest2
from swsync import utils from swsync import utils

4
tests/units/test_accounts.py
View File

@ -301,12 +301,12 @@ class TestAccountSync(TestAccountBase):
self.accounts_cls.container_cls = Containers() self.accounts_cls.container_cls = Containers()
def get_account(*args, **kwargs): def get_account(*args, **kwargs):
#ORIG # ORIG
if len(ret) == 0: if len(ret) == 0:
ret.append("TESTED") ret.append("TESTED")
return ({'x-account-container-count': 1}, return ({'x-account-container-count': 1},
[{'name': 'foo'}]) [{'name': 'foo'}])
#DEST # DEST
else: else:
return ({'x-account-container-count': 2}, return ({'x-account-container-count': 2},
[{'name': 'foo', 'name': 'bar'}]) [{'name': 'foo', 'name': 'bar'}])

5
tests/units/test_containers.py
View File

@ -20,7 +20,6 @@ import urlparse
import swiftclient import swiftclient
import swsync.containers import swsync.containers
import tests.units.base as test_base import tests.units.base as test_base
import tests.units.fakes as fakes import tests.units.fakes as fakes
@ -391,12 +390,12 @@ class TestContainers(TestContainersBase):
called_on_dest = [] called_on_dest = []
def get_container(*args, **kwargs): def get_container(*args, **kwargs):
#ORIG # ORIG
if len(called) == 0: if len(called) == 0:
called.append("TESTED") called.append("TESTED")
return ({}, [{'name': 'PARISESTMAGIQUE', return ({}, [{'name': 'PARISESTMAGIQUE',
'last_modified': '2010'}]) 'last_modified': '2010'}])
#DEST # DEST
else: else:
called_on_dest.append("TESTED") called_on_dest.append("TESTED")
raise swiftclient.client.ClientException("TESTED") raise swiftclient.client.ClientException("TESTED")

9
tests/units/test_filler.py
View File

@ -16,19 +16,16 @@
# under the License. # under the License.
import eventlet import eventlet
import swiftclient
from keystoneclient.exceptions import ClientException as KSClientException
from fakes import FakeKSClient from fakes import FakeKSClient
from fakes import FakeKSTenant from fakes import FakeKSTenant
from fakes import FakeKSUser from fakes import FakeKSUser
from fakes import FakeSWConnection from fakes import FakeSWConnection
from keystoneclient.exceptions import ClientException as KSClientException
from tests.units import base import swiftclient
from swsync import filler from swsync import filler
from swsync import utils from swsync import utils
from tests.units import base
class TestFiller(base.TestCase): class TestFiller(base.TestCase):

1
tests/units/test_middleware_lm.py
View File

@ -20,6 +20,7 @@
import unittest import unittest
from middlewares import last_modified as middleware from middlewares import last_modified as middleware
import swift.common.swob as swob import swift.common.swob as swob