stackforge
/
swiftsync
Code Issues Proposed changes

Add reverse option and fix pep8 errors 

Add the reverse option and fix some pep8 errors along the way to have the gate passing.

Change-Id: I6d23ce883123fd66ebb030e373ac011a9e1baa46
This commit is contained in:
James Downs 2015-01-01 15:06:35 -08:00 committed by Chmouel Boudjnah
parent ea4c2c2903
commit 89c639e38a
14 changed files with 132 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
bin/swsync
View File

@ -34,7 +34,15 @@ class Main(object):
dest='log_level',
default='info',
help='Number of containers to distribute objects among')
parser.add_option(
'-R', '--reverse',
action="store_true",
dest='reverse',
default=False,
help='Traverse container list forward or reversed')
self.options, args = parser.parse_args()
if self.options:
swsync.utils.REVERSE = self.options.reverse
if args:
conf = swsync.utils.parse_ini(args[0])
else:

9
swsync/__init__.py
View File

@ -1 +1,10 @@
# -*- encoding: utf-8 -*-
"""
:mod:`swsync` -- A massive Swift syncer
===================================
.. automodule:: swsync
:platform: Unix
:synopsis: A massive Swift syncer
"""

14
swsync/accounts.py
View File

@ -24,8 +24,8 @@ import keystoneclient.v2_0.client
import swiftclient
import swsync.containers
from utils import ConfigurationError
from utils import get_config
from swsync.utils import ConfigurationError
from swsync.utils import get_config
class Accounts(object):
@ -55,10 +55,12 @@ class Accounts(object):
def get_target_tenant_filter(self):
"""Returns a set of target tenants from the tenant_list_file.
tenant_list_file is defined in the config file or given as a command
line argument.
If tenant_list_file is not defined, returns None (an empty filter).
"""
try:
tenant_filter_filename = get_config('sync', 'tenant_filter_file')
@ -137,7 +139,11 @@ class Accounts(object):
# let's pass it on for the next pass
return
for container in orig_containers:
container_list = iter(orig_containers)
if swsync.utils.REVERSE:
container_list = reversed(orig_containers)
for container in container_list:
logging.info("Syncronizing container %s: %s",
container['name'], container)
dt1 = datetime.datetime.fromtimestamp(time.time())
@ -150,7 +156,7 @@ class Accounts(object):
dt2 = datetime.datetime.fromtimestamp(time.time())
rd = dateutil.relativedelta.relativedelta(dt2, dt1)
#TODO(chmou): use logging
# TODO(chmou): use logging
logging.info("%s done: %d hours, %d minutes and %d seconds",
container['name'],
rd.hours,

24
swsync/filler.py
View File

@ -26,26 +26,23 @@
# of objects/containers store in swift for each account then delete
# accounts.
import os
import sys
import copy
import logging
import os
import pickle
import random
import string
import StringIO
import sys
import eventlet
from keystoneclient.exceptions import ClientException as KSClientException
from swiftclient import client as sclient
from swiftclient.client import ClientException
from keystoneclient.exceptions import ClientException as KSClientException
import eventlet
from swsync.utils import get_config
sys.path.append("../")
from utils import get_config
eventlet.patcher.monkey_patch()
# Some unicode codepoint
@ -66,8 +63,8 @@ def customize(bstr, mdl):
elif mdl == 1:
return bstr + " s"
elif mdl == 2:
return unicode(bstr, 'utf8') + u'_' + u"".\
join([random.choice(ucodes) for i in range(3)])
return (unicode(bstr, 'utf8') + u'_' +
u"".join([random.choice(ucodes) for i in range(3)]))
else:
return bstr
@ -147,8 +144,7 @@ def delete_account_content(acc, user):
in container_infos[1]]
# Delete objects
for obj in object_names:
logging.info("\
Deleting object %s in container %s for account %s" %
logging.info("Deleting object %s in container %s for account %s" %
(obj, container, str(acc)))
cnx.delete_object(container, obj)
@ -251,7 +247,7 @@ def create_account_meta(cnx):
for i in range(3):
# python-swiftclient does not quote correctly meta ... need
# to investigate why it does not work when key are utf8
#meta_keys.extend([customize(m, (i + 1) % 3) for m in
# meta_keys.extend([customize(m, (i + 1) % 3) for m in
# map(get_rand_str, ('X-Account-Meta-',) * 1)])
meta_keys.extend(map(get_rand_str, ('X-Account-Meta-',) * 3))
meta_values.extend([customize(m, (i + 1) % 3) for m in
@ -266,7 +262,7 @@ def fill_swift(pool, created_account, c_amount,
o_amount, fmax, index_containers):
cnx = swift_cnx(acc, users[0][0])
# Use the first user we find for fill in the swift account
#TODO(fbo) must keep track of the account meta
# TODO(fbo) must keep track of the account meta
create_account_meta(cnx)
create_containers(cnx, acc, c_amount, index_containers)
create_objects(cnx, acc, o_amount, fmax, index_containers)

5
swsync/objects.py
View File

@ -15,6 +15,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import logging
import urllib
import urllib2
import eventlet
import swift.common.bufferedhttp
@ -24,10 +26,7 @@ try:
except ImportError:
# Nov2013: swift.common.utils now include a more generic object
from swift.common.utils import FileLikeIter
from swiftclient import client as swiftclient
import urllib
import urllib2
def quote(value, safe='/'):

5
swsync/utils.py
View File

@ -24,6 +24,7 @@ curdir = os.path.abspath(os.path.dirname(__file__))
INIFILE = os.path.abspath(os.path.join(curdir, '..', 'etc', "config.ini"))
SAMPLE_INIFILE = os.path.abspath(os.path.join(curdir, '..',
'etc', "config.ini-sample"))
REVERSE = False
class ConfigurationError(Exception):
@ -55,7 +56,7 @@ def parse_ini(inicfg=None):
elif inicfg is None and os.path.exists(INIFILE):
fp = open(INIFILE)
else:
raise ConfigurationError("Cannot found inicfg")
raise ConfigurationError("Cannot find inicfg")
config = ConfigParser.RawConfigParser()
config.readfp(fp)
@ -75,7 +76,7 @@ def get_config(section, option, default=None, _config=None):
section)
if CONFIG.has_option(section, option):
return CONFIG.get(section, option)
elif not default is None:
elif default is not None:
return default
else:
raise ConfigurationError("Invalid configuration, missing "

3
tests/functional/test_middleware_lm.py
View File

@ -20,9 +20,10 @@
# Last-Modified middleware must be installed in the proxy-server
# pipeline.
import swiftclient
import unittest
import swiftclient
CONF = {
'user': ('demo:demo', 'wxcvbn'),
'auth_url': 'http://192.168.56.101:5000/v2.0',

119
tests/functional/test_syncer.py
View File

@ -24,11 +24,12 @@
# to synchronize the destination swift must own the ResellerAdmin role in
# keystone.
import eventlet
import unittest
import eventlet
from keystoneclient.v2_0 import client as ksclient
from swiftclient import client as sclient
from swsync import accounts
from swsync import filler
from swsync.utils import get_config
@ -63,19 +64,19 @@ class TestSyncer(unittest.TestCase):
password=self.o_admin_password,
tenant_name=self.o_admin_tenant)
# Retreive admin (ResellerAdmin) token
(self.o_admin_auth_url, self.o_admin_token) = \
(self.o_admin_auth_url, self.o_admin_token) = (
sclient.Connection(self.o_st,
"%s:%s" % (self.o_admin_tenant,
self.o_admin_user),
self.o_admin_password,
auth_version=2).get_auth()
auth_version=2).get_auth())
# Retreive admin (ResellerAdmin) token
(self.d_admin_auth_url, self.d_admin_token) = \
(self.d_admin_auth_url, self.d_admin_token) = (
sclient.Connection(self.d_st,
"%s:%s" % (self.o_admin_tenant,
self.o_admin_user),
self.o_admin_password,
auth_version=2).get_auth()
auth_version=2).get_auth())
# Instanciate syncer
self.swsync = accounts.Accounts()
@ -88,10 +89,10 @@ class TestSyncer(unittest.TestCase):
yield account, account_id, username
def create_st_account_url(self, account_id):
o_account_url = \
self.o_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id
d_account_url = \
self.d_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id
o_account_url = (
self.o_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id)
d_account_url = (
self.d_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id)
return o_account_url, d_account_url
def verify_aco_diff(self, alo, ald):
@ -203,16 +204,15 @@ class TestSyncer(unittest.TestCase):
http_conn=cnx)
def test_01_sync_one_empty_account(self):
"""one empty account with meta data
"""
"""One empty account with meta data."""
index = {}
# create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
1, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
# post meta data on account
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
@ -224,8 +224,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
alo = self.get_account_detail(account_id,
self.o_admin_token, 'orig')
ald = self.get_account_detail(account_id,
@ -233,16 +233,15 @@ class TestSyncer(unittest.TestCase):
self.verify_aco_diff(alo, ald)
def test_02_sync_many_empty_account(self):
"""Many empty account with meta data
"""
"""Many empty account with meta data."""
index = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
3, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
# Post meta data on account
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
@ -254,8 +253,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
alo = self.get_account_detail(account_id,
self.o_admin_token, 'orig')
ald = self.get_account_detail(account_id,
@ -263,8 +262,7 @@ class TestSyncer(unittest.TestCase):
self.verify_aco_diff(alo, ald)
def test_03_sync_many_accounts_with_many_containers_meta(self):
"""Many accounts with many containers and container meta data
"""
"""Many accounts with many containers and container meta data."""
index = {}
index_container = {}
# Create account
@ -272,8 +270,8 @@ class TestSyncer(unittest.TestCase):
self.pile,
3, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
@ -285,8 +283,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
# Verify container listing
clo = self.list_containers(account_id,
self.o_admin_token, 'orig')
@ -307,8 +305,7 @@ class TestSyncer(unittest.TestCase):
self.verify_aco_diff(cdo, cdd)
def test_04_sync_many_accounts_many_containers_and_obj_meta(self):
"""Many accounts with many containers and some object
"""
"""Many accounts with many containers and some object."""
index = {}
index_container = {}
# Create account
@ -316,8 +313,8 @@ class TestSyncer(unittest.TestCase):
self.pile,
1, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
@ -330,8 +327,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
# Verify container listing
olo = self.list_objects_in_containers(account_id,
self.o_admin_token, 'orig')
@ -363,16 +360,15 @@ class TestSyncer(unittest.TestCase):
self.assertEqual(objd_o[1], objd_d[1])
def test_05_account_two_passes(self):
"""Account modified two sync passes
"""
"""Account modified two sync passes."""
index = {}
# create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
3, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
# post meta data on account
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
@ -384,8 +380,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process()
# Add more meta to account
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
# Modify meta data on account
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
@ -408,8 +404,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
alo = self.get_account_detail(account_id,
self.o_admin_token, 'orig')
ald = self.get_account_detail(account_id,
@ -417,8 +413,7 @@ class TestSyncer(unittest.TestCase):
self.verify_aco_diff(alo, ald)
def test_06_container_two_passes(self):
"""Containers modified two sync passes
"""
"""Containers modified two sync passes."""
index = {}
index_container = {}
# Create account
@ -426,8 +421,8 @@ class TestSyncer(unittest.TestCase):
self.pile,
3, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
@ -439,8 +434,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process()
# Modify container in account
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
@ -475,8 +470,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
# Verify container listing
clo = self.list_containers(account_id,
self.o_admin_token, 'orig')
@ -497,8 +492,7 @@ class TestSyncer(unittest.TestCase):
self.verify_aco_diff(cdo, cdd)
def test_07_object_two_passes(self):
"""Objects modified two passes
"""
"""Objects modified two passes."""
index = {}
index_container = {}
# Create account
@ -506,8 +500,8 @@ class TestSyncer(unittest.TestCase):
self.pile,
1, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
@ -520,8 +514,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process()
# Modify objects in containers
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
@ -565,8 +559,8 @@ class TestSyncer(unittest.TestCase):
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
# Verify container listing
olo = self.list_objects_in_containers(account_id,
self.o_admin_token, 'orig')
@ -601,8 +595,7 @@ class TestSyncer(unittest.TestCase):
self.assertEqual(objd_o[1], objd_d[1])
def test_08_sync_containers_with_last_modified(self):
"""Containers with last-modified middleware
"""
"""Containers with last-modified middleware."""
index = {}
index_container = {}
# Create account
@ -612,8 +605,8 @@ class TestSyncer(unittest.TestCase):
# Create container and store new account && container
account_dest, container_dest = None, None
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
@ -661,8 +654,8 @@ class TestSyncer(unittest.TestCase):
for k, v in self.created.items():
user_info_list = [user[1] for user in v]
account_id = k[1]
o_account_url, d_account_url = \
self.create_st_account_url(account_id)
o_account_url, d_account_url = (
self.create_st_account_url(account_id))
# Remove account content on origin and destination
self.delete_account_cont(o_account_url, self.o_admin_token)
self.delete_account_cont(d_account_url, self.d_admin_token)

45
tests/functional/test_syncer_filter.py
View File

@ -24,12 +24,13 @@
# In your config.ini file, you should uncomment the field tenant_filter_file
# and specify a path to a file where you're allowed to read and write.
import eventlet
import random
import unittest
import eventlet
from keystoneclient.v2_0 import client as ksclient
from swiftclient import client as sclient
from swsync import accounts
from swsync import filler
from swsync.utils import get_config
@ -65,19 +66,19 @@ class TestSyncer(unittest.TestCase):
password=self.o_admin_password,
tenant_name=self.o_admin_tenant)
# Retreive admin (ResellerAdmin) token
(self.o_admin_auth_url, self.o_admin_token) = \
(self.o_admin_auth_url, self.o_admin_token) = (
sclient.Connection(self.o_st,
"%s:%s" % (self.o_admin_tenant,
self.o_admin_user),
self.o_admin_password,
auth_version=2).get_auth()
auth_version=2).get_auth())
# Retreive admin (ResellerAdmin) token
(self.d_admin_auth_url, self.d_admin_token) = \
(self.d_admin_auth_url, self.d_admin_token) = (
sclient.Connection(self.d_st,
"%s:%s" % (self.o_admin_tenant,
self.o_admin_user),
self.o_admin_password,
auth_version=2).get_auth()
auth_version=2).get_auth())
# Instanciate syncer
self.swsync = accounts.Accounts()
@ -90,15 +91,14 @@ class TestSyncer(unittest.TestCase):
yield account, account_id, username
def create_st_account_url(self, account_id):
o_account_url = \
self.o_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id
d_account_url = \
self.d_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id
o_account_url = (
self.o_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id)
d_account_url = (
self.d_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id)
return o_account_url, d_account_url
def verify_aco_diff(self, alo, ald):
"""Verify that 2 accounts are similar to validate migration
"""
"""Verify that 2 accounts are similar to validate migration."""
for k, v in alo[0].items():
if k not in ('x-timestamp', 'x-trans-id',
'date', 'last-modified'):
@ -206,8 +206,7 @@ class TestSyncer(unittest.TestCase):
http_conn=cnx)
def test_01_sync_one_of_two_empty_accounts(self):
"""create two empty accounts, Sync only one
"""
"""Create two empty accounts, Sync only one."""
index = {}
# create account
@ -215,8 +214,8 @@ class TestSyncer(unittest.TestCase):
self.pile,
2, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
# post meta data on account
tenant_cnx = sclient.Connection(self.o_st,
@ -235,22 +234,22 @@ class TestSyncer(unittest.TestCase):
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
alo = self.get_account_detail(account_id,
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
alo = self.get_account_detail(account_id,
self.o_admin_token, 'orig')
ald = self.get_account_detail(account_id,
ald = self.get_account_detail(account_id,
self.d_admin_token, 'dest')
if account == t_account:
self.verify_aco_diff(alo, ald)
if account == t_account:
self.verify_aco_diff(alo, ald)
def tearDown(self):
if self.created:
for k, v in self.created.items():
user_info_list = [user[1] for user in v]
account_id = k[1]
o_account_url, d_account_url = \
self.create_st_account_url(account_id)
o_account_url, d_account_url = (
self.create_st_account_url(account_id))
# Remove account content on origin and destination
self.delete_account_cont(o_account_url, self.o_admin_token)
self.delete_account_cont(d_account_url, self.d_admin_token)

4
tests/units/base.py
View File

@ -18,10 +18,10 @@
"""Test base classes imported from ceilometer.
"""
import unittest2
import mox
import stubout
import unittest2
from swsync import utils

4
tests/units/test_accounts.py
View File

@ -301,12 +301,12 @@ class TestAccountSync(TestAccountBase):
self.accounts_cls.container_cls = Containers()
def get_account(*args, **kwargs):
#ORIG
# ORIG
if len(ret) == 0:
ret.append("TESTED")
return ({'x-account-container-count': 1},
[{'name': 'foo'}])
#DEST
# DEST
else:
return ({'x-account-container-count': 2},
[{'name': 'foo', 'name': 'bar'}])

5
tests/units/test_containers.py
View File

@ -20,7 +20,6 @@ import urlparse
import swiftclient
import swsync.containers
import tests.units.base as test_base
import tests.units.fakes as fakes
@ -391,12 +390,12 @@ class TestContainers(TestContainersBase):
called_on_dest = []
def get_container(*args, **kwargs):
#ORIG
# ORIG
if len(called) == 0:
called.append("TESTED")
return ({}, [{'name': 'PARISESTMAGIQUE',
'last_modified': '2010'}])
#DEST
# DEST
else:
called_on_dest.append("TESTED")
raise swiftclient.client.ClientException("TESTED")

9
tests/units/test_filler.py
View File

@ -16,19 +16,16 @@
# under the License.
import eventlet
import swiftclient
from keystoneclient.exceptions import ClientException as KSClientException
from fakes import FakeKSClient
from fakes import FakeKSTenant
from fakes import FakeKSUser
from fakes import FakeSWConnection
from tests.units import base
from keystoneclient.exceptions import ClientException as KSClientException
import swiftclient
from swsync import filler
from swsync import utils
from tests.units import base
class TestFiller(base.TestCase):

1
tests/units/test_middleware_lm.py
View File

@ -20,6 +20,7 @@
import unittest
from middlewares import last_modified as middleware
import swift.common.swob as swob