From 0fd9a7fa43349c7c673849edf1d22ad9eb6529ea Mon Sep 17 00:00:00 2001 From: "Joe H. Rahme" Date: Thu, 30 May 2013 12:50:57 +0200 Subject: [PATCH 1/9] Adds the tenant_filter_file to the config file tenant_filter_file is the path to a file that contains a list of tenant ids to migrate. If missing, swsync will migrate all the tenants. --- etc/config.ini-sample | 6 ++++++ swsync/accounts.py | 29 +++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/etc/config.ini-sample b/etc/config.ini-sample index 995af57..626e3ce 100644 --- a/etc/config.ini-sample +++ b/etc/config.ini-sample @@ -21,3 +21,9 @@ filler_keystone_client_concurrency = 5 filler_swift_client_concurrency = 10 # This is usually bound to the max open files. sync_swift_client_concurrency = 10 + +[sync] +# This fields holds the path the a file containing the list of tenant ids to +# migrate. If this field is left blank or commented out, swsync will migrate all +# the tenants. +tenant_filter_file = diff --git a/swsync/accounts.py b/swsync/accounts.py index 3c54d70..dc9d5e7 100644 --- a/swsync/accounts.py +++ b/swsync/accounts.py @@ -24,7 +24,7 @@ import keystoneclient.v2_0.client import swiftclient import swsync.containers -from utils import get_config +from utils import get_config, ConfigurationError class Accounts(object): @@ -52,6 +52,22 @@ class Accounts(object): password=password, tenant_name=tenant_name) + def get_target_tenant_filter(self): + """Returns a set of target tenants from the tenant_list_file. + tenant_list_file is defined in the config file or given as a command + line argument. + + If the file is not defined, returns None. + """ + try: + tenant_filter_filename = get_config('sync', 'tenant_filter_file') + + with open(tenant_filter_filename) as tenantsfile: + return {name.strip() for name in tenantsfile.readlines()} + except (ConfigurationError, IOError): + return None + + def account_headers_clean(self, account_headers, to_null=False): ret = {} for key, value in account_headers.iteritems(): @@ -160,7 +176,16 @@ class Accounts(object): self.keystone_cnx = self.get_ks_auth_orig() - for tenant in self.keystone_cnx.tenants.list(): + # if user has defined target tenants, limit the migration + # to them + _targets_filters = self.get_target_tenant_filter() + if _targets_filters is not None: + _targets = (tenant for tenant in self.keystone_cnx.tenants.list() + if tenant.id in _targets) + else: + _targets = self.keystone_cnx.tenants.list() + + for tenant in _targets: user_orig_st_url = bare_oa_st_url + tenant.id user_dst_st_url = bare_dst_st_url + tenant.id From ee2cb04c3651381691f200f54f06ba14a1e78ad5 Mon Sep 17 00:00:00 2001 From: "Joe H. Rahme" Date: Thu, 30 May 2013 16:56:27 +0200 Subject: [PATCH 2/9] Prevent catching of IOError in case tenant_filter_list is defined This means the only way to do a full sync is to omit the field from the config file. --- swsync/accounts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swsync/accounts.py b/swsync/accounts.py index dc9d5e7..6fc1e3d 100644 --- a/swsync/accounts.py +++ b/swsync/accounts.py @@ -57,14 +57,14 @@ class Accounts(object): tenant_list_file is defined in the config file or given as a command line argument. - If the file is not defined, returns None. + If tenant_list_file is not defined, returns None (an empty filter). """ try: tenant_filter_filename = get_config('sync', 'tenant_filter_file') with open(tenant_filter_filename) as tenantsfile: return {name.strip() for name in tenantsfile.readlines()} - except (ConfigurationError, IOError): + except ConfigurationError: return None From e29cfb8d64db2f9255927bd2e6605794e01a8289 Mon Sep 17 00:00:00 2001 From: "Joe H. Rahme" Date: Thu, 30 May 2013 17:35:20 +0200 Subject: [PATCH 3/9] Fixes flake8 issues --- swsync/accounts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swsync/accounts.py b/swsync/accounts.py index 6fc1e3d..9b20fda 100644 --- a/swsync/accounts.py +++ b/swsync/accounts.py @@ -24,7 +24,8 @@ import keystoneclient.v2_0.client import swiftclient import swsync.containers -from utils import get_config, ConfigurationError +from utils import ConfigurationError +from utils import get_config class Accounts(object): @@ -67,7 +68,6 @@ class Accounts(object): except ConfigurationError: return None - def account_headers_clean(self, account_headers, to_null=False): ret = {} for key, value in account_headers.iteritems(): From 634a668024f139300cb6be51a5c2a9db9069a82b Mon Sep 17 00:00:00 2001 From: "Joe H. Rahme" Date: Sun, 2 Jun 2013 03:19:28 +0200 Subject: [PATCH 4/9] Fix test failing when missing a config field --- tests/units/fakes.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/units/fakes.py b/tests/units/fakes.py index f0492fa..6473685 100644 --- a/tests/units/fakes.py +++ b/tests/units/fakes.py @@ -19,6 +19,8 @@ import random import urlparse import uuid +from swsync.utils import ConfigurationError + STORAGE_ORIG = 'http://storage-orig.com' STORAGE_DEST = 'http://storage-dest.com' @@ -67,7 +69,10 @@ CONFIGDICT = {'auth': def fake_get_config(section, option): - return CONFIGDICT[section][option] + try: + return CONFIGDICT[section][option] + except KeyError: + raise ConfigurationError class FakeSWConnection(object): From e7faae0df657c48bd79ef25a58067ad491660d63 Mon Sep 17 00:00:00 2001 From: "Joe H. Rahme" Date: Mon, 3 Jun 2013 15:42:57 +0200 Subject: [PATCH 5/9] Replaces IDs with tenant_name in filter list --- swsync/accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swsync/accounts.py b/swsync/accounts.py index 9b20fda..a01fe5c 100644 --- a/swsync/accounts.py +++ b/swsync/accounts.py @@ -181,7 +181,7 @@ class Accounts(object): _targets_filters = self.get_target_tenant_filter() if _targets_filters is not None: _targets = (tenant for tenant in self.keystone_cnx.tenants.list() - if tenant.id in _targets) + if tenant.tenant_name in _targets_filters) else: _targets = self.keystone_cnx.tenants.list() From 445b134e9cec582b7adbab31bce04a03e2920c95 Mon Sep 17 00:00:00 2001 From: "Joe H. Rahme" Date: Mon, 3 Jun 2013 15:43:31 +0200 Subject: [PATCH 6/9] Unit tests coverage for get_target_tenant_filter --- tests/units/fakes.py | 4 ++++ tests/units/test_accounts.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/tests/units/fakes.py b/tests/units/fakes.py index 6473685..8e17ee6 100644 --- a/tests/units/fakes.py +++ b/tests/units/fakes.py @@ -75,6 +75,10 @@ def fake_get_config(section, option): raise ConfigurationError +def fake_get_filter(self): + return {'foo1', 'foo2', 'foo3'} + + class FakeSWConnection(object): def __init__(self, *args, **kwargs): self.mainargs = args diff --git a/tests/units/test_accounts.py b/tests/units/test_accounts.py index c893a65..6f014ff 100644 --- a/tests/units/test_accounts.py +++ b/tests/units/test_accounts.py @@ -35,6 +35,8 @@ class TestAccountBase(tests.units.base.TestCase): self.stubs.Set(swiftclient.client, 'Connection', fakes.FakeSWConnection) self.stubs.Set(swsync.accounts, 'get_config', fakes.fake_get_config) + self.stubs.Set(swsync.accounts.Accounts, 'get_target_tenant_filter', + fakes.fake_get_filter) self.stubs.Set(swiftclient, 'http_connection', fakes.FakeSWClient.http_connection) From 714824efa70d6f3654e64bef6beb81bf467ccfd0 Mon Sep 17 00:00:00 2001 From: "Joe H. Rahme" Date: Tue, 4 Jun 2013 15:48:27 +0200 Subject: [PATCH 7/9] Fixes tenant_name vs name confusion --- swsync/accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swsync/accounts.py b/swsync/accounts.py index a01fe5c..b80cd48 100644 --- a/swsync/accounts.py +++ b/swsync/accounts.py @@ -181,7 +181,7 @@ class Accounts(object): _targets_filters = self.get_target_tenant_filter() if _targets_filters is not None: _targets = (tenant for tenant in self.keystone_cnx.tenants.list() - if tenant.tenant_name in _targets_filters) + if tenant.name in _targets_filters) else: _targets = self.keystone_cnx.tenants.list() From 0144024269c6f5c43419e6c4c943256162cc0483 Mon Sep 17 00:00:00 2001 From: "Joe H. Rahme" Date: Tue, 4 Jun 2013 15:49:26 +0200 Subject: [PATCH 8/9] Adds tests for the tenant filter feature --- tests/functional/test_syncer_filter.py | 263 +++++++++++++++++++++++++ tests/units/fakes.py | 1 + 2 files changed, 264 insertions(+) create mode 100644 tests/functional/test_syncer_filter.py diff --git a/tests/functional/test_syncer_filter.py b/tests/functional/test_syncer_filter.py new file mode 100644 index 0000000..e935f49 --- /dev/null +++ b/tests/functional/test_syncer_filter.py @@ -0,0 +1,263 @@ +# -*- encoding: utf-8 -*- + +# Copyright 2013 eNovance. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Author : "Joe Hakim Rahme " +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# To start this functional test the admin users (on both keystone) used +# to synchronize the destination swift must own the ResellerAdmin role in +# keystone. +# +# You must also create a file containing the list of users to migrate and +# specify this file in your config.ini file. + +import eventlet +import random +import unittest + +from keystoneclient.v2_0 import client as ksclient +from swiftclient import client as sclient +from swsync import accounts +from swsync import filler +from swsync.utils import get_config + + +class TestSyncer(unittest.TestCase): + + def setUp(self): + self.o_st = get_config('auth', 'keystone_origin') + self.d_st = get_config('auth', 'keystone_dest') + self.default_user_password = get_config('filler', + 'default_user_password') + # Retreive configuration for filler + self.o_admin_tenant, self.o_admin_user, self.o_admin_password = ( + get_config('auth', 'keystone_origin_admin_credentials').split(':')) + self.sw_c_concu = int(get_config('concurrency', + 'filler_swift_client_concurrency')) + self.ks_c_concu = int(get_config('concurrency', + 'filler_keystone_client_concurrency')) + self.filter_filename = get_config('sync', 'tenant_filter_file') + self.pile = eventlet.GreenPile(self.sw_c_concu) + self.pool = eventlet.GreenPool(self.ks_c_concu) + # Set a keystone connection to origin server + self.o_ks_client = ksclient.Client( + auth_url=self.o_st, + username=self.o_admin_user, + password=self.o_admin_password, + tenant_name=self.o_admin_tenant) + # Set a keystone connection to destination server + self.d_ks_client = ksclient.Client( + auth_url=self.d_st, + username=self.o_admin_user, + password=self.o_admin_password, + tenant_name=self.o_admin_tenant) + # Retreive admin (ResellerAdmin) token + (self.o_admin_auth_url, self.o_admin_token) = \ + sclient.Connection(self.o_st, + "%s:%s" % (self.o_admin_tenant, + self.o_admin_user), + self.o_admin_password, + auth_version=2).get_auth() + # Retreive admin (ResellerAdmin) token + (self.d_admin_auth_url, self.d_admin_token) = \ + sclient.Connection(self.d_st, + "%s:%s" % (self.o_admin_tenant, + self.o_admin_user), + self.o_admin_password, + auth_version=2).get_auth() + # Instanciate syncer + self.swsync = accounts.Accounts() + + def extract_created_a_u_iter(self, created): + for ad, usd in created.items(): + account = ad[0] + account_id = ad[1] + # Retreive the first user as we only need one + username = usd[0][0] + yield account, account_id, username + + def create_st_account_url(self, account_id): + o_account_url = \ + self.o_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id + d_account_url = \ + self.d_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id + return o_account_url, d_account_url + + def verify_aco_diff(self, alo, ald): + """Verify that 2 accounts are similar to validate migration + """ + for k, v in alo[0].items(): + if k not in ('x-timestamp', 'x-trans-id', + 'date', 'last-modified'): + self.assertEqual(ald[0][k], v, msg='%s differs' % k) + + def delete_account_cont(self, account_url, token): + cnx = sclient.http_connection(account_url) + al = sclient.get_account(None, token, + http_conn=cnx, + full_listing=True) + for container in [c['name'] for c in al[1]]: + ci = sclient.get_container(None, token, + container, http_conn=cnx, + full_listing=True) + on = [od['name'] for od in ci[1]] + for obj in on: + sclient.delete_object('', token, container, + obj, http_conn=cnx) + sclient.delete_container('', token, container, http_conn=cnx) + + def get_url(self, account_id, s_type): + # Create account storage url + o_account_url, d_account_url = self.create_st_account_url(account_id) + if s_type == 'orig': + url = o_account_url + elif s_type == 'dest': + url = d_account_url + else: + raise Exception('Unknown type') + return url + + def get_cnx(self, account_id, s_type): + url = self.get_url(account_id, s_type) + return sclient.http_connection(url) + + def get_account_detail(self, account_id, token, s_type): + cnx = self.get_cnx(account_id, s_type) + return sclient.get_account(None, token, + http_conn=cnx, + full_listing=True) + + def list_containers(self, account_id, token, s_type): + cd = self.get_account_detail(account_id, token, s_type) + return cd[1] + + def get_container_detail(self, account_id, token, s_type, container): + cnx = self.get_cnx(account_id, s_type) + return sclient.get_container(None, token, container, + http_conn=cnx, full_listing=True) + + def list_objects(self, account_id, token, s_type, container): + cd = self.get_container_detail(account_id, token, s_type, container) + return cd[1] + + def list_objects_in_containers(self, account_id, token, s_type): + ret = {} + cl = self.list_containers(account_id, token, s_type) + for c in [c['name'] for c in cl]: + objs = self.list_objects(account_id, token, s_type, c) + ret[c] = objs + return ret + + def get_object_detail(self, account_id, token, s_type, container, obj): + cnx = self.get_cnx(account_id, s_type) + return sclient.get_object("", token, container, obj, http_conn=cnx) + + def get_account_meta(self, account_id, token, s_type): + d = self.get_account_detail(account_id, token, s_type) + return {k: v for k, v in d[0].iteritems() + if k.startswith('x-account-meta')} + + def get_container_meta(self, account_id, token, s_type, container): + d = self.get_container_detail(account_id, token, s_type, container) + return {k: v for k, v in d[0].iteritems() + if k.startswith('x-container-meta')} + + def post_account(self, account_id, token, s_type, headers): + cnx = self.get_cnx(account_id, s_type) + sclient.post_account("", token, headers, http_conn=cnx) + + def post_container(self, account_id, token, s_type, container, headers): + cnx = self.get_cnx(account_id, s_type) + sclient.post_container("", token, container, headers, http_conn=cnx) + + def put_container(self, account_id, token, s_type, container): + cnx = self.get_cnx(account_id, s_type) + sclient.put_container("", token, container, http_conn=cnx) + + def delete_container(self, account_id, token, s_type, container): + cnx = self.get_cnx(account_id, s_type) + sclient.delete_container("", token, container, http_conn=cnx) + + def post_object(self, account_id, token, s_type, container, name, headers): + cnx = self.get_cnx(account_id, s_type) + sclient.post_object("", token, container, name, headers, http_conn=cnx) + + def put_object(self, account_id, token, s_type, container, name, content): + cnx = self.get_cnx(account_id, s_type) + sclient.put_object("", token, container, name, content, http_conn=cnx) + + def delete_object(self, account_id, token, s_type, + container, name): + cnx = self.get_cnx(account_id, s_type) + sclient.delete_object("", token, container, name, + http_conn=cnx) + + def test_01_sync_one_of_two_empty_accounts(self): + """create two empty accounts, Sync only one + """ + index = {} + test_account_name = "account_test" + + # create account + self.created = filler.create_swift_account(self.o_ks_client, + self.pile, + 2, 1, index) + + for account, account_id, username in \ + self.extract_created_a_u_iter(self.created): + + # post meta data on account + tenant_cnx = sclient.Connection(self.o_st, + "%s:%s" % (account, username), + self.default_user_password, + auth_version=2) + filler.create_account_meta(tenant_cnx) + + # select random account and write it in the filter file + t_account, t_account_id, t_username = random.choice(list( + self.extract_created_a_u_iter(self.created))) + with open(self.filter_filename, "w") as filterlist: + filterlist.write(t_account + "\n") + + # start sync process + self.swsync.process() + + # Now verify dest + for account, account_id, username in \ + self.extract_created_a_u_iter(self.created): + alo = self.get_account_detail(account_id, + self.o_admin_token, 'orig') + ald = self.get_account_detail(account_id, + self.d_admin_token, 'dest') + if account == t_account: + self.verify_aco_diff(alo, ald) + + def tearDown(self): + if self.created: + for k, v in self.created.items(): + user_info_list = [user[1] for user in v] + account_id = k[1] + o_account_url, d_account_url = \ + self.create_st_account_url(account_id) + # Remove account content on origin and destination + self.delete_account_cont(o_account_url, self.o_admin_token) + self.delete_account_cont(d_account_url, self.d_admin_token) + # We just need to delete keystone accounts and users + # in origin keystone as syncer does not sync + # keystone database + filler.delete_account(self.o_ks_client, + user_info_list, + k) diff --git a/tests/units/fakes.py b/tests/units/fakes.py index 8e17ee6..43ed3cc 100644 --- a/tests/units/fakes.py +++ b/tests/units/fakes.py @@ -130,6 +130,7 @@ def fake_get_auth(auth_url, tenant, user, password): class FakeKSTenant(object): def __init__(self, tenant_name): self.tenant_name = tenant_name + self.name = tenant_name @property def id(self): From f50cd083ed1c354ba223a3906a588d04975875b4 Mon Sep 17 00:00:00 2001 From: "Joe H. Rahme" Date: Tue, 11 Jun 2013 09:28:00 +0200 Subject: [PATCH 9/9] Updates README and other documentation fixes --- README.md | 8 ++++++++ etc/config.ini-sample | 7 +++---- tests/functional/test_syncer_filter.py | 4 ++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ba8a442..c505be9 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,14 @@ first pass goes well, if for example there is network failure swsync will just skip it and hope to do it on the next run. So the tool can for instance be launched by a cron job to perform diff synchronization each night. +Tenant Filter File +------------------ + +It is possible to limit the migration to a subset of the total number of +tenants, by uncommenting the field "tenant_filter_file". This field should +hold the path to a file containing a list of tenant names to migrate, one +per line. If left commented, swsync will migrate all the tenants. + Swift Middleware last-modified ------------------------------ diff --git a/etc/config.ini-sample b/etc/config.ini-sample index 626e3ce..6497901 100644 --- a/etc/config.ini-sample +++ b/etc/config.ini-sample @@ -23,7 +23,6 @@ filler_swift_client_concurrency = 10 sync_swift_client_concurrency = 10 [sync] -# This fields holds the path the a file containing the list of tenant ids to -# migrate. If this field is left blank or commented out, swsync will migrate all -# the tenants. -tenant_filter_file = +# Uncomment this field to designate a file containing a list of tenant names +# to be migrated. If left commented, all the tenants will be targeted. +# tenant_filter_file = etc/tenants.list diff --git a/tests/functional/test_syncer_filter.py b/tests/functional/test_syncer_filter.py index e935f49..da4dff0 100644 --- a/tests/functional/test_syncer_filter.py +++ b/tests/functional/test_syncer_filter.py @@ -21,8 +21,8 @@ # to synchronize the destination swift must own the ResellerAdmin role in # keystone. # -# You must also create a file containing the list of users to migrate and -# specify this file in your config.ini file. +# In your config.ini file, you should uncomment the field tenant_filter_file +# and specify a path to a file where you're allowed to read and write. import eventlet import random