Merge pull request #62 from enovance/tenant_filter
Adds the tenant_filter_file to the config file
This commit is contained in:
Joe Rahme
commit
850baa2c6a
|
|
@ -162,6 +162,14 @@ first pass goes well, if for example there is network failure swsync will
|
|||
just skip it and hope to do it on the next run. So the tool can for instance
|
||||
be launched by a cron job to perform diff synchronization each night.
|
||||
|
||||
Tenant Filter File
|
||||
------------------
|
||||
|
||||
It is possible to limit the migration to a subset of the total number of
|
||||
tenants, by uncommenting the field "tenant_filter_file". This field should
|
||||
hold the path to a file containing a list of tenant names to migrate, one
|
||||
per line. If left commented, swsync will migrate all the tenants.
|
||||
|
||||
Swift Middleware last-modified
|
||||
------------------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -21,3 +21,8 @@ filler_keystone_client_concurrency = 5
|
|||
filler_swift_client_concurrency = 10
|
||||
# This is usually bound to the max open files.
|
||||
sync_swift_client_concurrency = 10
|
||||
|
||||
[sync]
|
||||
# Uncomment this field to designate a file containing a list of tenant names
|
||||
# to be migrated. If left commented, all the tenants will be targeted.
|
||||
# tenant_filter_file = etc/tenants.list
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import keystoneclient.v2_0.client
|
|||
import swiftclient
|
||||
|
||||
import swsync.containers
|
||||
from utils import ConfigurationError
|
||||
from utils import get_config
|
||||
|
||||
|
||||
|
|
@ -52,6 +53,21 @@ class Accounts(object):
|
|||
password=password,
|
||||
tenant_name=tenant_name)
|
||||
|
||||
def get_target_tenant_filter(self):
|
||||
"""Returns a set of target tenants from the tenant_list_file.
|
||||
tenant_list_file is defined in the config file or given as a command
|
||||
line argument.
|
||||
|
||||
If tenant_list_file is not defined, returns None (an empty filter).
|
||||
"""
|
||||
try:
|
||||
tenant_filter_filename = get_config('sync', 'tenant_filter_file')
|
||||
|
||||
with open(tenant_filter_filename) as tenantsfile:
|
||||
return {name.strip() for name in tenantsfile.readlines()}
|
||||
except ConfigurationError:
|
||||
return None
|
||||
|
||||
def account_headers_clean(self, account_headers, to_null=False):
|
||||
ret = {}
|
||||
for key, value in account_headers.iteritems():
|
||||
|
|
@ -160,7 +176,16 @@ class Accounts(object):
|
|||
|
||||
self.keystone_cnx = self.get_ks_auth_orig()
|
||||
|
||||
for tenant in self.keystone_cnx.tenants.list():
|
||||
# if user has defined target tenants, limit the migration
|
||||
# to them
|
||||
_targets_filters = self.get_target_tenant_filter()
|
||||
if _targets_filters is not None:
|
||||
_targets = (tenant for tenant in self.keystone_cnx.tenants.list()
|
||||
if tenant.name in _targets_filters)
|
||||
else:
|
||||
_targets = self.keystone_cnx.tenants.list()
|
||||
|
||||
for tenant in _targets:
|
||||
user_orig_st_url = bare_oa_st_url + tenant.id
|
||||
user_dst_st_url = bare_dst_st_url + tenant.id
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,263 @@
|
|||
# -*- encoding: utf-8 -*-
|
||||
|
||||
# Copyright 2013 eNovance.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Author : "Joe Hakim Rahme <joe.hakim.rahme@enovance.com>"
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# To start this functional test the admin users (on both keystone) used
|
||||
# to synchronize the destination swift must own the ResellerAdmin role in
|
||||
# keystone.
|
||||
#
|
||||
# In your config.ini file, you should uncomment the field tenant_filter_file
|
||||
# and specify a path to a file where you're allowed to read and write.
|
||||
|
||||
import eventlet
|
||||
import random
|
||||
import unittest
|
||||
|
||||
from keystoneclient.v2_0 import client as ksclient
|
||||
from swiftclient import client as sclient
|
||||
from swsync import accounts
|
||||
from swsync import filler
|
||||
from swsync.utils import get_config
|
||||
|
||||
|
||||
class TestSyncer(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.o_st = get_config('auth', 'keystone_origin')
|
||||
self.d_st = get_config('auth', 'keystone_dest')
|
||||
self.default_user_password = get_config('filler',
|
||||
'default_user_password')
|
||||
# Retreive configuration for filler
|
||||
self.o_admin_tenant, self.o_admin_user, self.o_admin_password = (
|
||||
get_config('auth', 'keystone_origin_admin_credentials').split(':'))
|
||||
self.sw_c_concu = int(get_config('concurrency',
|
||||
'filler_swift_client_concurrency'))
|
||||
self.ks_c_concu = int(get_config('concurrency',
|
||||
'filler_keystone_client_concurrency'))
|
||||
self.filter_filename = get_config('sync', 'tenant_filter_file')
|
||||
self.pile = eventlet.GreenPile(self.sw_c_concu)
|
||||
self.pool = eventlet.GreenPool(self.ks_c_concu)
|
||||
# Set a keystone connection to origin server
|
||||
self.o_ks_client = ksclient.Client(
|
||||
auth_url=self.o_st,
|
||||
username=self.o_admin_user,
|
||||
password=self.o_admin_password,
|
||||
tenant_name=self.o_admin_tenant)
|
||||
# Set a keystone connection to destination server
|
||||
self.d_ks_client = ksclient.Client(
|
||||
auth_url=self.d_st,
|
||||
username=self.o_admin_user,
|
||||
password=self.o_admin_password,
|
||||
tenant_name=self.o_admin_tenant)
|
||||
# Retreive admin (ResellerAdmin) token
|
||||
(self.o_admin_auth_url, self.o_admin_token) = \
|
||||
sclient.Connection(self.o_st,
|
||||
"%s:%s" % (self.o_admin_tenant,
|
||||
self.o_admin_user),
|
||||
self.o_admin_password,
|
||||
auth_version=2).get_auth()
|
||||
# Retreive admin (ResellerAdmin) token
|
||||
(self.d_admin_auth_url, self.d_admin_token) = \
|
||||
sclient.Connection(self.d_st,
|
||||
"%s:%s" % (self.o_admin_tenant,
|
||||
self.o_admin_user),
|
||||
self.o_admin_password,
|
||||
auth_version=2).get_auth()
|
||||
# Instanciate syncer
|
||||
self.swsync = accounts.Accounts()
|
||||
|
||||
def extract_created_a_u_iter(self, created):
|
||||
for ad, usd in created.items():
|
||||
account = ad[0]
|
||||
account_id = ad[1]
|
||||
# Retreive the first user as we only need one
|
||||
username = usd[0][0]
|
||||
yield account, account_id, username
|
||||
|
||||
def create_st_account_url(self, account_id):
|
||||
o_account_url = \
|
||||
self.o_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id
|
||||
d_account_url = \
|
||||
self.d_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id
|
||||
return o_account_url, d_account_url
|
||||
|
||||
def verify_aco_diff(self, alo, ald):
|
||||
"""Verify that 2 accounts are similar to validate migration
|
||||
"""
|
||||
for k, v in alo[0].items():
|
||||
if k not in ('x-timestamp', 'x-trans-id',
|
||||
'date', 'last-modified'):
|
||||
self.assertEqual(ald[0][k], v, msg='%s differs' % k)
|
||||
|
||||
def delete_account_cont(self, account_url, token):
|
||||
cnx = sclient.http_connection(account_url)
|
||||
al = sclient.get_account(None, token,
|
||||
http_conn=cnx,
|
||||
full_listing=True)
|
||||
for container in [c['name'] for c in al[1]]:
|
||||
ci = sclient.get_container(None, token,
|
||||
container, http_conn=cnx,
|
||||
full_listing=True)
|
||||
on = [od['name'] for od in ci[1]]
|
||||
for obj in on:
|
||||
sclient.delete_object('', token, container,
|
||||
obj, http_conn=cnx)
|
||||
sclient.delete_container('', token, container, http_conn=cnx)
|
||||
|
||||
def get_url(self, account_id, s_type):
|
||||
# Create account storage url
|
||||
o_account_url, d_account_url = self.create_st_account_url(account_id)
|
||||
if s_type == 'orig':
|
||||
url = o_account_url
|
||||
elif s_type == 'dest':
|
||||
url = d_account_url
|
||||
else:
|
||||
raise Exception('Unknown type')
|
||||
return url
|
||||
|
||||
def get_cnx(self, account_id, s_type):
|
||||
url = self.get_url(account_id, s_type)
|
||||
return sclient.http_connection(url)
|
||||
|
||||
def get_account_detail(self, account_id, token, s_type):
|
||||
cnx = self.get_cnx(account_id, s_type)
|
||||
return sclient.get_account(None, token,
|
||||
http_conn=cnx,
|
||||
full_listing=True)
|
||||
|
||||
def list_containers(self, account_id, token, s_type):
|
||||
cd = self.get_account_detail(account_id, token, s_type)
|
||||
return cd[1]
|
||||
|
||||
def get_container_detail(self, account_id, token, s_type, container):
|
||||
cnx = self.get_cnx(account_id, s_type)
|
||||
return sclient.get_container(None, token, container,
|
||||
http_conn=cnx, full_listing=True)
|
||||
|
||||
def list_objects(self, account_id, token, s_type, container):
|
||||
cd = self.get_container_detail(account_id, token, s_type, container)
|
||||
return cd[1]
|
||||
|
||||
def list_objects_in_containers(self, account_id, token, s_type):
|
||||
ret = {}
|
||||
cl = self.list_containers(account_id, token, s_type)
|
||||
for c in [c['name'] for c in cl]:
|
||||
objs = self.list_objects(account_id, token, s_type, c)
|
||||
ret[c] = objs
|
||||
return ret
|
||||
|
||||
def get_object_detail(self, account_id, token, s_type, container, obj):
|
||||
cnx = self.get_cnx(account_id, s_type)
|
||||
return sclient.get_object("", token, container, obj, http_conn=cnx)
|
||||
|
||||
def get_account_meta(self, account_id, token, s_type):
|
||||
d = self.get_account_detail(account_id, token, s_type)
|
||||
return {k: v for k, v in d[0].iteritems()
|
||||
if k.startswith('x-account-meta')}
|
||||
|
||||
def get_container_meta(self, account_id, token, s_type, container):
|
||||
d = self.get_container_detail(account_id, token, s_type, container)
|
||||
return {k: v for k, v in d[0].iteritems()
|
||||
if k.startswith('x-container-meta')}
|
||||
|
||||
def post_account(self, account_id, token, s_type, headers):
|
||||
cnx = self.get_cnx(account_id, s_type)
|
||||
sclient.post_account("", token, headers, http_conn=cnx)
|
||||
|
||||
def post_container(self, account_id, token, s_type, container, headers):
|
||||
cnx = self.get_cnx(account_id, s_type)
|
||||
sclient.post_container("", token, container, headers, http_conn=cnx)
|
||||
|
||||
def put_container(self, account_id, token, s_type, container):
|
||||
cnx = self.get_cnx(account_id, s_type)
|
||||
sclient.put_container("", token, container, http_conn=cnx)
|
||||
|
||||
def delete_container(self, account_id, token, s_type, container):
|
||||
cnx = self.get_cnx(account_id, s_type)
|
||||
sclient.delete_container("", token, container, http_conn=cnx)
|
||||
|
||||
def post_object(self, account_id, token, s_type, container, name, headers):
|
||||
cnx = self.get_cnx(account_id, s_type)
|
||||
sclient.post_object("", token, container, name, headers, http_conn=cnx)
|
||||
|
||||
def put_object(self, account_id, token, s_type, container, name, content):
|
||||
cnx = self.get_cnx(account_id, s_type)
|
||||
sclient.put_object("", token, container, name, content, http_conn=cnx)
|
||||
|
||||
def delete_object(self, account_id, token, s_type,
|
||||
container, name):
|
||||
cnx = self.get_cnx(account_id, s_type)
|
||||
sclient.delete_object("", token, container, name,
|
||||
http_conn=cnx)
|
||||
|
||||
def test_01_sync_one_of_two_empty_accounts(self):
|
||||
"""create two empty accounts, Sync only one
|
||||
"""
|
||||
index = {}
|
||||
test_account_name = "account_test"
|
||||
|
||||
# create account
|
||||
self.created = filler.create_swift_account(self.o_ks_client,
|
||||
self.pile,
|
||||
2, 1, index)
|
||||
|
||||
for account, account_id, username in \
|
||||
self.extract_created_a_u_iter(self.created):
|
||||
|
||||
# post meta data on account
|
||||
tenant_cnx = sclient.Connection(self.o_st,
|
||||
"%s:%s" % (account, username),
|
||||
self.default_user_password,
|
||||
auth_version=2)
|
||||
filler.create_account_meta(tenant_cnx)
|
||||
|
||||
# select random account and write it in the filter file
|
||||
t_account, t_account_id, t_username = random.choice(list(
|
||||
self.extract_created_a_u_iter(self.created)))
|
||||
with open(self.filter_filename, "w") as filterlist:
|
||||
filterlist.write(t_account + "\n")
|
||||
|
||||
# start sync process
|
||||
self.swsync.process()
|
||||
|
||||
# Now verify dest
|
||||
for account, account_id, username in \
|
||||
self.extract_created_a_u_iter(self.created):
|
||||
alo = self.get_account_detail(account_id,
|
||||
self.o_admin_token, 'orig')
|
||||
ald = self.get_account_detail(account_id,
|
||||
self.d_admin_token, 'dest')
|
||||
if account == t_account:
|
||||
self.verify_aco_diff(alo, ald)
|
||||
|
||||
def tearDown(self):
|
||||
if self.created:
|
||||
for k, v in self.created.items():
|
||||
user_info_list = [user[1] for user in v]
|
||||
account_id = k[1]
|
||||
o_account_url, d_account_url = \
|
||||
self.create_st_account_url(account_id)
|
||||
# Remove account content on origin and destination
|
||||
self.delete_account_cont(o_account_url, self.o_admin_token)
|
||||
self.delete_account_cont(d_account_url, self.d_admin_token)
|
||||
# We just need to delete keystone accounts and users
|
||||
# in origin keystone as syncer does not sync
|
||||
# keystone database
|
||||
filler.delete_account(self.o_ks_client,
|
||||
user_info_list,
|
||||
k)
|
||||
|
|
@ -19,6 +19,8 @@ import random
|
|||
import urlparse
|
||||
import uuid
|
||||
|
||||
from swsync.utils import ConfigurationError
|
||||
|
||||
STORAGE_ORIG = 'http://storage-orig.com'
|
||||
STORAGE_DEST = 'http://storage-dest.com'
|
||||
|
||||
|
|
@ -67,7 +69,14 @@ CONFIGDICT = {'auth':
|
|||
|
||||
|
||||
def fake_get_config(section, option):
|
||||
return CONFIGDICT[section][option]
|
||||
try:
|
||||
return CONFIGDICT[section][option]
|
||||
except KeyError:
|
||||
raise ConfigurationError
|
||||
|
||||
|
||||
def fake_get_filter(self):
|
||||
return {'foo1', 'foo2', 'foo3'}
|
||||
|
||||
|
||||
class FakeSWConnection(object):
|
||||
|
|
@ -121,6 +130,7 @@ def fake_get_auth(auth_url, tenant, user, password):
|
|||
class FakeKSTenant(object):
|
||||
def __init__(self, tenant_name):
|
||||
self.tenant_name = tenant_name
|
||||
self.name = tenant_name
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
|
|
|
|||
|
|
@ -35,6 +35,8 @@ class TestAccountBase(tests.units.base.TestCase):
|
|||
self.stubs.Set(swiftclient.client, 'Connection',
|
||||
fakes.FakeSWConnection)
|
||||
self.stubs.Set(swsync.accounts, 'get_config', fakes.fake_get_config)
|
||||
self.stubs.Set(swsync.accounts.Accounts, 'get_target_tenant_filter',
|
||||
fakes.fake_get_filter)
|
||||
self.stubs.Set(swiftclient, 'http_connection',
|
||||
fakes.FakeSWClient.http_connection)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue