Merge pull request #62 from enovance/tenant_filter
Adds the tenant_filter_file to the config file
This commit is contained in:
commit
850baa2c6a
|
@ -162,6 +162,14 @@ first pass goes well, if for example there is network failure swsync will
|
||||||
just skip it and hope to do it on the next run. So the tool can for instance
|
just skip it and hope to do it on the next run. So the tool can for instance
|
||||||
be launched by a cron job to perform diff synchronization each night.
|
be launched by a cron job to perform diff synchronization each night.
|
||||||
|
|
||||||
|
Tenant Filter File
|
||||||
|
------------------
|
||||||
|
|
||||||
|
It is possible to limit the migration to a subset of the total number of
|
||||||
|
tenants, by uncommenting the field "tenant_filter_file". This field should
|
||||||
|
hold the path to a file containing a list of tenant names to migrate, one
|
||||||
|
per line. If left commented, swsync will migrate all the tenants.
|
||||||
|
|
||||||
Swift Middleware last-modified
|
Swift Middleware last-modified
|
||||||
------------------------------
|
------------------------------
|
||||||
|
|
||||||
|
|
|
@ -21,3 +21,8 @@ filler_keystone_client_concurrency = 5
|
||||||
filler_swift_client_concurrency = 10
|
filler_swift_client_concurrency = 10
|
||||||
# This is usually bound to the max open files.
|
# This is usually bound to the max open files.
|
||||||
sync_swift_client_concurrency = 10
|
sync_swift_client_concurrency = 10
|
||||||
|
|
||||||
|
[sync]
|
||||||
|
# Uncomment this field to designate a file containing a list of tenant names
|
||||||
|
# to be migrated. If left commented, all the tenants will be targeted.
|
||||||
|
# tenant_filter_file = etc/tenants.list
|
||||||
|
|
|
@ -24,6 +24,7 @@ import keystoneclient.v2_0.client
|
||||||
import swiftclient
|
import swiftclient
|
||||||
|
|
||||||
import swsync.containers
|
import swsync.containers
|
||||||
|
from utils import ConfigurationError
|
||||||
from utils import get_config
|
from utils import get_config
|
||||||
|
|
||||||
|
|
||||||
|
@ -52,6 +53,21 @@ class Accounts(object):
|
||||||
password=password,
|
password=password,
|
||||||
tenant_name=tenant_name)
|
tenant_name=tenant_name)
|
||||||
|
|
||||||
|
def get_target_tenant_filter(self):
|
||||||
|
"""Returns a set of target tenants from the tenant_list_file.
|
||||||
|
tenant_list_file is defined in the config file or given as a command
|
||||||
|
line argument.
|
||||||
|
|
||||||
|
If tenant_list_file is not defined, returns None (an empty filter).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
tenant_filter_filename = get_config('sync', 'tenant_filter_file')
|
||||||
|
|
||||||
|
with open(tenant_filter_filename) as tenantsfile:
|
||||||
|
return {name.strip() for name in tenantsfile.readlines()}
|
||||||
|
except ConfigurationError:
|
||||||
|
return None
|
||||||
|
|
||||||
def account_headers_clean(self, account_headers, to_null=False):
|
def account_headers_clean(self, account_headers, to_null=False):
|
||||||
ret = {}
|
ret = {}
|
||||||
for key, value in account_headers.iteritems():
|
for key, value in account_headers.iteritems():
|
||||||
|
@ -160,7 +176,16 @@ class Accounts(object):
|
||||||
|
|
||||||
self.keystone_cnx = self.get_ks_auth_orig()
|
self.keystone_cnx = self.get_ks_auth_orig()
|
||||||
|
|
||||||
for tenant in self.keystone_cnx.tenants.list():
|
# if user has defined target tenants, limit the migration
|
||||||
|
# to them
|
||||||
|
_targets_filters = self.get_target_tenant_filter()
|
||||||
|
if _targets_filters is not None:
|
||||||
|
_targets = (tenant for tenant in self.keystone_cnx.tenants.list()
|
||||||
|
if tenant.name in _targets_filters)
|
||||||
|
else:
|
||||||
|
_targets = self.keystone_cnx.tenants.list()
|
||||||
|
|
||||||
|
for tenant in _targets:
|
||||||
user_orig_st_url = bare_oa_st_url + tenant.id
|
user_orig_st_url = bare_oa_st_url + tenant.id
|
||||||
user_dst_st_url = bare_dst_st_url + tenant.id
|
user_dst_st_url = bare_dst_st_url + tenant.id
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,263 @@
|
||||||
|
# -*- encoding: utf-8 -*-
|
||||||
|
|
||||||
|
# Copyright 2013 eNovance.
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Author : "Joe Hakim Rahme <joe.hakim.rahme@enovance.com>"
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
# To start this functional test the admin users (on both keystone) used
|
||||||
|
# to synchronize the destination swift must own the ResellerAdmin role in
|
||||||
|
# keystone.
|
||||||
|
#
|
||||||
|
# In your config.ini file, you should uncomment the field tenant_filter_file
|
||||||
|
# and specify a path to a file where you're allowed to read and write.
|
||||||
|
|
||||||
|
import eventlet
|
||||||
|
import random
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from keystoneclient.v2_0 import client as ksclient
|
||||||
|
from swiftclient import client as sclient
|
||||||
|
from swsync import accounts
|
||||||
|
from swsync import filler
|
||||||
|
from swsync.utils import get_config
|
||||||
|
|
||||||
|
|
||||||
|
class TestSyncer(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.o_st = get_config('auth', 'keystone_origin')
|
||||||
|
self.d_st = get_config('auth', 'keystone_dest')
|
||||||
|
self.default_user_password = get_config('filler',
|
||||||
|
'default_user_password')
|
||||||
|
# Retreive configuration for filler
|
||||||
|
self.o_admin_tenant, self.o_admin_user, self.o_admin_password = (
|
||||||
|
get_config('auth', 'keystone_origin_admin_credentials').split(':'))
|
||||||
|
self.sw_c_concu = int(get_config('concurrency',
|
||||||
|
'filler_swift_client_concurrency'))
|
||||||
|
self.ks_c_concu = int(get_config('concurrency',
|
||||||
|
'filler_keystone_client_concurrency'))
|
||||||
|
self.filter_filename = get_config('sync', 'tenant_filter_file')
|
||||||
|
self.pile = eventlet.GreenPile(self.sw_c_concu)
|
||||||
|
self.pool = eventlet.GreenPool(self.ks_c_concu)
|
||||||
|
# Set a keystone connection to origin server
|
||||||
|
self.o_ks_client = ksclient.Client(
|
||||||
|
auth_url=self.o_st,
|
||||||
|
username=self.o_admin_user,
|
||||||
|
password=self.o_admin_password,
|
||||||
|
tenant_name=self.o_admin_tenant)
|
||||||
|
# Set a keystone connection to destination server
|
||||||
|
self.d_ks_client = ksclient.Client(
|
||||||
|
auth_url=self.d_st,
|
||||||
|
username=self.o_admin_user,
|
||||||
|
password=self.o_admin_password,
|
||||||
|
tenant_name=self.o_admin_tenant)
|
||||||
|
# Retreive admin (ResellerAdmin) token
|
||||||
|
(self.o_admin_auth_url, self.o_admin_token) = \
|
||||||
|
sclient.Connection(self.o_st,
|
||||||
|
"%s:%s" % (self.o_admin_tenant,
|
||||||
|
self.o_admin_user),
|
||||||
|
self.o_admin_password,
|
||||||
|
auth_version=2).get_auth()
|
||||||
|
# Retreive admin (ResellerAdmin) token
|
||||||
|
(self.d_admin_auth_url, self.d_admin_token) = \
|
||||||
|
sclient.Connection(self.d_st,
|
||||||
|
"%s:%s" % (self.o_admin_tenant,
|
||||||
|
self.o_admin_user),
|
||||||
|
self.o_admin_password,
|
||||||
|
auth_version=2).get_auth()
|
||||||
|
# Instanciate syncer
|
||||||
|
self.swsync = accounts.Accounts()
|
||||||
|
|
||||||
|
def extract_created_a_u_iter(self, created):
|
||||||
|
for ad, usd in created.items():
|
||||||
|
account = ad[0]
|
||||||
|
account_id = ad[1]
|
||||||
|
# Retreive the first user as we only need one
|
||||||
|
username = usd[0][0]
|
||||||
|
yield account, account_id, username
|
||||||
|
|
||||||
|
def create_st_account_url(self, account_id):
|
||||||
|
o_account_url = \
|
||||||
|
self.o_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id
|
||||||
|
d_account_url = \
|
||||||
|
self.d_admin_auth_url.split('AUTH_')[0] + 'AUTH_' + account_id
|
||||||
|
return o_account_url, d_account_url
|
||||||
|
|
||||||
|
def verify_aco_diff(self, alo, ald):
|
||||||
|
"""Verify that 2 accounts are similar to validate migration
|
||||||
|
"""
|
||||||
|
for k, v in alo[0].items():
|
||||||
|
if k not in ('x-timestamp', 'x-trans-id',
|
||||||
|
'date', 'last-modified'):
|
||||||
|
self.assertEqual(ald[0][k], v, msg='%s differs' % k)
|
||||||
|
|
||||||
|
def delete_account_cont(self, account_url, token):
|
||||||
|
cnx = sclient.http_connection(account_url)
|
||||||
|
al = sclient.get_account(None, token,
|
||||||
|
http_conn=cnx,
|
||||||
|
full_listing=True)
|
||||||
|
for container in [c['name'] for c in al[1]]:
|
||||||
|
ci = sclient.get_container(None, token,
|
||||||
|
container, http_conn=cnx,
|
||||||
|
full_listing=True)
|
||||||
|
on = [od['name'] for od in ci[1]]
|
||||||
|
for obj in on:
|
||||||
|
sclient.delete_object('', token, container,
|
||||||
|
obj, http_conn=cnx)
|
||||||
|
sclient.delete_container('', token, container, http_conn=cnx)
|
||||||
|
|
||||||
|
def get_url(self, account_id, s_type):
|
||||||
|
# Create account storage url
|
||||||
|
o_account_url, d_account_url = self.create_st_account_url(account_id)
|
||||||
|
if s_type == 'orig':
|
||||||
|
url = o_account_url
|
||||||
|
elif s_type == 'dest':
|
||||||
|
url = d_account_url
|
||||||
|
else:
|
||||||
|
raise Exception('Unknown type')
|
||||||
|
return url
|
||||||
|
|
||||||
|
def get_cnx(self, account_id, s_type):
|
||||||
|
url = self.get_url(account_id, s_type)
|
||||||
|
return sclient.http_connection(url)
|
||||||
|
|
||||||
|
def get_account_detail(self, account_id, token, s_type):
|
||||||
|
cnx = self.get_cnx(account_id, s_type)
|
||||||
|
return sclient.get_account(None, token,
|
||||||
|
http_conn=cnx,
|
||||||
|
full_listing=True)
|
||||||
|
|
||||||
|
def list_containers(self, account_id, token, s_type):
|
||||||
|
cd = self.get_account_detail(account_id, token, s_type)
|
||||||
|
return cd[1]
|
||||||
|
|
||||||
|
def get_container_detail(self, account_id, token, s_type, container):
|
||||||
|
cnx = self.get_cnx(account_id, s_type)
|
||||||
|
return sclient.get_container(None, token, container,
|
||||||
|
http_conn=cnx, full_listing=True)
|
||||||
|
|
||||||
|
def list_objects(self, account_id, token, s_type, container):
|
||||||
|
cd = self.get_container_detail(account_id, token, s_type, container)
|
||||||
|
return cd[1]
|
||||||
|
|
||||||
|
def list_objects_in_containers(self, account_id, token, s_type):
|
||||||
|
ret = {}
|
||||||
|
cl = self.list_containers(account_id, token, s_type)
|
||||||
|
for c in [c['name'] for c in cl]:
|
||||||
|
objs = self.list_objects(account_id, token, s_type, c)
|
||||||
|
ret[c] = objs
|
||||||
|
return ret
|
||||||
|
|
||||||
|
def get_object_detail(self, account_id, token, s_type, container, obj):
|
||||||
|
cnx = self.get_cnx(account_id, s_type)
|
||||||
|
return sclient.get_object("", token, container, obj, http_conn=cnx)
|
||||||
|
|
||||||
|
def get_account_meta(self, account_id, token, s_type):
|
||||||
|
d = self.get_account_detail(account_id, token, s_type)
|
||||||
|
return {k: v for k, v in d[0].iteritems()
|
||||||
|
if k.startswith('x-account-meta')}
|
||||||
|
|
||||||
|
def get_container_meta(self, account_id, token, s_type, container):
|
||||||
|
d = self.get_container_detail(account_id, token, s_type, container)
|
||||||
|
return {k: v for k, v in d[0].iteritems()
|
||||||
|
if k.startswith('x-container-meta')}
|
||||||
|
|
||||||
|
def post_account(self, account_id, token, s_type, headers):
|
||||||
|
cnx = self.get_cnx(account_id, s_type)
|
||||||
|
sclient.post_account("", token, headers, http_conn=cnx)
|
||||||
|
|
||||||
|
def post_container(self, account_id, token, s_type, container, headers):
|
||||||
|
cnx = self.get_cnx(account_id, s_type)
|
||||||
|
sclient.post_container("", token, container, headers, http_conn=cnx)
|
||||||
|
|
||||||
|
def put_container(self, account_id, token, s_type, container):
|
||||||
|
cnx = self.get_cnx(account_id, s_type)
|
||||||
|
sclient.put_container("", token, container, http_conn=cnx)
|
||||||
|
|
||||||
|
def delete_container(self, account_id, token, s_type, container):
|
||||||
|
cnx = self.get_cnx(account_id, s_type)
|
||||||
|
sclient.delete_container("", token, container, http_conn=cnx)
|
||||||
|
|
||||||
|
def post_object(self, account_id, token, s_type, container, name, headers):
|
||||||
|
cnx = self.get_cnx(account_id, s_type)
|
||||||
|
sclient.post_object("", token, container, name, headers, http_conn=cnx)
|
||||||
|
|
||||||
|
def put_object(self, account_id, token, s_type, container, name, content):
|
||||||
|
cnx = self.get_cnx(account_id, s_type)
|
||||||
|
sclient.put_object("", token, container, name, content, http_conn=cnx)
|
||||||
|
|
||||||
|
def delete_object(self, account_id, token, s_type,
|
||||||
|
container, name):
|
||||||
|
cnx = self.get_cnx(account_id, s_type)
|
||||||
|
sclient.delete_object("", token, container, name,
|
||||||
|
http_conn=cnx)
|
||||||
|
|
||||||
|
def test_01_sync_one_of_two_empty_accounts(self):
|
||||||
|
"""create two empty accounts, Sync only one
|
||||||
|
"""
|
||||||
|
index = {}
|
||||||
|
test_account_name = "account_test"
|
||||||
|
|
||||||
|
# create account
|
||||||
|
self.created = filler.create_swift_account(self.o_ks_client,
|
||||||
|
self.pile,
|
||||||
|
2, 1, index)
|
||||||
|
|
||||||
|
for account, account_id, username in \
|
||||||
|
self.extract_created_a_u_iter(self.created):
|
||||||
|
|
||||||
|
# post meta data on account
|
||||||
|
tenant_cnx = sclient.Connection(self.o_st,
|
||||||
|
"%s:%s" % (account, username),
|
||||||
|
self.default_user_password,
|
||||||
|
auth_version=2)
|
||||||
|
filler.create_account_meta(tenant_cnx)
|
||||||
|
|
||||||
|
# select random account and write it in the filter file
|
||||||
|
t_account, t_account_id, t_username = random.choice(list(
|
||||||
|
self.extract_created_a_u_iter(self.created)))
|
||||||
|
with open(self.filter_filename, "w") as filterlist:
|
||||||
|
filterlist.write(t_account + "\n")
|
||||||
|
|
||||||
|
# start sync process
|
||||||
|
self.swsync.process()
|
||||||
|
|
||||||
|
# Now verify dest
|
||||||
|
for account, account_id, username in \
|
||||||
|
self.extract_created_a_u_iter(self.created):
|
||||||
|
alo = self.get_account_detail(account_id,
|
||||||
|
self.o_admin_token, 'orig')
|
||||||
|
ald = self.get_account_detail(account_id,
|
||||||
|
self.d_admin_token, 'dest')
|
||||||
|
if account == t_account:
|
||||||
|
self.verify_aco_diff(alo, ald)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
if self.created:
|
||||||
|
for k, v in self.created.items():
|
||||||
|
user_info_list = [user[1] for user in v]
|
||||||
|
account_id = k[1]
|
||||||
|
o_account_url, d_account_url = \
|
||||||
|
self.create_st_account_url(account_id)
|
||||||
|
# Remove account content on origin and destination
|
||||||
|
self.delete_account_cont(o_account_url, self.o_admin_token)
|
||||||
|
self.delete_account_cont(d_account_url, self.d_admin_token)
|
||||||
|
# We just need to delete keystone accounts and users
|
||||||
|
# in origin keystone as syncer does not sync
|
||||||
|
# keystone database
|
||||||
|
filler.delete_account(self.o_ks_client,
|
||||||
|
user_info_list,
|
||||||
|
k)
|
|
@ -19,6 +19,8 @@ import random
|
||||||
import urlparse
|
import urlparse
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
|
from swsync.utils import ConfigurationError
|
||||||
|
|
||||||
STORAGE_ORIG = 'http://storage-orig.com'
|
STORAGE_ORIG = 'http://storage-orig.com'
|
||||||
STORAGE_DEST = 'http://storage-dest.com'
|
STORAGE_DEST = 'http://storage-dest.com'
|
||||||
|
|
||||||
|
@ -67,7 +69,14 @@ CONFIGDICT = {'auth':
|
||||||
|
|
||||||
|
|
||||||
def fake_get_config(section, option):
|
def fake_get_config(section, option):
|
||||||
return CONFIGDICT[section][option]
|
try:
|
||||||
|
return CONFIGDICT[section][option]
|
||||||
|
except KeyError:
|
||||||
|
raise ConfigurationError
|
||||||
|
|
||||||
|
|
||||||
|
def fake_get_filter(self):
|
||||||
|
return {'foo1', 'foo2', 'foo3'}
|
||||||
|
|
||||||
|
|
||||||
class FakeSWConnection(object):
|
class FakeSWConnection(object):
|
||||||
|
@ -121,6 +130,7 @@ def fake_get_auth(auth_url, tenant, user, password):
|
||||||
class FakeKSTenant(object):
|
class FakeKSTenant(object):
|
||||||
def __init__(self, tenant_name):
|
def __init__(self, tenant_name):
|
||||||
self.tenant_name = tenant_name
|
self.tenant_name = tenant_name
|
||||||
|
self.name = tenant_name
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def id(self):
|
def id(self):
|
||||||
|
|
|
@ -35,6 +35,8 @@ class TestAccountBase(tests.units.base.TestCase):
|
||||||
self.stubs.Set(swiftclient.client, 'Connection',
|
self.stubs.Set(swiftclient.client, 'Connection',
|
||||||
fakes.FakeSWConnection)
|
fakes.FakeSWConnection)
|
||||||
self.stubs.Set(swsync.accounts, 'get_config', fakes.fake_get_config)
|
self.stubs.Set(swsync.accounts, 'get_config', fakes.fake_get_config)
|
||||||
|
self.stubs.Set(swsync.accounts.Accounts, 'get_target_tenant_filter',
|
||||||
|
fakes.fake_get_filter)
|
||||||
self.stubs.Set(swiftclient, 'http_connection',
|
self.stubs.Set(swiftclient, 'http_connection',
|
||||||
fakes.FakeSWClient.http_connection)
|
fakes.FakeSWClient.http_connection)
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue