stackforge
/
swiftsync
Code Issues Proposed changes

sync account.

This commit is contained in:
Chmouel Boudjnah 2013-03-20 18:18:33 +01:00
parent 6a17e9ee27
commit 63de1098d1
7 changed files with 98 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bin/filler.py
View File

@ -9,7 +9,6 @@ import sys
import eventlet
from keystoneclient.v2_0 import client as ksclient
sys.path.append("../")
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from common.utils import get_config
from sync.filler import (load_index, load_containers_index,

33
bin/sync-accounts.py
View File

@ -1,33 +0,0 @@
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
import swiftclient
from sync.containers import sync_container
from utils import get_config, get_auth
orig_auth_url = get_config('auth', 'keystone_origin')
orig_tenant, orig_user, orig_password = \
get_config('auth', 'keystone_origin_credentials').split(':')
dest_auth_url = get_config('auth', 'keystone_dest')
dest_tenant, dest_user, dest_password = \
get_config('auth', 'keystone_dest_credentials').split(':')
orig_storage_url, orig_token = \
get_auth(orig_auth_url, orig_tenant, orig_user, orig_password)
orig_storage_cnx = swiftclient.http_connection(orig_storage_url)
dest_storage_url, dest_token = \
get_auth(dest_auth_url, dest_tenant, dest_user, dest_password)
dest_storage_cnx = swiftclient.http_connection(dest_storage_url)
orig_account_stats, orig_containers = swiftclient.get_account(
None, orig_token, http_conn=orig_storage_cnx, full_listing=True
)
for container in orig_containers:
print "Synching %s.." % (container)
sync_container(orig_storage_cnx, orig_storage_url, orig_token,
dest_storage_cnx, dest_storage_url, dest_token,
container['name'])

11
bin/syncit.py Executable file
View File

@ -0,0 +1,11 @@
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
import sys
import os
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from sync.accounts import sync_accounts
if __name__ == '__main__':
sync_accounts()

1
etc/config-sample.ini
View File

@ -3,6 +3,7 @@ keystone_origin = http://vm:5000/v2.0
keystone_origin_admin_credentials = admin:admin:ADMIN
keystone_dest = http://vm2:5000/v2.0
keystone_origin_demo_credentials = demo:demo:ADMIN
keystone_dest_credentials = demo:demo:ADMIN

71
sync/accounts.py Normal file
View File

@ -0,0 +1,71 @@
# -*- encoding: utf-8 -*-
import os
import sys
import swiftclient
from keystoneclient.v2_0 import client as ksclient
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from common.utils import get_config, get_auth
from sync.containers import sync_container
def get_ks_auth_orig():
orig_auth_url = get_config('auth', 'keystone_origin')
tenant_name, username, password = \
get_config('auth', 'keystone_origin_admin_credentials').split(':')
return ksclient.Client(auth_url=orig_auth_url,
username=username,
password=password,
tenant_name=tenant_name)
def list_accounts(cnx):
for x in cnx.tenants.list():
yield x
def sync_an_account(orig_storage_url,
orig_token,
dest_storage_url,
dest_token):
orig_storage_cnx = swiftclient.http_connection(orig_storage_url)
dest_storage_cnx = swiftclient.http_connection(dest_storage_url)
orig_account_stats, orig_containers = swiftclient.get_account(
None, orig_token, http_conn=orig_storage_cnx, full_listing=True)
for container in orig_containers:
print "Synching %s.." % (container)
sync_container(orig_storage_cnx, orig_storage_url, orig_token,
dest_storage_cnx, dest_storage_url, dest_token,
container['name'])
def sync_accounts():
orig_auth_url = get_config('auth', 'keystone_origin')
orig_admin_tenant, orig_admin_user, orig_admin_password = (
get_config('auth', 'keystone_origin_admin_credentials').split(':'))
oa_st_url, orig_admin_token = (get_auth(orig_auth_url, orig_admin_tenant,
orig_admin_user,
orig_admin_password))
dest_auth_url = get_config('auth', 'keystone_dest')
# we assume orig and dest passwd are the same obv synchronized.
dst_st_url, dest_admin_token = (get_auth(dest_auth_url, orig_admin_tenant,
orig_admin_user,
orig_admin_password))
bare_oa_st_url = oa_st_url[:oa_st_url.find('AUTH_')] + "AUTH_"
bare_dst_st_url = dst_st_url[:dst_st_url.find('AUTH_')] + "AUTH_"
for x in list_accounts(get_ks_auth_orig()):
user_orig_st_url = bare_oa_st_url + x.id
user_dst_st_url = bare_dst_st_url + x.id
sync_an_account(user_orig_st_url,
orig_admin_token,
user_dst_st_url,
dest_admin_token)

3
sync/containers.py
View File

@ -1,5 +1,3 @@
import sys
import swiftclient
import eventlet
@ -29,7 +27,6 @@ def sync_container(orig_storage_cnx, orig_storage_url,
dest_token, container_name,
headers=container_headers)
dest_container_stats, dest_objects = swiftclient.get_container(
None, dest_token, container_name, http_conn=dest_storage_cnx,
)

16
sync/objects.py
View File

@ -5,6 +5,16 @@ from swift.common.http import is_success
from swift.container.sync import _Iter2FileLikeObject
from eventlet import Timeout
import urllib2
from urllib import quote as urllib_quote
def quote(value, safe='/'):
"""
Patched version of urllib.quote that encodes utf-8 strings before quoting
"""
if isinstance(value, unicode):
value = value.encode('utf-8')
return urllib_quote(value, safe)
def get_object(storage_url, token,
@ -17,6 +27,7 @@ def get_object(storage_url, token,
x = urllib2.urlparse.urlparse(storage_url)
path = x.path + '/' + container_name + '/' + object_name
path = quote(path)
with Timeout(conn_timeout):
conn = http_connect_raw(
x.hostname,
@ -58,10 +69,13 @@ def sync_object(orig_storage_url, orig_token, dest_storage_url,
container_name,
object_name_etag[1],
)
container_name = quote(container_name)
object_name = quote(object_name_etag[1])
post_headers = orig_headers
post_headers['x-auth-token'] = dest_token
sync_to = dest_storage_url + "/" + container_name
swiftclient.put_object(sync_to, name=object_name_etag[1],
swiftclient.put_object(sync_to, name=object_name,
headers=post_headers,
contents=_Iter2FileLikeObject(orig_body))