Add a basic syncer.
- Just basic sync accounts, there is plenty of more works to do.
This commit is contained in:
Chmouel Boudjnah
parent
966937da0d
commit
9837669022
|
|
@ -0,0 +1,33 @@
|
|||
#!/usr/bin/python
|
||||
# -*- encoding: utf-8 -*-
|
||||
import swiftclient
|
||||
|
||||
from sync.containers import sync_container
|
||||
from utils import get_config, get_auth
|
||||
|
||||
orig_auth_url = get_config('auth', 'keystone_origin')
|
||||
orig_tenant, orig_user, orig_password = \
|
||||
get_config('auth', 'keystone_origin_credentials').split(':')
|
||||
|
||||
dest_auth_url = get_config('auth', 'keystone_dest')
|
||||
dest_tenant, dest_user, dest_password = \
|
||||
get_config('auth', 'keystone_dest_credentials').split(':')
|
||||
|
||||
orig_storage_url, orig_token = \
|
||||
get_auth(orig_auth_url, orig_tenant, orig_user, orig_password)
|
||||
orig_storage_cnx = swiftclient.http_connection(orig_storage_url)
|
||||
|
||||
dest_storage_url, dest_token = \
|
||||
get_auth(dest_auth_url, dest_tenant, dest_user, dest_password)
|
||||
dest_storage_cnx = swiftclient.http_connection(dest_storage_url)
|
||||
|
||||
|
||||
orig_account_stats, orig_containers = swiftclient.get_account(
|
||||
None, orig_token, http_conn=orig_storage_cnx, full_listing=True
|
||||
)
|
||||
|
||||
for container in orig_containers:
|
||||
print "Synching %s.." % (container)
|
||||
sync_container(orig_storage_cnx, orig_storage_url, orig_token,
|
||||
dest_storage_cnx, dest_storage_url, dest_token,
|
||||
container['name'])
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
# -*- encoding: utf-8 -*-
|
||||
__author__ = "Chmouel Boudjnah <chmouel@chmouel.com>"
|
||||
import os
|
||||
import ConfigParser
|
||||
import swiftclient.client as swclient
|
||||
|
||||
|
||||
CONFIG = {}
|
||||
curdir = os.path.abspath(os.path.dirname(__file__))
|
||||
INIFILE = os.path.abspath(os.path.join(curdir, '..', 'etc', "config.ini"))
|
||||
|
||||
|
||||
class ConfigurationError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def parse_ini(inifile=INIFILE):
|
||||
if not os.path.exists(inifile):
|
||||
raise ConfigurationError("Error while parsing inifile")
|
||||
|
||||
config = ConfigParser.RawConfigParser()
|
||||
config.read(inifile)
|
||||
return config
|
||||
|
||||
|
||||
def get_config(section, option, default=None):
|
||||
"""Get section/option from ConfigParser or print default if specified"""
|
||||
global CONFIG
|
||||
|
||||
if not CONFIG:
|
||||
CONFIG = parse_ini()
|
||||
|
||||
if not CONFIG.has_section(section):
|
||||
raise ConfigurationError("Invalid configuration, missing section: %s" %
|
||||
section)
|
||||
if CONFIG.has_option(section, option):
|
||||
return CONFIG.get(section, option)
|
||||
elif not default is None:
|
||||
return default
|
||||
else:
|
||||
raise ConfigurationError("Invalid configuration, missing "
|
||||
"section/option: %s/%s" % (section, option))
|
||||
|
||||
|
||||
def get_auth(auth_url, tenant, user, password):
|
||||
return swclient.Connection(
|
||||
auth_url,
|
||||
'%s:%s' % (tenant, user),
|
||||
password,
|
||||
auth_version=2).get_auth()
|
||||
|
||||
if __name__ == '__main__':
|
||||
get_config("foo", "bar")
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
[auth]
|
||||
keystone_origin = http://vm:5000/v2.0
|
||||
keystone_origin_credentials = demo:demo:ADMIN
|
||||
keystone_dest = http://vm2:5000/v2.0
|
||||
keystone_dest_credentials = demo:demo:ADMIN
|
||||
|
|
@ -0,0 +1 @@
|
|||
# -*- encoding: utf-8 -*-
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
import sys
|
||||
|
||||
import swiftclient
|
||||
import eventlet
|
||||
|
||||
from sync.objects import sync_object
|
||||
|
||||
|
||||
def sync_container(orig_storage_cnx, orig_storage_url,
|
||||
orig_token, dest_storage_cnx,
|
||||
dest_storage_url, dest_token,
|
||||
container_name):
|
||||
orig_container_stats, orig_objects = swiftclient.get_container(
|
||||
None, orig_token, container_name, http_conn=orig_storage_cnx,
|
||||
)
|
||||
|
||||
try:
|
||||
swiftclient.head_container(
|
||||
"", dest_token, container_name, http_conn=dest_storage_cnx
|
||||
)
|
||||
except(swiftclient.client.ClientException):
|
||||
container_headers = orig_container_stats.copy()
|
||||
for h in ('x-container-object-count', 'x-trans-id',
|
||||
'x-container-bytes-used'):
|
||||
del container_headers[h]
|
||||
p = dest_storage_cnx[0]
|
||||
url = "%s://%s%s" % (p.scheme, p.netloc, p.path)
|
||||
swiftclient.put_container(url,
|
||||
dest_token, container_name,
|
||||
headers=container_headers)
|
||||
|
||||
|
||||
dest_container_stats, dest_objects = swiftclient.get_container(
|
||||
None, dest_token, container_name, http_conn=dest_storage_cnx,
|
||||
)
|
||||
|
||||
set1 = set((x['hash'], x['name']) for x in orig_objects)
|
||||
set2 = set((x['hash'], x['name']) for x in dest_objects)
|
||||
diff = set1 - set2
|
||||
if not diff:
|
||||
return
|
||||
|
||||
pool = eventlet.GreenPool()
|
||||
cnt = 0
|
||||
for obj in diff:
|
||||
pool.spawn_n(sync_object,
|
||||
orig_storage_url,
|
||||
orig_token,
|
||||
dest_storage_url,
|
||||
dest_token, container_name,
|
||||
obj)
|
||||
if cnt == 20:
|
||||
pool.waitall()
|
||||
cnt = 0
|
||||
else:
|
||||
cnt += 1
|
||||
|
||||
pool.waitall()
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
# -*- encoding: utf-8 -*-
|
||||
from swiftclient import client as swiftclient
|
||||
from swift.common.bufferedhttp import http_connect_raw
|
||||
from swift.common.http import is_success
|
||||
from swift.container.sync import _Iter2FileLikeObject
|
||||
from eventlet import Timeout
|
||||
import urllib2
|
||||
|
||||
|
||||
def get_object(storage_url, token,
|
||||
container_name,
|
||||
object_name,
|
||||
response_timeout=15,
|
||||
conn_timeout=5,
|
||||
resp_chunk_size=65536):
|
||||
headers = {'x-auth-token': token}
|
||||
x = urllib2.urlparse.urlparse(storage_url)
|
||||
|
||||
path = x.path + '/' + container_name + '/' + object_name
|
||||
with Timeout(conn_timeout):
|
||||
conn = http_connect_raw(
|
||||
x.hostname,
|
||||
x.port,
|
||||
'GET',
|
||||
path,
|
||||
headers=headers,
|
||||
ssl=False)
|
||||
|
||||
with Timeout(response_timeout):
|
||||
resp = conn.getresponse()
|
||||
|
||||
if not is_success(resp.status):
|
||||
resp.read()
|
||||
raise swiftclient.ClientException(
|
||||
'status %s %s' % (resp.status, resp.reason))
|
||||
|
||||
if resp_chunk_size:
|
||||
def _object_body():
|
||||
buf = resp.read(resp_chunk_size)
|
||||
while buf:
|
||||
yield buf
|
||||
buf = resp.read(resp_chunk_size)
|
||||
object_body = _object_body()
|
||||
else:
|
||||
object_body = resp.read()
|
||||
|
||||
resp_headers = {}
|
||||
for header, value in resp.getheaders():
|
||||
resp_headers[header.lower()] = value
|
||||
|
||||
return (resp_headers, object_body)
|
||||
|
||||
|
||||
def sync_object(orig_storage_url, orig_token, dest_storage_url,
|
||||
dest_token, container_name, object_name_etag):
|
||||
orig_headers, orig_body = get_object(orig_storage_url,
|
||||
orig_token,
|
||||
container_name,
|
||||
object_name_etag[1],
|
||||
)
|
||||
post_headers = orig_headers
|
||||
post_headers['x-auth-token'] = dest_token
|
||||
sync_to = dest_storage_url + "/" + container_name
|
||||
swiftclient.put_object(sync_to, name=object_name_etag[1],
|
||||
headers=post_headers,
|
||||
contents=_Iter2FileLikeObject(orig_body))
|
||||
|
||||
if __name__ == '__main__':
|
||||
pass
|
||||
Loading…
Reference in New Issue