Change the threadpool is constructed.

This commit is contained in:
Chmouel Boudjnah 2013-03-25 14:16:48 +01:00
parent c0601d7a4e
commit 53f95305e0
3 changed files with 12 additions and 14 deletions

View File

@ -9,7 +9,7 @@ keystone_dest_credentials = demo:demo:ADMIN
[sync]
# This is usually bound to the max open files.
max_gthreads = 500
max_gthreads = 10
[filler]
swift_operator_role = Member

View File

@ -41,7 +41,7 @@ def sync_an_account(orig_storage_url,
None, orig_token, http_conn=orig_storage_cnx, full_listing=True)
for container in orig_containers:
print container,
print container
dt1 = datetime.datetime.fromtimestamp(time.time())
sync_container(orig_storage_cnx, orig_storage_url, orig_token,
dest_storage_cnx, dest_storage_url, dest_token,
@ -51,6 +51,7 @@ def sync_an_account(orig_storage_url,
rd = dateutil.relativedelta.relativedelta(dt2, dt1)
print "%d hours, %d minutes and %d seconds" % (rd.hours, rd.minutes,
rd.seconds)
print
def sync_accounts():

View File

@ -1,6 +1,10 @@
import sys
import os
import swiftclient
import eventlet
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from sync.objects import sync_object
from common.utils import get_config
@ -34,28 +38,21 @@ def sync_container(orig_storage_cnx, orig_storage_url,
None, dest_token, container_name, http_conn=dest_storage_cnx,
)
set1 = set((x['hash'], x['name']) for x in orig_objects)
set2 = set((x['hash'], x['name']) for x in dest_objects)
set1 = set((x['last_modified'], x['name']) for x in orig_objects)
set2 = set((x['last_modified'], x['name']) for x in dest_objects)
diff = set1 - set2
if not diff:
return
pool = eventlet.GreenPool()
count = 0
pool = eventlet.GreenPool(size=int(MAX_GTHREADS))
pile = eventlet.GreenPile(pool)
for obj in diff:
print obj
pile.spawn(sync_object,
orig_storage_url,
orig_token,
dest_storage_url,
dest_token, container_name,
obj)
if count == MAX_GTHREADS:
pool.waitall()
pile = eventlet.GreenPile(pool)
count = 0
else:
count += 1
pool.waitall()