From ab1f11f7bc0eb8f9e7d1a2c8ea2dbec87cff466d Mon Sep 17 00:00:00 2001 From: Guillaume Espanel Date: Fri, 9 Sep 2016 17:38:10 +0200 Subject: [PATCH] Parallelize API calls for resources fetching This change improves the performance of flame by fetching the different Openstack resources through a ThreadPoolExecutor. The resources to fetch are declared in a dict inside flame.TemplateGenerator.extract_vm_details of the form {'resource_type' : (fetch_method, filter_method), ...} Co-Authored-By: zarrouk Change-Id: I8f34ecbfff236e5469b83d1c79d1f98accb125c0 --- flameclient/flame.py | 55 +++++++++++++++++++++++++++++++------------ requirements.txt | 1 + test-requirements.txt | 1 + 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/flameclient/flame.py b/flameclient/flame.py index d9d2dc5..781faa3 100644 --- a/flameclient/flame.py +++ b/flameclient/flame.py @@ -24,7 +24,9 @@ import logging +import concurrent.futures import netaddr +import six import yaml from flameclient import managers @@ -107,6 +109,7 @@ class TemplateGenerator(object): def __init__(self, username, password, tenant_name, auth_url, auth_token=None, insecure=False, endpoint_type='publicURL', region_name=None): + self.thread_pool = concurrent.futures.ThreadPoolExecutor(10) self.generate_data = False self._setup_templates() self._setup_managers(username, password, tenant_name, auth_url, @@ -136,34 +139,56 @@ class TemplateGenerator(object): self.nova = managers.NovaManager(self.keystone) self.cinder = managers.CinderManager(self.keystone) + def async_fetch_data(self, fetch_map): + """Call the methods in fetch_map in parallel and filter the results. + + fetch_map is a dict of the form : + {resource_type, (fetch_method, filter_method), ...} + + This method returns an iterator on the filtered results. + The iterator items are of the form (res_type, result) + """ + + future_res = {} + with self.thread_pool as tp: + for res_type, (method, data_filter) in six.iteritems(fetch_map): + future_res[tp.submit(method)] = res_type + for res_available in concurrent.futures.as_completed(future_res): + res = res_available.result() + res_type = future_res[res_available] + yield res_type, fetch_map[res_type][1](res) + def extract_vm_details(self, exclude_servers, exclude_volumes, exclude_keypairs, generate_data): self.exclude_servers = exclude_servers self.exclude_volumes = exclude_volumes self.exclude_keypairs = exclude_keypairs self.generate_data = generate_data - - self.subnets = self.build_data(self.neutron.subnet_list()) - self.networks = self.build_data(self.neutron.network_list()) - self.routers = self.neutron.router_list() - self.secgroups = self.build_data(self.neutron.secgroup_list()) - self.servergroups = self.build_data(self.nova.servergroup_list()) - self.floatingips = self.neutron.floatingip_list() - self.ports = self.build_data(self.neutron.port_list()) self.external_networks = [] - + fetch_map = { + 'subnets': (self.neutron.subnet_list, self.build_data), + 'networks': (self.neutron.network_list, self.build_data), + 'routers': (self.neutron.router_list, lambda x: x), + 'secgroups': (self.neutron.secgroup_list, self.build_data), + 'servergroups': (self.nova.servergroup_list, self.build_data), + 'floatingips': (self.neutron.floatingip_list, lambda x: x), + 'ports': (self.neutron.port_list, self.build_data), + } if not exclude_keypairs: - self.keys = dict( - (key.name, (index, key)) - for index, key in enumerate(self.nova.keypair_list())) + fetch_map['keys'] = (self.nova.keypair_list, + lambda l: {key.name: (index, key) for + index, key in enumerate(l)}) if not exclude_servers: - self.flavors = self.build_data(self.nova.flavor_list()) - self.servers = self.build_data(self.nova.server_list()) + fetch_map['flavors'] = (self.nova.flavor_list, self.build_data) + fetch_map['servers'] = (self.nova.server_list, self.build_data) if (not exclude_volumes or (exclude_volumes and not exclude_servers)): - self.volumes = self.build_data(self.cinder.volume_list()) + fetch_map['volumes'] = (self.cinder.volume_list, self.build_data) + + for res_type, result in self.async_fetch_data(fetch_map): + self.__setattr__(res_type, result) def build_data(self, data): if not data: diff --git a/requirements.txt b/requirements.txt index a007723..f21f768 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ pbr>=1.6 Babel>=1.3 +futures netaddr>=0.7.12,!=0.7.16 python-keystoneclient diff --git a/test-requirements.txt b/test-requirements.txt index c7fbc6c..0cf0d0a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,6 @@ hacking>=0.10.2,<0.11 +futures coverage>=3.6 discover mock>=1.2