From 94c077be50ee150bb94cb9ccb92e7bb7123c0327 Mon Sep 17 00:00:00 2001 From: Amelia Cordwell Date: Mon, 11 Dec 2017 13:55:45 +1300 Subject: [PATCH] Move more quota logic into QuotaManager * Makes it easier to add more services to the quota actions * Fixes issues with endpoints that don't exist * Allows deployers to override the services using the quotas available Change-Id: Iff64d33a7f3773d5c9b1674c3dccb4804804b0a0 --- adjutant/actions/v1/resources.py | 47 +--- .../actions/v1/tests/test_resource_actions.py | 21 +- adjutant/common/quota.py | 222 ++++++++++++------ adjutant/settings.py | 4 + adjutant/test_settings.py | 3 + conf/conf.yaml | 6 + 6 files changed, 172 insertions(+), 131 deletions(-) diff --git a/adjutant/actions/v1/resources.py b/adjutant/actions/v1/resources.py index c940c3c..0361613 100644 --- a/adjutant/actions/v1/resources.py +++ b/adjutant/actions/v1/resources.py @@ -235,37 +235,6 @@ class UpdateProjectQuotasAction(BaseAction, QuotaMixin): default_days_between_autoapprove = 30 - class ServiceQuotaFunctor(object): - def __call__(self, project_id, values): - self.client.quotas.update(project_id, **values) - - class ServiceQuotaCinderFunctor(ServiceQuotaFunctor): - def __init__(self, region_name): - self.client = openstack_clients.get_cinderclient( - region=region_name) - - class ServiceQuotaNovaFunctor(ServiceQuotaFunctor): - def __init__(self, region_name): - self.client = openstack_clients.get_novaclient( - region=region_name) - - class ServiceQuotaNeutronFunctor(ServiceQuotaFunctor): - def __init__(self, region_name): - self.client = openstack_clients.get_neutronclient( - region=region_name) - - def __call__(self, project_id, values): - body = { - 'quota': values - } - self.client.update_quota(project_id, body) - - _quota_updaters = { - 'cinder': ServiceQuotaCinderFunctor, - 'nova': ServiceQuotaNovaFunctor, - 'neutron': ServiceQuotaNeutronFunctor - } - def __init__(self, *args, **kwargs): super(UpdateProjectQuotasAction, self).__init__(*args, **kwargs) self.size_difference_threshold = settings.TASK_SETTINGS.get( @@ -302,15 +271,11 @@ class UpdateProjectQuotasAction(BaseAction, QuotaMixin): quota_size, region_name)) return - for service_name, values in quota_settings.items(): - updater_class = self._quota_updaters.get(service_name) - if not updater_class: - self.add_note("No quota updater found for %s. Ignoring" % - service_name) - continue - # functor for the service+region - service_functor = updater_class(region_name) - service_functor(self.project_id, values) + quota_manager = QuotaManager(self.project_id, + self.size_difference_threshold) + + quota_manager.set_region_quota(region_name, quota_settings) + self.add_note("Project quota for region %s set to %s" % ( region_name, quota_size)) @@ -387,6 +352,8 @@ class UpdateProjectQuotasAction(BaseAction, QuotaMixin): if not self.valid or self.action.state == "completed": return + # Use manager here instead, it will make it easier to add has_more + # in later for region in self.regions: self._set_region_quota(region, self.size) diff --git a/adjutant/actions/v1/tests/test_resource_actions.py b/adjutant/actions/v1/tests/test_resource_actions.py index 2c90b18..476df58 100644 --- a/adjutant/actions/v1/tests/test_resource_actions.py +++ b/adjutant/actions/v1/tests/test_resource_actions.py @@ -35,12 +35,13 @@ from adjutant.common.tests.fake_clients import ( 'openstack_clients.get_neutronclient', get_fake_neutron) @mock.patch( - 'adjutant.actions.v1.resources.' + - 'openstack_clients.get_novaclient', + 'adjutant.common.quota.get_neutronclient', + get_fake_neutron) +@mock.patch( + 'adjutant.common.quota.get_novaclient', get_fake_novaclient) @mock.patch( - 'adjutant.actions.v1.resources.' + - 'openstack_clients.get_cinderclient', + 'adjutant.common.quota.get_cinderclient', get_fake_cinderclient) class ProjectSetupActionTests(TestCase): @@ -475,18 +476,6 @@ class ProjectSetupActionTests(TestCase): @mock.patch( 'adjutant.common.quota.get_cinderclient', get_fake_cinderclient) -@mock.patch( - 'adjutant.actions.v1.resources.' + - 'openstack_clients.get_neutronclient', - get_fake_neutron) -@mock.patch( - 'adjutant.actions.v1.resources.' + - 'openstack_clients.get_novaclient', - get_fake_novaclient) -@mock.patch( - 'adjutant.actions.v1.resources.' + - 'openstack_clients.get_cinderclient', - get_fake_cinderclient) class QuotaActionTests(TestCase): def test_update_quota(self): diff --git a/adjutant/common/quota.py b/adjutant/common/quota.py index f5a4679..4a7deca 100644 --- a/adjutant/common/quota.py +++ b/adjutant/common/quota.py @@ -27,22 +27,133 @@ class QuotaManager(object): default_size_diff_threshold = .2 + class ServiceQuotaHelper(object): + def set_quota(self, values): + self.client.quotas.update(self.project_id, **values) + + class ServiceQuotaCinderHelper(ServiceQuotaHelper): + def __init__(self, region_name, project_id): + self.client = get_cinderclient( + region=region_name) + self.project_id = project_id + + def get_quota(self): + return self.client.quotas.get(self.project_id).to_dict() + + def get_usage(self): + volumes = self.client.volumes.list( + search_opts={'all_tenants': 1, 'project_id': self.project_id}) + snapshots = self.client.volume_snapshots.list( + search_opts={'all_tenants': 1, 'project_id': self.project_id}) + + # gigabytesUsed should be a total of volumes and snapshots + gigabytes = sum([getattr(volume, 'size', 0) for volume + in volumes]) + gigabytes += sum([getattr(snap, 'size', 0) for snap + in snapshots]) + + return {'gigabytes': gigabytes, + 'volumes': len(volumes), + 'snapshots': len(snapshots) + } + + class ServiceQuotaNovaHelper(ServiceQuotaHelper): + def __init__(self, region_name, project_id): + self.client = get_novaclient( + region=region_name) + self.project_id = project_id + + def get_quota(self): + return self.client.quotas.get(self.project_id).to_dict() + + def get_usage(self): + nova_usage = self.client.limits.get( + tenant_id=self.project_id).to_dict()['absolute'] + nova_usage_keys = [ + ('instances', 'totalInstancesUsed'), + ('floating_ips', 'totalFloatingIpsUsed'), + ('ram', 'totalRAMUsed'), + ('cores', 'totalCoresUsed'), + ('secuirty_groups', 'totalSecurityGroupsUsed') + ] + + nova_usage_dict = {} + for key, usage_key in nova_usage_keys: + nova_usage_dict[key] = nova_usage[usage_key] + + return nova_usage_dict + + class ServiceQuotaNeutronHelper(ServiceQuotaHelper): + def __init__(self, region_name, project_id): + self.client = get_neutronclient( + region=region_name) + self.project_id = project_id + + def set_quota(self, values): + body = { + 'quota': values + } + self.client.update_quota(self.project_id, body) + + def get_usage(self): + networks = self.client.list_networks( + tenant_id=self.project_id)['networks'] + routers = self.client.list_routers( + tenant_id=self.project_id)['routers'] + floatingips = self.client.list_floatingips( + tenant_id=self.project_id)['floatingips'] + ports = self.client.list_ports( + tenant_id=self.project_id)['ports'] + subnets = self.client.list_subnets( + tenant_id=self.project_id)['subnets'] + security_groups = self.client.list_security_groups( + tenant_id=self.project_id)['security_groups'] + security_group_rules = self.client.list_security_group_rules( + tenant_id=self.project_id)['security_group_rules'] + + return {'network': len(networks), + 'router': len(routers), + 'floatingip': len(floatingips), + 'port': len(ports), + 'subnet': len(subnets), + 'secuirty_group': len(security_groups), + 'security_group_rule': len(security_group_rules) + } + + def get_quota(self): + return self.client.show_quota(self.project_id)['quota'] + + _quota_updaters = { + 'cinder': ServiceQuotaCinderHelper, + 'nova': ServiceQuotaNovaHelper, + 'neutron': ServiceQuotaNeutronHelper + } + def __init__(self, project_id, size_difference_threshold=None): + # TODO(amelia): Try to find out which endpoints are available and get + # the non enabled ones out of the list + + # Check configured removal of quota updaters + self.helpers = dict(self._quota_updaters) + + # Configurable services + if settings.QUOTA_SERVICES: + self.helpers = {} + for name in settings.QUOTA_SERVICES: + if name in self._quota_updaters: + self.helpers[name] = self._quota_updaters[name] + self.project_id = project_id self.size_diff_threshold = (size_difference_threshold or self.default_size_diff_threshold) def get_current_region_quota(self, region_id): - ci_quota = get_cinderclient(region_id) \ - .quotas.get(self.project_id).to_dict() - neutron_quota = get_neutronclient(region_id) \ - .show_quota(self.project_id)['quota'] - nova_quota = get_novaclient(region_id) \ - .quotas.get(self.project_id).to_dict() + current_quota = {} + for name, service in self.helpers.items(): + helper = service(region_id, self.project_id) + current_quota[name] = helper.get_quota() - return {'cinder': ci_quota, - 'nova': nova_quota, - 'neutron': neutron_quota} + return current_quota def get_quota_size(self, current_quota, difference_threshold=None): """ Gets the closest matching quota size for a given quota """ @@ -104,73 +215,34 @@ class QuotaManager(object): } def get_current_usage(self, region_id): - cinder_usage = self.get_cinder_usage(region_id) - nova_usage = self.get_nova_usage(region_id) - neutron_usage = self.get_neutron_usage(region_id) - return {'cinder': cinder_usage, - 'nova': nova_usage, - 'neutron': neutron_usage} + current_usage = {} - def get_cinder_usage(self, region_id): - client = get_cinderclient(region_id) + for name, service in self.helpers.items(): + try: + helper = service(region_id, self.project_id) + current_usage[name] = helper.get_usage() + except Exception: + pass + return current_usage - volumes = client.volumes.list( - search_opts={'all_tenants': 1, 'project_id': self.project_id}) - snapshots = client.volume_snapshots.list( - search_opts={'all_tenants': 1, 'project_id': self.project_id}) + def set_region_quota(self, region_id, quota_dict): + notes = [] + for service_name, values in quota_dict.items(): + updater_class = self.helpers.get(service_name) + if not updater_class: + notes.append("No quota updater found for %s. Ignoring" % + service_name) + continue - # gigabytesUsed should be a total of volumes and snapshots - gigabytes = sum([getattr(volume, 'size', 0) for volume - in volumes]) - gigabytes += sum([getattr(snap, 'size', 0) for snap - in snapshots]) + try: + service_helper = updater_class(region_id, self.project_id) + except Exception: + # NOTE(amelia): We will assume if there are issues connecting + # to a service that it will be due to the + # service not existing in this region. + notes.append("Couldn't access %s client, region %s" % + (service_name, region_id)) + continue - return {'gigabytes': gigabytes, - 'volumes': len(volumes), - 'snapshots': len(snapshots) - } - - def get_neutron_usage(self, region_id): - client = get_neutronclient(region_id) - - networks = client.list_networks( - tenant_id=self.project_id)['networks'] - routers = client.list_routers( - tenant_id=self.project_id)['routers'] - floatingips = client.list_floatingips( - tenant_id=self.project_id)['floatingips'] - ports = client.list_ports( - tenant_id=self.project_id)['ports'] - subnets = client.list_subnets( - tenant_id=self.project_id)['subnets'] - security_groups = client.list_security_groups( - tenant_id=self.project_id)['security_groups'] - security_group_rules = client.list_security_group_rules( - tenant_id=self.project_id)['security_group_rules'] - - return {'network': len(networks), - 'router': len(routers), - 'floatingip': len(floatingips), - 'port': len(ports), - 'subnet': len(subnets), - 'secuirty_group': len(security_groups), - 'security_group_rule': len(security_group_rules) - } - - def get_nova_usage(self, region_id): - client = get_novaclient(region_id) - nova_usage = client.limits.get( - tenant_id=self.project_id).to_dict()['absolute'] - nova_usage_keys = [ - ('instances', 'totalInstancesUsed'), - ('floating_ips', 'totalFloatingIpsUsed'), - ('ram', 'totalRAMUsed'), - ('cores', 'totalCoresUsed'), - ('secuirty_groups', 'totalSecurityGroupsUsed') - ] - - nova_usage_dict = {} - for key, usage_key in nova_usage_keys: - nova_usage_dict[key] = nova_usage[usage_key] - - return nova_usage_dict + service_helper.set_quota(values) + return notes diff --git a/adjutant/settings.py b/adjutant/settings.py index 66724fd..316b096 100644 --- a/adjutant/settings.py +++ b/adjutant/settings.py @@ -191,6 +191,10 @@ ACTIVE_TASKVIEWS = CONFIG.get( 'RoleList' ]) +# Default services for which to check and update quotas for +QUOTA_SERVICES = CONFIG.get('QUOTA_SERVICES', + ['cinder', 'neutron', 'nova']) + # Dict of TaskViews and their url_paths. # - This is populated by registering taskviews. TASKVIEW_CLASSES = {} diff --git a/adjutant/test_settings.py b/adjutant/test_settings.py index 805ff35..bf840c6 100644 --- a/adjutant/test_settings.py +++ b/adjutant/test_settings.py @@ -384,6 +384,8 @@ PROJECT_QUOTA_SIZES = { QUOTA_SIZES_ASC = ['small', 'medium', 'large'] +QUOTA_SERVICES = ['neutron', 'nova', 'cinder'] + SHOW_ACTION_ENDPOINTS = True TOKEN_CACHE_TIME = 60 @@ -408,4 +410,5 @@ conf_dict = { "SHOW_ACTION_ENDPOINTS": SHOW_ACTION_ENDPOINTS, "QUOTA_SIZES_ASC": QUOTA_SIZES_ASC, "TOKEN_CACHE_TIME": TOKEN_CACHE_TIME, + "QUOTA_SERVICES": QUOTA_SERVICES, } diff --git a/conf/conf.yaml b/conf/conf.yaml index 048edea..5e2f9d6 100644 --- a/conf/conf.yaml +++ b/conf/conf.yaml @@ -418,3 +418,9 @@ QUOTA_SIZES_ASC: - small - medium - large + +# Services to check through the quotas for +QUOTA_SERVICES: + - nova + - neutron + - cinder