From d528d6d42997de08bfc6d6c289a97ce802393b87 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Mon, 11 Apr 2016 20:32:16 +0530 Subject: [PATCH] Add remaining tempest testcase for Kingbird Added test case for quota sync which creates a VM in one region, calls quota sync, wait for quota sync to finish, calculated new limits manually and assert these limits with updated nova limits. Added test case to test quota exceeded scenario. Sets quota limit to some small value, call quota sync, try to create more than quota limit VMs. Then assert for Quota exceeded exception for the second VM. Delete all the created resources such as Project, User, VMs, Flavor, Subnet and Network in resouce clean. This should wind up our quota sync for nova resources. Change-Id: Ib5e1a5873219c702d27ac07de81477aafb45d923 --- tempest/README.rst | 10 +- tempest/tests/api/kingbird/base.py | 107 ++++++++- .../tests/api/kingbird/test_kingbird_api.py | 113 ++++++---- tempest/tests/common/kingbird.py | 206 +++++++++++++++--- 4 files changed, 353 insertions(+), 83 deletions(-) diff --git a/tempest/README.rst b/tempest/README.rst index 49f09ca..a72cd3a 100644 --- a/tempest/README.rst +++ b/tempest/README.rst @@ -20,6 +20,9 @@ I) Add kingbird configurations to config file:: cfg.StrOpt('endpoint_type', default='publicURL', help="Endpoint type of Kingbird service."), + cfg.IntOpt('TIME_TO_SYNC', + default=30), + help="Maximum time to wait for a sync call to complete."), cfg.StrOpt('endpoint_url', help="Endpoint URL of Kingbird service."), cfg.StrOpt('api_version', @@ -39,12 +42,15 @@ III) Add kingbird_group and KingbirdGroup to list of opts(_opts) It generates etc/tempest.conf.sample. Copy it to /etc/tempest/ and rename as tempest.conf -4. Copy tempest testcases for Kingbird:: +4. Make sure the default values represented by DEFAULT_QUOTAS in tempest/api/kingbird/base.py + has to be same as kingbird_global_limit section in kingbird.conf. + +5. Copy tempest testcases for Kingbird:: $ cp -r tempest/tests/api/kingbird /tempest/api/ $ cp tempest/tests/common/kingbird.py /tempest/common/ -5. Set kingbird = True under [service_available] section in tempest.conf:: +6. Set kingbird = True under [service_available] section in tempest.conf:: To list all Kingbird tempest cases, go to tempest directory, then run:: diff --git a/tempest/tests/api/kingbird/base.py b/tempest/tests/api/kingbird/base.py index f6ae24d..4472df1 100644 --- a/tempest/tests/api/kingbird/base.py +++ b/tempest/tests/api/kingbird/base.py @@ -12,14 +12,29 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - +import collections +import time from tempest.common import kingbird from tempest import config from tempest.lib.common import api_version_utils +from tempest.lib.common.utils import data_utils import tempest.test CONF = config.CONF +Global_instance_limit = 10 +DEFAULT_QUOTAS = { + u'quota_set': { + u'metadata_items': 128, u'subnet': 10, u'consistencygroups': 10, + u'floatingip': 50, u'gigabytes': 1000, u'backup_gigabytes': 1000, + u'ram': 51200, u'floating_ips': 10, u'snapshots': 10, + u'instances': 10, u'key_pairs': 100, u'volumes': 10, u'router': 10, + u'security_group': 10, u'cores': 20, u'backups': 10, u'fixed_ips': -1, + u'port': 50, u'security_groups': 10, u'network': 10 + } + } +# Time to wait for sync to finish +TIME_TO_SYNC = CONF.kingbird.TIME_TO_SYNC class BaseKingbirdTest(api_version_utils.BaseMicroversionTest, @@ -35,9 +50,11 @@ class BaseKingbirdTest(api_version_utils.BaseMicroversionTest, @classmethod def setup_credentials(cls): - cls.set_network_resources() super(BaseKingbirdTest, cls).setup_credentials() - cls.auth_token = kingbird.get_keystone_authtoken() + session = kingbird.get_session() + cls.auth_token = session.get_token() + cls.key_client = kingbird.get_key_client(session) + cls.regions = kingbird.get_regions(cls.key_client) @classmethod def setup_clients(cls): @@ -46,10 +63,30 @@ class BaseKingbirdTest(api_version_utils.BaseMicroversionTest, @classmethod def resource_setup(cls): super(BaseKingbirdTest, cls).resource_setup() + # Create Project, User, flavor, subnet & network for test + project_name = data_utils.rand_name(__name__ + '-project') + user_name = data_utils.rand_name(__name__ + '-user') + password = data_utils.rand_name(__name__ + '-password') + openstack_details = kingbird.get_openstack_drivers(cls.key_client, + cls.regions[0], + project_name, + user_name, + password) + cls.openstack_drivers = openstack_details['os_drivers'] + cls.resource_ids = kingbird.create_resources(cls.openstack_drivers) + cls.resource_ids.update(openstack_details) + cls.session = openstack_details['session'] @classmethod def resource_cleanup(cls): super(BaseKingbirdTest, cls).resource_cleanup() + default_quota = {'instances': DEFAULT_QUOTAS['quota_set']['instances'], + 'cores': DEFAULT_QUOTAS['quota_set']['cores'], + 'ram': DEFAULT_QUOTAS['quota_set']['ram']} + cls.set_default_quota(CONF.kingbird.project_id, default_quota) + kingbird.resource_cleanup(cls.openstack_drivers, cls.resource_ids) + kingbird.delete_custom_kingbird_quota( + cls.auth_token, CONF.kingbird.project_id, None) def setUp(self): super(BaseKingbirdTest, self).setUp() @@ -95,3 +132,67 @@ class BaseKingbirdTest(api_version_utils.BaseMicroversionTest, new_values = kingbird.create_custom_kingbird_quota_wrong_token( cls.auth_token, project_id, new_quota_values) return new_values + + @classmethod + def create_instance(cls, count=1): + try: + server_ids = kingbird.create_instance(cls.openstack_drivers, + cls.resource_ids, count) + except Exception as e: + server_ids = {'server_ids': list(e.args)} + raise + finally: + cls.resource_ids.update(server_ids) + + @classmethod + def delete_instance(cls): + kingbird.delete_instance(cls.openstack_drivers, cls.resource_ids) + cls.resource_ids['instances'] = None + + @classmethod + def calculate_quota_limits(cls, project_id): + calculated_quota_limits = collections.defaultdict(dict) + resource_usage = kingbird.get_usage_from_os_client( + cls.session, cls.regions, project_id) + total_usages = cls.get_summation(resource_usage) + for current_region in cls.regions: + global_remaining_limit = Global_instance_limit - \ + total_usages['instances'] + new_limit_for_region = global_remaining_limit + resource_usage[ + current_region]['instances'] + calculated_quota_limits.update( + {current_region: new_limit_for_region}) + return calculated_quota_limits + + @classmethod + def get_summation(cls, regions_dict): + # Adds resources usages from different regions + single_region = {} + resultant_dict = collections.Counter() + for current_region in regions_dict: + single_region[current_region] = collections.Counter( + regions_dict[current_region]) + resultant_dict += single_region[current_region] + return dict(resultant_dict) + + @classmethod + def get_usage_manually(cls, project_id): + resource_usage = kingbird.get_usage_from_os_client( + cls.session, cls.regions, project_id) + resource_usage = cls.get_summation(resource_usage) + return {'quota_set': resource_usage} + + @classmethod + def get_actual_limits(cls, project_id): + actual_limits = kingbird.get_actual_limits( + cls.session, cls.regions, project_id) + return actual_limits + + @classmethod + def wait_sometime_for_sync(cls): + time.sleep(TIME_TO_SYNC) + + @classmethod + def set_default_quota(cls, project_id, quota_to_set): + kingbird.set_default_quota( + cls.session, cls.regions, project_id, **quota_to_set) diff --git a/tempest/tests/api/kingbird/test_kingbird_api.py b/tempest/tests/api/kingbird/test_kingbird_api.py index eef8387..19fafe0 100644 --- a/tempest/tests/api/kingbird/test_kingbird_api.py +++ b/tempest/tests/api/kingbird/test_kingbird_api.py @@ -13,103 +13,130 @@ # License for the specific language governing permissions and limitations # under the License. -import uuid - from tempest.api.kingbird import base from tempest import config +import novaclient + CONF = config.CONF -FAKE_PROJECT = str(uuid.uuid4()) -DEFAULT_QUOTAS = { - u'quota_set': { - u'metadata_items': 128, u'subnet': 10, u'consistencygroups': 10, - u'floatingip': 50, u'gigabytes': 1000, u'backup_gigabytes': 1000, - u'ram': 51200, u'floating_ips': 10, u'snapshots': 10, - u'instances': 10, u'key_pairs': 100, u'volumes': 10, u'router': 10, - u'security_group': 10, u'cores': 20, u'backups': 10, u'fixed_ips': -1, - u'port': 50, u'security_groups': 10, u'network': 10 - } - } +DEFAULT_QUOTAS = base.DEFAULT_QUOTAS class KingbirdTestJSON(base.BaseKingbirdTest): @classmethod - def setup_clients(cls): - super(KingbirdTestJSON, cls).setup_clients() + def setup_clients(self): + super(KingbirdTestJSON, self).setup_clients() def tearDown(self): super(KingbirdTestJSON, self).tearDown() + @classmethod + def resource_setup(self): + super(KingbirdTestJSON, self).resource_setup() + self.PROJECT_ID = self.resource_ids["project_id"] + def test_kingbird_put_method(self): new_quota = {"quota_set": {"instances": 15, "cores": 10}} - actual_value = self.create_custom_kingbird_quota(FAKE_PROJECT, + actual_value = self.create_custom_kingbird_quota(self.PROJECT_ID, new_quota) - expected_value = {FAKE_PROJECT: new_quota["quota_set"]} + expected_value = {self.PROJECT_ID: new_quota["quota_set"]} self.assertEqual(expected_value, eval(actual_value)) def test_kingbird_get_method(self): new_quota = {"quota_set": {"instances": 15, "cores": 10}} - self.create_custom_kingbird_quota(FAKE_PROJECT, + self.create_custom_kingbird_quota(self.PROJECT_ID, new_quota) - actual_value = self.get_custom_kingbird_quota(FAKE_PROJECT) - new_quota["quota_set"].update({'project_id': FAKE_PROJECT}) + actual_value = self.get_custom_kingbird_quota(self.PROJECT_ID) + new_quota["quota_set"].update({'project_id': self.PROJECT_ID}) self.assertEqual(new_quota, eval(actual_value)) def test_kingbird_delete_method(self): new_quota = {"quota_set": {"instances": 15, "cores": 10}} quota_to_delete = {"quota_set": ["cores"]} - self.create_custom_kingbird_quota(FAKE_PROJECT, + self.create_custom_kingbird_quota(self.PROJECT_ID, new_quota) - self.delete_custom_kingbird_quota(FAKE_PROJECT, + self.delete_custom_kingbird_quota(self.PROJECT_ID, quota_to_delete) quota_after_delete = eval(self.get_custom_kingbird_quota( - FAKE_PROJECT)) + self.PROJECT_ID)) self.assertNotIn("cores", quota_after_delete["quota_set"]) def test_kingbird_delete_all_method(self): new_quota = {"quota_set": {"instances": 15, "cores": 10}} - self.create_custom_kingbird_quota(FAKE_PROJECT, + self.create_custom_kingbird_quota(self.PROJECT_ID, new_quota) - self.delete_custom_kingbird_quota(FAKE_PROJECT) + self.delete_custom_kingbird_quota(self.PROJECT_ID) actual_quota_after_delete = eval(self.get_custom_kingbird_quota( - FAKE_PROJECT)) + self.PROJECT_ID)) expected_quota_after_delete = {"quota_set": - {"project_id": FAKE_PROJECT}} + {"project_id": self.PROJECT_ID}} self.assertEqual(expected_quota_after_delete, actual_quota_after_delete) def test_kingbird_get_default_method_after_update(self): new_quota = {"quota_set": {"instances": 15, "cores": 10}} - self.create_custom_kingbird_quota(FAKE_PROJECT, + self.create_custom_kingbird_quota(self.PROJECT_ID, new_quota) actual_value = self.get_default_kingbird_quota() self.assertEqual(eval(actual_value), DEFAULT_QUOTAS) - self.delete_custom_kingbird_quota(FAKE_PROJECT) - - def test_quota_sync_for_project(self): - actual_value = self.quota_sync_for_project(FAKE_PROJECT) - expected_value = u'triggered quota sync for ' + FAKE_PROJECT - self.assertEqual(eval(actual_value), expected_value) + self.delete_custom_kingbird_quota(self.PROJECT_ID) def test_get_quota_usage_for_project(self): - actual_usage = self.get_quota_usage_for_project(FAKE_PROJECT) - expected_usage = {u'quota_set': {u'key_pairs': 1}} - # Assert nova usage, which will be common for all projects - self.assertEqual(eval(actual_usage), expected_usage) + self.create_instance(count=1) + actual_usage = self.get_quota_usage_for_project(self.PROJECT_ID) + expected_usage = self.get_usage_manually(self.PROJECT_ID) + self.assertEqual(eval(actual_usage)["quota_set"]["ram"], + expected_usage["quota_set"]["ram"]) + self.assertEqual(eval(actual_usage)["quota_set"]["cores"], + expected_usage["quota_set"]["cores"]) + self.assertEqual(eval(actual_usage)["quota_set"]["instances"], + expected_usage["quota_set"]["instances"]) + self.delete_instance() def test_kingbird_put_method_wrong_token(self): new_quota = {"quota_set": {"instances": 15, "cores": 10}} - response = self.create_custom_kingbird_quota_wrong_token(FAKE_PROJECT, - new_quota) + response = self.create_custom_kingbird_quota_wrong_token( + self.PROJECT_ID, new_quota) self.assertEqual(response.status_code, 401) self.assertEqual(response.text, u'Authentication required') def test_kingbird_get_default_method_after_delete(self): new_quota = {"quota_set": {"instances": 15, "cores": 10}} - self.create_custom_kingbird_quota(FAKE_PROJECT, + self.create_custom_kingbird_quota(self.PROJECT_ID, new_quota) - self.delete_custom_kingbird_quota(FAKE_PROJECT) + self.delete_custom_kingbird_quota(self.PROJECT_ID) actual_value = self.get_default_kingbird_quota() self.assertEqual(eval(actual_value), DEFAULT_QUOTAS) - self.delete_custom_kingbird_quota(FAKE_PROJECT) + self.delete_custom_kingbird_quota(self.PROJECT_ID) + + def test_quota_sync_for_project(self): + # Delete custom quota if there are any for this project + self.delete_custom_kingbird_quota(self.PROJECT_ID) + self.create_instance() + sync_status = self.quota_sync_for_project(self.PROJECT_ID) + expected_status = u"triggered quota sync for " + self.PROJECT_ID + calculated_limits = self.calculate_quota_limits(self.PROJECT_ID) + self.wait_sometime_for_sync() + actual_limits = self.get_actual_limits(self.PROJECT_ID) + self.assertEqual(calculated_limits, actual_limits) + self.assertEqual(eval(sync_status), expected_status) + self.delete_instance() + + def test_quota_exceed_after_sync(self): + new_quota = {"quota_set": {"instances": 2}} + self.create_custom_kingbird_quota(self.PROJECT_ID, + new_quota) + self.quota_sync_for_project(self.PROJECT_ID) + self.wait_sometime_for_sync() + try: + self.create_instance(count=3) + except Exception as exp: + self.assertIsInstance(exp, novaclient.exceptions.Forbidden) + message = exp.message + self.assertIn(u"Quota exceeded for instances", message) + self.delete_instance() + default_quota = {'instances': DEFAULT_QUOTAS['quota_set']['instances'], + 'cores': DEFAULT_QUOTAS['quota_set']['cores'], + 'ram': DEFAULT_QUOTAS['quota_set']['ram']} + self.set_default_quota(self.PROJECT_ID, default_quota) diff --git a/tempest/tests/common/kingbird.py b/tempest/tests/common/kingbird.py index 6962300..85fb53e 100644 --- a/tempest/tests/common/kingbird.py +++ b/tempest/tests/common/kingbird.py @@ -13,52 +13,91 @@ # See the License for the specific language governing permissions and # limitations under the License. +import collections import json import requests +import time +from keystoneclient.auth.identity import v3 +from keystoneclient import session +from keystoneclient.v3 import client as ks_client +from neutronclient.neutron import client as nt_client +from novaclient import client as nv_client from oslo_log import log as logging from tempest import config CONF = config.CONF +NOVA_API_VERSION = "2.1" +NEUTRON_API_VERSION = "2.0" +FLAVOR_NAME = "kb_test_flavor" +NETWORK_NAME = "kb_test_network" +SUBNET_NAME = "kb_test_subnet" +SERVER_NAME = "kb_test_server" +SUBNET_RANGE = "192.168.199.0/24" LOG = logging.getLogger(__name__) -def get_keystone_authtoken(): - headers = { - 'Content-type': 'application/json', - } - data = { - "auth": { - "identity": { - "methods": [ - "password" - ], - "password": { - "user": { - "domain": { - "name": CONF.auth.admin_domain_name - }, - "name": CONF.auth.admin_username, - "password": CONF.auth.admin_password - } - } - }, - "scope": { - "project": { - "domain": { - "name": CONF.auth.admin_domain_name - }, - "name": CONF.auth.admin_tenant_name - } - } - } - } - url_string = CONF.identity.uri_v3 + "/auth/tokens" - body = json.dumps(data) - response = requests.post(url_string, headers=headers, data=body) - token = response.headers['X-Subject-Token'] - return token +def get_session(): + return get_current_session( + CONF.identity.username, + CONF.identity.password, + CONF.identity.tenant_name + ) + + +def get_current_session(username, password, tenant_name): + auth = v3.Password( + auth_url=CONF.identity.uri_v3, + username=username, + password=password, + project_name=tenant_name, + user_domain_name=CONF.identity.domain_name, + project_domain_name=CONF.identity.default_domain_id) + sess = session.Session(auth=auth) + return sess + + +def get_openstack_drivers(key_client, region, project_name, user_name, + password): + # Create Project, User and asign role to new user + project = key_client.projects.create(project_name, + CONF.identity.domain_name) + user = key_client.users.create(user_name, CONF.identity.domain_name, + project.id, password) + admin_role = [current_role.id for current_role in + key_client.roles.list() if current_role.name == 'admin'][0] + + key_client.roles.grant(admin_role, user=user, project=project) + session = get_current_session(user_name, password, project_name) + nova_client = nv_client.Client(NOVA_API_VERSION, + session=session, + region_name=region) + neutron_client = nt_client.Client(NEUTRON_API_VERSION, session=session, + region_name=region) + return {"user_id": user.id, "project_id": project.id, "session": session, + "os_drivers": [key_client, nova_client, neutron_client]} + + +def get_key_client(session): + return ks_client.Client(session=session) + + +def create_instance(openstack_drivers, resource_ids, count=1): + nova_client = openstack_drivers[1] + server_ids = [] + image = nova_client.images.find(id=CONF.compute.image_ref) + flavor = nova_client.flavors.find(id=resource_ids['flavor_id']) + try: + for x in range(count): + server = nova_client.servers.create( + SERVER_NAME, image, flavor, + nics=[{'net-id': resource_ids['network_id']}]) + server_ids.append(server.id) + return {'server_ids': server_ids} + except Exception as e: + e.args = tuple(server_ids) + raise e def get_urlstring_and_headers(token): @@ -128,3 +167,100 @@ def create_custom_kingbird_quota_wrong_token(token, body = json.dumps(new_quota_values) response = requests.put(url_string, headers=headers, data=body) return response + + +def get_regions(key_client): + return [current_region.id for current_region in + key_client.regions.list()] + + +def delete_instance(openstack_drivers, resource_ids): + nova_client = openstack_drivers[1] + if 'server_ids' in resource_ids: + for server_id in resource_ids['server_ids']: + nova_client.servers.delete(server_id) + retries = 6 + # Delete may take time, So wait(with timeout) till the + # instance is deleted + while retries > 0: + LOG.debug("waiting for instance to get deleted") + time.sleep(1) + nova_list = [current_server.id for current_server in + nova_client.servers.list()] + if len(set(resource_ids['server_ids']) & set(nova_list)): + continue + else: + return + LOG.exception('Resource deleting failed, manually delete with IDs %s' + % resource_ids) + + +def resource_cleanup(openstack_drivers, resource_ids): + key_client = openstack_drivers[0] + nova_client = openstack_drivers[1] + neutron_client = openstack_drivers[2] + nova_client.flavors.delete(resource_ids['flavor_id']) + neutron_client.delete_subnet(resource_ids['subnet_id']) + neutron_client.delete_network(resource_ids['network_id']) + key_client.projects.delete(resource_ids['project_id']) + key_client.users.delete(resource_ids['user_id']) + + +def get_usage_from_os_client(session, regions, project_id): + resource_usage_all = collections.defaultdict(dict) + for current_region in regions: + resource_usage = collections.defaultdict(dict) + nova_client = nv_client.Client(NOVA_API_VERSION, + session=session, + region_name=current_region) + limits = nova_client.limits.get().to_dict() + resource_usage['ram'] = limits['absolute']['totalRAMUsed'] + resource_usage['cores'] = limits['absolute']['totalCoresUsed'] + resource_usage['instances'] = limits['absolute']['totalInstancesUsed'] + resource_usage['key_pairs'] = len(nova_client.keypairs.list()) + resource_usage_all[current_region] = resource_usage + return resource_usage_all + + +def get_actual_limits(session, regions, project_id): + resource_usage = collections.defaultdict(dict) + for current_region in regions: + nova_client = nv_client.Client(NOVA_API_VERSION, + session=session, + region_name=current_region) + updated_quota = nova_client.quotas.get(project_id) + resource_usage.update({current_region: updated_quota.instances}) + return resource_usage + + +def create_resources(openstack_drivers): + nova_client = openstack_drivers[1] + neutron_client = openstack_drivers[2] + flavor = nova_client.flavors.create( + FLAVOR_NAME, 128, 1, 1, flavorid='auto') + network_body = {'network': {'name': NETWORK_NAME, 'admin_state_up': True}} + network = neutron_client.create_network(body=network_body) + body_create_subnet = { + "subnets": [ + { + 'cidr': SUBNET_RANGE, + 'ip_version': 4, + 'network_id': network['network']['id'], + 'name': SUBNET_NAME + } + ] + } + subnet = neutron_client.create_subnet(body=body_create_subnet) + return { + 'subnet_id': subnet['subnets'][0]['id'], + 'network_id': network['network']['id'], + 'flavor_id': flavor.id + } + + +def set_default_quota(session, regions, project_id, **quota_to_set): + for current_region in regions: + nova_client = nv_client.Client(NOVA_API_VERSION, + session=session, + region_name=current_region) + nova_client.quotas.update(project_id, **quota_to_set)