From a2e655cbf32dc153e999de22933edcb4d3317a70 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 2 Mar 2018 15:11:34 +0000 Subject: [PATCH] Fix up amulet for queens * Fix up amulet tests to use keystone v3 clients. * Remove admin_* and auth_* for Queens l3_agent and metadata config files as they no longer appears to be used. * charm helper sync Change-Id: I880b7f710fa6b5ba480fb41d4095871fd9dabed7 --- hooks/charmhelpers/contrib/openstack/utils.py | 2 +- tests/basic_deployment.py | 88 +++++-- .../contrib/openstack/amulet/deployment.py | 10 +- .../contrib/openstack/amulet/utils.py | 214 ++++++++++++++++-- ...xenial-queens => gate-basic-xenial-queens} | 0 5 files changed, 277 insertions(+), 37 deletions(-) rename tests/{dev-basic-xenial-queens => gate-basic-xenial-queens} (100%) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index b753275..e719426 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -182,7 +182,7 @@ SWIFT_CODENAMES = OrderedDict([ ('pike', ['2.13.0', '2.15.0']), ('queens', - ['2.16.0']), + ['2.16.0', '2.17.0']), ]) # >= Liberty version->codename mapping diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 734ace3..3e7606e 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -27,6 +27,9 @@ from charmhelpers.contrib.openstack.amulet.utils import ( DEBUG, # ERROR ) +import keystoneclient +from keystoneclient.v2_0 import client as keystone_client +from keystoneclient.v3 import client as keystone_client_v3 # Use DEBUG to turn on debug logging u = OpenStackAmuletUtils(DEBUG) @@ -132,10 +135,22 @@ class HAClusterBasicDeployment(OpenStackAmuletDeployment): # Authenticate keystone admin u.log.debug('Authenticating keystone admin against VIP: ' '{}'.format(self._vip)) - self.keystone = u.authenticate_keystone_admin(self.keystone_sentry, - user='admin', - password='openstack', - tenant='admin') +# + api_version = 2 + client_class = keystone_client.Client + if self._get_openstack_release() >= self.xenial_queens: + api_version = 3 + client_class = keystone_client_v3.Client + self.keystone_session, auth = u.get_keystone_session( + self._vip, + api_version=api_version, + username='admin', + password='openstack', + project_name='admin', + user_domain_name='admin_domain', + project_domain_name='admin_domain') + self.keystone = client_class(session=self.keystone_session) + self.keystone.auth_ref = auth.get_access(self.keystone_session) # Create a demo tenant/role/user u.log.debug('Creating keystone demo tenant, role and user against ' @@ -143,25 +158,72 @@ class HAClusterBasicDeployment(OpenStackAmuletDeployment): self.demo_tenant = 'demoTenant' self.demo_role = 'demoRole' self.demo_user = 'demoUser' + self.demo_project = 'demoProject' + self.demo_domain = 'demoDomain' + if self._get_openstack_release() >= self.xenial_queens: + self.create_users_v3() + self.demo_user_session, auth = u.get_keystone_session( + self._vip, + self.demo_user, + 'password', + api_version=3, + user_domain_name=self.demo_domain, + project_domain_name=self.demo_domain, + project_name=self.demo_project + ) + self.keystone_demo = keystone_client_v3.Client( + session=self.demo_user_session) + else: + self.create_users_v2() + # Authenticate demo user with keystone (authenticate_keystone_user + # looks up identity-service in service catalogue so uses vip) + self.keystone_demo = \ + u.authenticate_keystone_user( + self.keystone, user=self.demo_user, + password='password', + tenant=self.demo_tenant) + + def create_users_v3(self): + try: + self.keystone.projects.find(name=self.demo_project) + except keystoneclient.exceptions.NotFound: + domain = self.keystone.domains.create( + self.demo_domain, + description='Demo Domain', + enabled=True + ) + project = self.keystone.projects.create( + self.demo_project, + domain, + description='Demo Project', + enabled=True, + ) + user = self.keystone.users.create( + self.demo_user, + domain=domain.id, + project=self.demo_project, + password='password', + email='demov3@demo.com', + description='Demo', + enabled=True) + role = self.keystone.roles.find(name='Admin') + self.keystone.roles.grant( + role.id, + user=user.id, + project=project.id) + + def create_users_v2(self): if not u.tenant_exists(self.keystone, self.demo_tenant): tenant = self.keystone.tenants.create(tenant_name=self.demo_tenant, description='demo tenant', enabled=True) + self.keystone.roles.create(name=self.demo_role) self.keystone.users.create(name=self.demo_user, password='password', tenant_id=tenant.id, email='demo@demo.com') - # Authenticate keystone demo - u.log.debug('Authenticating keystone demo user against VIP: ' - '{}'.format(self._vip)) - self.keystone_demo = u.authenticate_keystone_user( - self.keystone, - user=self.demo_user, - password='password', - tenant=self.demo_tenant) - def _toggle_maintenance_and_wait(self, expected): SLEEP = 10 TIMEOUT = 900 # secs diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 5afbbd8..66beeda 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -21,6 +21,9 @@ from collections import OrderedDict from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment ) +from charmhelpers.contrib.openstack.amulet.utils import ( + OPENSTACK_RELEASES_PAIRS +) DEBUG = logging.DEBUG ERROR = logging.ERROR @@ -271,11 +274,8 @@ class OpenStackAmuletDeployment(AmuletDeployment): release. """ # Must be ordered by OpenStack release (not by Ubuntu release): - (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty, - self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton, - self.yakkety_newton, self.xenial_ocata, self.zesty_ocata, - self.xenial_pike, self.artful_pike, self.xenial_queens, - self.bionic_queens,) = range(13) + for i, os_pair in enumerate(OPENSTACK_RELEASES_PAIRS): + setattr(self, os_pair, i) releases = { ('trusty', None): self.trusty_icehouse, diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index d93cff3..5fdcead 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -50,6 +50,13 @@ ERROR = logging.ERROR NOVA_CLIENT_VERSION = "2" +OPENSTACK_RELEASES_PAIRS = [ + 'trusty_icehouse', 'trusty_kilo', 'trusty_liberty', + 'trusty_mitaka', 'xenial_mitaka', 'xenial_newton', + 'yakkety_newton', 'xenial_ocata', 'zesty_ocata', + 'xenial_pike', 'artful_pike', 'xenial_queens', + 'bionic_queens'] + class OpenStackAmuletUtils(AmuletUtils): """OpenStack amulet utilities. @@ -63,7 +70,34 @@ class OpenStackAmuletUtils(AmuletUtils): super(OpenStackAmuletUtils, self).__init__(log_level) def validate_endpoint_data(self, endpoints, admin_port, internal_port, - public_port, expected): + public_port, expected, openstack_release=None): + """Validate endpoint data. Pick the correct validator based on + OpenStack release. Expected data should be in the v2 format: + { + 'id': id, + 'region': region, + 'adminurl': adminurl, + 'internalurl': internalurl, + 'publicurl': publicurl, + 'service_id': service_id} + + """ + validation_function = self.validate_v2_endpoint_data + xenial_queens = OPENSTACK_RELEASES_PAIRS.index('xenial_queens') + if openstack_release and openstack_release >= xenial_queens: + validation_function = self.validate_v3_endpoint_data + expected = { + 'id': expected['id'], + 'region': expected['region'], + 'region_id': 'RegionOne', + 'url': self.valid_url, + 'interface': self.not_null, + 'service_id': expected['service_id']} + return validation_function(endpoints, admin_port, internal_port, + public_port, expected) + + def validate_v2_endpoint_data(self, endpoints, admin_port, internal_port, + public_port, expected): """Validate endpoint data. Validate actual endpoint data vs expected endpoint data. The ports @@ -141,7 +175,86 @@ class OpenStackAmuletUtils(AmuletUtils): if len(found) != expected_num_eps: return 'Unexpected number of endpoints found' - def validate_svc_catalog_endpoint_data(self, expected, actual): + def convert_svc_catalog_endpoint_data_to_v3(self, ep_data): + """Convert v2 endpoint data into v3. + + { + 'service_name1': [ + { + 'adminURL': adminURL, + 'id': id, + 'region': region. + 'publicURL': publicURL, + 'internalURL': internalURL + }], + 'service_name2': [ + { + 'adminURL': adminURL, + 'id': id, + 'region': region. + 'publicURL': publicURL, + 'internalURL': internalURL + }], + } + """ + self.log.warn("Endpoint ID and Region ID validation is limited to not " + "null checks after v2 to v3 conversion") + for svc in ep_data.keys(): + assert len(ep_data[svc]) == 1, "Unknown data format" + svc_ep_data = ep_data[svc][0] + ep_data[svc] = [ + { + 'url': svc_ep_data['adminURL'], + 'interface': 'admin', + 'region': svc_ep_data['region'], + 'region_id': self.not_null, + 'id': self.not_null}, + { + 'url': svc_ep_data['publicURL'], + 'interface': 'public', + 'region': svc_ep_data['region'], + 'region_id': self.not_null, + 'id': self.not_null}, + { + 'url': svc_ep_data['internalURL'], + 'interface': 'internal', + 'region': svc_ep_data['region'], + 'region_id': self.not_null, + 'id': self.not_null}] + return ep_data + + def validate_svc_catalog_endpoint_data(self, expected, actual, + openstack_release=None): + """Validate service catalog endpoint data. Pick the correct validator + for the OpenStack version. Expected data should be in the v2 format: + { + 'service_name1': [ + { + 'adminURL': adminURL, + 'id': id, + 'region': region. + 'publicURL': publicURL, + 'internalURL': internalURL + }], + 'service_name2': [ + { + 'adminURL': adminURL, + 'id': id, + 'region': region. + 'publicURL': publicURL, + 'internalURL': internalURL + }], + } + + """ + validation_function = self.validate_v2_svc_catalog_endpoint_data + xenial_queens = OPENSTACK_RELEASES_PAIRS.index('xenial_queens') + if openstack_release and openstack_release >= xenial_queens: + validation_function = self.validate_v3_svc_catalog_endpoint_data + expected = self.convert_svc_catalog_endpoint_data_to_v3(expected) + return validation_function(expected, actual) + + def validate_v2_svc_catalog_endpoint_data(self, expected, actual): """Validate service catalog endpoint data. Validate a list of actual service catalog endpoints vs a list of @@ -367,13 +480,36 @@ class OpenStackAmuletUtils(AmuletUtils): project_domain_name=None, project_name=None): """Authenticate with Keystone""" self.log.debug('Authenticating with keystone...') - port = 5000 - if admin_port: - port = 35357 - base_ep = "http://{}:{}".format(keystone_ip.strip().decode('utf-8'), - port) - if not api_version or api_version == 2: - ep = base_ep + "/v2.0" + if not api_version: + api_version = 2 + sess, auth = self.get_keystone_session( + keystone_ip=keystone_ip, + username=username, + password=password, + api_version=api_version, + admin_port=admin_port, + user_domain_name=user_domain_name, + domain_name=domain_name, + project_domain_name=project_domain_name, + project_name=project_name + ) + if api_version == 2: + client = keystone_client.Client(session=sess) + else: + client = keystone_client_v3.Client(session=sess) + # This populates the client.service_catalog + client.auth_ref = auth.get_access(sess) + return client + + def get_keystone_session(self, keystone_ip, username, password, + api_version=False, admin_port=False, + user_domain_name=None, domain_name=None, + project_domain_name=None, project_name=None): + """Return a keystone session object""" + ep = self.get_keystone_endpoint(keystone_ip, + api_version=api_version, + admin_port=admin_port) + if api_version == 2: auth = v2.Password( username=username, password=password, @@ -381,12 +517,7 @@ class OpenStackAmuletUtils(AmuletUtils): auth_url=ep ) sess = keystone_session.Session(auth=auth) - client = keystone_client.Client(session=sess) - # This populates the client.service_catalog - client.auth_ref = auth.get_access(sess) - return client else: - ep = base_ep + "/v3" auth = v3.Password( user_domain_name=user_domain_name, username=username, @@ -397,10 +528,57 @@ class OpenStackAmuletUtils(AmuletUtils): auth_url=ep ) sess = keystone_session.Session(auth=auth) - client = keystone_client_v3.Client(session=sess) - # This populates the client.service_catalog - client.auth_ref = auth.get_access(sess) - return client + return (sess, auth) + + def get_keystone_endpoint(self, keystone_ip, api_version=None, + admin_port=False): + """Return keystone endpoint""" + port = 5000 + if admin_port: + port = 35357 + base_ep = "http://{}:{}".format(keystone_ip.strip().decode('utf-8'), + port) + if api_version == 2: + ep = base_ep + "/v2.0" + else: + ep = base_ep + "/v3" + return ep + + def get_default_keystone_session(self, keystone_sentry, + openstack_release=None): + """Return a keystone session object and client object assuming standard + default settings + + Example call in amulet tests: + self.keystone_session, self.keystone = u.get_default_keystone_session( + self.keystone_sentry, + openstack_release=self._get_openstack_release()) + + The session can then be used to auth other clients: + neutronclient.Client(session=session) + aodh_client.Client(session=session) + eyc + """ + self.log.debug('Authenticating keystone admin...') + api_version = 2 + client_class = keystone_client.Client + # 11 => xenial_queens + if openstack_release and openstack_release >= 11: + api_version = 3 + client_class = keystone_client_v3.Client + keystone_ip = keystone_sentry.info['public-address'] + session, auth = self.get_keystone_session( + keystone_ip, + api_version=api_version, + username='admin', + password='openstack', + project_name='admin', + user_domain_name='admin_domain', + project_domain_name='admin_domain') + client = client_class(session=session) + # This populates the client.service_catalog + client.auth_ref = auth.get_access(session) + return session, client def authenticate_keystone_admin(self, keystone_sentry, user, password, tenant=None, api_version=None, diff --git a/tests/dev-basic-xenial-queens b/tests/gate-basic-xenial-queens similarity index 100% rename from tests/dev-basic-xenial-queens rename to tests/gate-basic-xenial-queens