From ec86562991b47b4530ed82e79efa649d4835ff50 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 8 May 2018 11:52:12 -0700 Subject: [PATCH] Enable Bionic as a gate test Change bionic test from dev to gate for 18.05. Change-Id: Iaff111e3ac3802481448ac95b936cf043a441fc0 --- hooks/charmhelpers/contrib/openstack/utils.py | 2 +- .../contrib/openstack/vaultlocker.py | 76 +++++++++++---- .../contrib/storage/linux/utils.py | 16 ++++ hooks/charmhelpers/core/hookenv.py | 8 +- hooks/charmhelpers/core/sysctl.py | 18 ++-- tests/basic_deployment.py | 94 +++++++++++++++---- tests/charmhelpers/core/hookenv.py | 8 +- tests/charmhelpers/core/sysctl.py | 18 ++-- ...bionic-queens => gate-basic-bionic-queens} | 0 tests/gate-basic-xenial-queens | 25 +++++ tox.ini | 2 +- 11 files changed, 210 insertions(+), 57 deletions(-) rename tests/{dev-basic-bionic-queens => gate-basic-bionic-queens} (100%) create mode 100755 tests/gate-basic-xenial-queens diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index e7194264..6184abd0 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -306,7 +306,7 @@ def get_os_codename_install_source(src): if src.startswith('cloud:'): ca_rel = src.split(':')[1] - ca_rel = ca_rel.split('%s-' % ubuntu_rel)[1].split('/')[0] + ca_rel = ca_rel.split('-')[1].split('/')[0] return ca_rel # Best guess match based on deb string provided diff --git a/hooks/charmhelpers/contrib/openstack/vaultlocker.py b/hooks/charmhelpers/contrib/openstack/vaultlocker.py index 0b78e7a4..a8e4bf88 100644 --- a/hooks/charmhelpers/contrib/openstack/vaultlocker.py +++ b/hooks/charmhelpers/contrib/openstack/vaultlocker.py @@ -21,6 +21,9 @@ import charmhelpers.contrib.openstack.context as context import charmhelpers.core.hookenv as hookenv import charmhelpers.core.host as host import charmhelpers.core.templating as templating +import charmhelpers.core.unitdata as unitdata + +VAULTLOCKER_BACKEND = 'charm-vaultlocker' class VaultKVContext(context.OSContextGenerator): @@ -34,30 +37,39 @@ class VaultKVContext(context.OSContextGenerator): ) def __call__(self): + db = unitdata.kv() + last_token = db.get('last-token') + secret_id = db.get('secret-id') for relation_id in hookenv.relation_ids(self.interfaces[0]): for unit in hookenv.related_units(relation_id): - vault_url = hookenv.relation_get( - 'vault_url', - unit=unit, - rid=relation_id - ) - role_id = hookenv.relation_get( - '{}_role_id'.format(hookenv.local_unit()), - unit=unit, - rid=relation_id - ) + data = hookenv.relation_get(unit=unit, + rid=relation_id) + vault_url = data.get('vault_url') + role_id = data.get('{}_role_id'.format(hookenv.local_unit())) + token = data.get('{}_token'.format(hookenv.local_unit())) + + if all([vault_url, role_id, token]): + token = json.loads(token) + vault_url = json.loads(vault_url) + + # Tokens may change when secret_id's are being + # reissued - if so use token to get new secret_id + if token != last_token: + secret_id = retrieve_secret_id( + url=vault_url, + token=token + ) + db.set('secret-id', secret_id) + db.set('last-token', token) + db.flush() - if vault_url and role_id: ctxt = { - 'vault_url': json.loads(vault_url), + 'vault_url': vault_url, 'role_id': json.loads(role_id), + 'secret_id': secret_id, 'secret_backend': self.secret_backend, } - vault_ca = hookenv.relation_get( - 'vault_ca', - unit=unit, - rid=relation_id - ) + vault_ca = data.get('vault_ca') if vault_ca: ctxt['vault_ca'] = json.loads(vault_ca) self.complete = True @@ -82,3 +94,33 @@ def write_vaultlocker_conf(context, priority=100): alternatives.install_alternative('vaultlocker.conf', '/etc/vaultlocker/vaultlocker.conf', charm_vl_path, priority) + + +def vault_relation_complete(backend=None): + """Determine whether vault relation is complete + + :param backend: Name of secrets backend requested + :ptype backend: string + :returns: whether the relation to vault is complete + :rtype: bool""" + vault_kv = VaultKVContext(secret_backend=backend or VAULTLOCKER_BACKEND) + vault_kv() + return vault_kv.complete + + +# TODO: contrib a high level unwrap method to hvac that works +def retrieve_secret_id(url, token): + """Retrieve a response-wrapped secret_id from Vault + + :param url: URL to Vault Server + :ptype url: str + :param token: One shot Token to use + :ptype token: str + :returns: secret_id to use for Vault Access + :rtype: str""" + import hvac + client = hvac.Client(url=url, token=token) + response = client._post('/v1/sys/wrapping/unwrap') + if response.status_code == 200: + data = response.json() + return data['data']['secret_id'] diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index c9428894..6f846b05 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -67,3 +67,19 @@ def is_device_mounted(device): except Exception: return False return bool(re.search(r'MOUNTPOINT=".+"', out)) + + +def mkfs_xfs(device, force=False): + """Format device with XFS filesystem. + + By default this should fail if the device already has a filesystem on it. + :param device: Full path to device to format + :ptype device: tr + :param force: Force operation + :ptype: force: boolean""" + cmd = ['mkfs.xfs'] + if force: + cmd.append("-f") + + cmd += ['-i', 'size=1024', device] + check_call(cmd) diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index e9df1509..627d8f79 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -290,7 +290,7 @@ class Config(dict): self.implicit_save = True self._prev_dict = None self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME) - if os.path.exists(self.path): + if os.path.exists(self.path) and os.stat(self.path).st_size: self.load_previous() atexit(self._implicit_save) @@ -310,7 +310,11 @@ class Config(dict): """ self.path = path or self.path with open(self.path) as f: - self._prev_dict = json.load(f) + try: + self._prev_dict = json.load(f) + except ValueError as e: + log('Unable to parse previous config data - {}'.format(str(e)), + level=ERROR) for k, v in copy.deepcopy(self._prev_dict).items(): if k not in self: self[k] = v diff --git a/hooks/charmhelpers/core/sysctl.py b/hooks/charmhelpers/core/sysctl.py index 6e413e31..1f188d8c 100644 --- a/hooks/charmhelpers/core/sysctl.py +++ b/hooks/charmhelpers/core/sysctl.py @@ -31,18 +31,22 @@ __author__ = 'Jorge Niedbalski R. ' def create(sysctl_dict, sysctl_file): """Creates a sysctl.conf file from a YAML associative array - :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }" + :param sysctl_dict: a dict or YAML-formatted string of sysctl + options eg "{ 'kernel.max_pid': 1337 }" :type sysctl_dict: str :param sysctl_file: path to the sysctl file to be saved :type sysctl_file: str or unicode :returns: None """ - try: - sysctl_dict_parsed = yaml.safe_load(sysctl_dict) - except yaml.YAMLError: - log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), - level=ERROR) - return + if type(sysctl_dict) is not dict: + try: + sysctl_dict_parsed = yaml.safe_load(sysctl_dict) + except yaml.YAMLError: + log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), + level=ERROR) + return + else: + sysctl_dict_parsed = sysctl_dict with open(sysctl_file, "w") as fd: for key, value in sysctl_dict_parsed.items(): diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 5929f8c3..3d7d4d67 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -19,6 +19,10 @@ import re import time import json +import keystoneclient +from keystoneclient.v3 import client as keystone_client_v3 +from novaclient import client as nova_client + from charmhelpers.contrib.openstack.amulet.deployment import ( OpenStackAmuletDeployment ) @@ -150,47 +154,97 @@ class CephBasicDeployment(OpenStackAmuletDeployment): self._get_openstack_release_string())) # Authenticate admin with keystone - self.keystone = u.authenticate_keystone_admin(self.keystone_sentry, - user='admin', - password='openstack', - tenant='admin') + self.keystone_session, self.keystone = u.get_default_keystone_session( + self.keystone_sentry, + openstack_release=self._get_openstack_release()) + # Authenticate admin with cinder endpoint self.cinder = u.authenticate_cinder_admin(self.keystone) # Authenticate admin with glance endpoint self.glance = u.authenticate_glance_admin(self.keystone) # Authenticate admin with nova endpoint - self.nova = u.authenticate_nova_user(self.keystone, - user='admin', - password='openstack', - tenant='admin') + self.nova = nova_client.Client(2, session=self.keystone_session) + + keystone_ip = self.keystone_sentry.info['public-address'] # Create a demo tenant/role/user self.demo_tenant = 'demoTenant' self.demo_role = 'demoRole' self.demo_user = 'demoUser' + self.demo_project = 'demoProject' + self.demo_domain = 'demoDomain' + if self._get_openstack_release() >= self.xenial_queens: + self.create_users_v3() + self.demo_user_session, auth = u.get_keystone_session( + keystone_ip, + self.demo_user, + 'password', + api_version=3, + user_domain_name=self.demo_domain, + project_domain_name=self.demo_domain, + project_name=self.demo_project + ) + self.keystone_demo = keystone_client_v3.Client( + session=self.demo_user_session) + self.nova_demo = nova_client.Client( + 2, + session=self.demo_user_session) + else: + self.create_users_v2() + # Authenticate demo user with keystone + self.keystone_demo = \ + u.authenticate_keystone_user( + self.keystone, user=self.demo_user, + password='password', + tenant=self.demo_tenant) + # Authenticate demo user with nova-api + self.nova_demo = u.authenticate_nova_user(self.keystone, + user=self.demo_user, + password='password', + tenant=self.demo_tenant) + + def create_users_v3(self): + try: + self.keystone.projects.find(name=self.demo_project) + except keystoneclient.exceptions.NotFound: + domain = self.keystone.domains.create( + self.demo_domain, + description='Demo Domain', + enabled=True + ) + project = self.keystone.projects.create( + self.demo_project, + domain, + description='Demo Project', + enabled=True, + ) + user = self.keystone.users.create( + self.demo_user, + domain=domain.id, + project=self.demo_project, + password='password', + email='demov3@demo.com', + description='Demo', + enabled=True) + role = self.keystone.roles.find(name='Admin') + self.keystone.roles.grant( + role.id, + user=user.id, + project=project.id) + + def create_users_v2(self): if not u.tenant_exists(self.keystone, self.demo_tenant): tenant = self.keystone.tenants.create(tenant_name=self.demo_tenant, description='demo tenant', enabled=True) + self.keystone.roles.create(name=self.demo_role) self.keystone.users.create(name=self.demo_user, password='password', tenant_id=tenant.id, email='demo@demo.com') - # Authenticate demo user with keystone - self.keystone_demo = u.authenticate_keystone_user(self.keystone, - self.demo_user, - 'password', - self.demo_tenant) - - # Authenticate demo user with nova-api - self.nova_demo = u.authenticate_nova_user(self.keystone, - self.demo_user, - 'password', - self.demo_tenant) - def test_100_ceph_processes(self): """Verify that the expected service processes are running on each ceph unit.""" diff --git a/tests/charmhelpers/core/hookenv.py b/tests/charmhelpers/core/hookenv.py index e9df1509..627d8f79 100644 --- a/tests/charmhelpers/core/hookenv.py +++ b/tests/charmhelpers/core/hookenv.py @@ -290,7 +290,7 @@ class Config(dict): self.implicit_save = True self._prev_dict = None self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME) - if os.path.exists(self.path): + if os.path.exists(self.path) and os.stat(self.path).st_size: self.load_previous() atexit(self._implicit_save) @@ -310,7 +310,11 @@ class Config(dict): """ self.path = path or self.path with open(self.path) as f: - self._prev_dict = json.load(f) + try: + self._prev_dict = json.load(f) + except ValueError as e: + log('Unable to parse previous config data - {}'.format(str(e)), + level=ERROR) for k, v in copy.deepcopy(self._prev_dict).items(): if k not in self: self[k] = v diff --git a/tests/charmhelpers/core/sysctl.py b/tests/charmhelpers/core/sysctl.py index 6e413e31..1f188d8c 100644 --- a/tests/charmhelpers/core/sysctl.py +++ b/tests/charmhelpers/core/sysctl.py @@ -31,18 +31,22 @@ __author__ = 'Jorge Niedbalski R. ' def create(sysctl_dict, sysctl_file): """Creates a sysctl.conf file from a YAML associative array - :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }" + :param sysctl_dict: a dict or YAML-formatted string of sysctl + options eg "{ 'kernel.max_pid': 1337 }" :type sysctl_dict: str :param sysctl_file: path to the sysctl file to be saved :type sysctl_file: str or unicode :returns: None """ - try: - sysctl_dict_parsed = yaml.safe_load(sysctl_dict) - except yaml.YAMLError: - log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), - level=ERROR) - return + if type(sysctl_dict) is not dict: + try: + sysctl_dict_parsed = yaml.safe_load(sysctl_dict) + except yaml.YAMLError: + log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), + level=ERROR) + return + else: + sysctl_dict_parsed = sysctl_dict with open(sysctl_file, "w") as fd: for key, value in sysctl_dict_parsed.items(): diff --git a/tests/dev-basic-bionic-queens b/tests/gate-basic-bionic-queens similarity index 100% rename from tests/dev-basic-bionic-queens rename to tests/gate-basic-bionic-queens diff --git a/tests/gate-basic-xenial-queens b/tests/gate-basic-xenial-queens new file mode 100755 index 00000000..5fa16a57 --- /dev/null +++ b/tests/gate-basic-xenial-queens @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Amulet tests on a basic ceph deployment on xenial-queens.""" + +from basic_deployment import CephBasicDeployment + +if __name__ == '__main__': + deployment = CephBasicDeployment(series='xenial', + openstack='cloud:xenial-queens', + source='cloud:xenial-updates/queens') + deployment.run_tests() diff --git a/tox.ini b/tox.ini index 2bbbc97d..9c21d57e 100644 --- a/tox.ini +++ b/tox.ini @@ -65,7 +65,7 @@ basepython = python2.7 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = - bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-pike --no-destroy + bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-queens --no-destroy [testenv:func27-dfs] # Charm Functional Test