From 7ecfa30b00823ac478486977288fcd6443b4505f Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 15 Nov 2017 16:09:49 -0800 Subject: [PATCH] Enable xenial-pike amulet test Make default func27-smoke xenial-pike Charm-helpers sync Change-Id: I289d38e4170d204fbf9b0281b28be28c9e847e65 --- charmhelpers/contrib/charmsupport/nrpe.py | 10 ++- charmhelpers/contrib/network/ip.py | 4 +- .../contrib/openstack/amulet/deployment.py | 32 +++++--- .../contrib/openstack/amulet/utils.py | 36 +++++++-- charmhelpers/contrib/openstack/context.py | 35 ++++---- .../contrib/openstack/templates/ceph.conf | 4 +- .../contrib/openstack/templates/haproxy.cfg | 2 + charmhelpers/contrib/openstack/templating.py | 2 + charmhelpers/contrib/openstack/utils.py | 16 ++-- charmhelpers/contrib/python/debug.py | 2 +- charmhelpers/contrib/storage/linux/ceph.py | 42 +++++++--- charmhelpers/contrib/storage/linux/lvm.py | 8 +- charmhelpers/contrib/storage/linux/utils.py | 2 +- charmhelpers/contrib/unison/__init__.py | 2 +- charmhelpers/core/hookenv.py | 80 +++++++++++++++++-- charmhelpers/core/strutils.py | 16 ++-- charmhelpers/core/unitdata.py | 2 +- charmhelpers/fetch/ubuntu.py | 2 +- tests/basic_deployment.py | 28 ++++++- .../contrib/openstack/amulet/deployment.py | 32 +++++--- .../contrib/openstack/amulet/utils.py | 36 +++++++-- tests/charmhelpers/core/hookenv.py | 80 +++++++++++++++++-- tests/charmhelpers/core/strutils.py | 16 ++-- tests/charmhelpers/core/unitdata.py | 2 +- tests/gate-basic-xenial-pike | 0 tox.ini | 2 +- unit_tests/test_keystone_contexts.py | 1 + 27 files changed, 379 insertions(+), 115 deletions(-) mode change 100644 => 100755 tests/gate-basic-xenial-pike diff --git a/charmhelpers/contrib/charmsupport/nrpe.py b/charmhelpers/contrib/charmsupport/nrpe.py index 80d574dc..1c55b30f 100644 --- a/charmhelpers/contrib/charmsupport/nrpe.py +++ b/charmhelpers/contrib/charmsupport/nrpe.py @@ -30,6 +30,7 @@ import yaml from charmhelpers.core.hookenv import ( config, + hook_name, local_unit, log, relation_ids, @@ -285,7 +286,7 @@ class NRPE(object): try: nagios_uid = pwd.getpwnam('nagios').pw_uid nagios_gid = grp.getgrnam('nagios').gr_gid - except: + except Exception: log("Nagios user not set up, nrpe checks not updated") return @@ -302,7 +303,12 @@ class NRPE(object): "command": nrpecheck.command, } - service('restart', 'nagios-nrpe-server') + # update-status hooks are configured to firing every 5 minutes by + # default. When nagios-nrpe-server is restarted, the nagios server + # reports checks failing causing unneccessary alerts. Let's not restart + # on update-status hooks. + if not hook_name() == 'update-status': + service('restart', 'nagios-nrpe-server') monitor_ids = relation_ids("local-monitors") + \ relation_ids("nrpe-external-master") diff --git a/charmhelpers/contrib/network/ip.py b/charmhelpers/contrib/network/ip.py index d7e6debf..a871ce37 100644 --- a/charmhelpers/contrib/network/ip.py +++ b/charmhelpers/contrib/network/ip.py @@ -490,7 +490,7 @@ def get_host_ip(hostname, fallback=None): if not ip_addr: try: ip_addr = socket.gethostbyname(hostname) - except: + except Exception: log("Failed to resolve hostname '%s'" % (hostname), level=WARNING) return fallback @@ -518,7 +518,7 @@ def get_hostname(address, fqdn=True): if not result: try: result = socket.gethostbyaddr(address)[0] - except: + except Exception: return None else: result = address diff --git a/charmhelpers/contrib/openstack/amulet/deployment.py b/charmhelpers/contrib/openstack/amulet/deployment.py index 5c041d2c..5e33eb71 100644 --- a/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/charmhelpers/contrib/openstack/amulet/deployment.py @@ -250,7 +250,14 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.log.debug('Waiting up to {}s for extended status on services: ' '{}'.format(timeout, services)) service_messages = {service: message for service in services} + + # Check for idleness + self.d.sentry.wait() + # Check for error states and bail early + self.d.sentry.wait_for_status(self.d.juju_env, services) + # Check for ready messages self.d.sentry.wait_for_messages(service_messages, timeout=timeout) + self.log.info('OK') def _get_openstack_release(self): @@ -303,20 +310,27 @@ class OpenStackAmuletDeployment(AmuletDeployment): test scenario, based on OpenStack release and whether ceph radosgw is flagged as present or not.""" - if self._get_openstack_release() >= self.trusty_kilo: - # Kilo or later - pools = [ - 'rbd', - 'cinder', - 'glance' - ] - else: + if self._get_openstack_release() <= self.trusty_juno: # Juno or earlier pools = [ 'data', 'metadata', 'rbd', - 'cinder', + 'cinder-ceph', + 'glance' + ] + elif (self.trust_kilo <= self._get_openstack_release() <= + self.zesty_ocata): + # Kilo through Ocata + pools = [ + 'rbd', + 'cinder-ceph', + 'glance' + ] + else: + # Pike and later + pools = [ + 'cinder-ceph', 'glance' ] diff --git a/charmhelpers/contrib/openstack/amulet/utils.py b/charmhelpers/contrib/openstack/amulet/utils.py index c8edbf65..b71b2b19 100644 --- a/charmhelpers/contrib/openstack/amulet/utils.py +++ b/charmhelpers/contrib/openstack/amulet/utils.py @@ -23,6 +23,7 @@ import urllib import urlparse import cinderclient.v1.client as cinder_client +import cinderclient.v2.client as cinder_clientv2 import glanceclient.v1.client as glance_client import heatclient.v1.client as heat_client from keystoneclient.v2_0 import client as keystone_client @@ -42,7 +43,6 @@ import swiftclient from charmhelpers.contrib.amulet.utils import ( AmuletUtils ) -from charmhelpers.core.decorators import retry_on_exception from charmhelpers.core.host import CompareHostReleases DEBUG = logging.DEBUG @@ -310,7 +310,6 @@ class OpenStackAmuletUtils(AmuletUtils): self.log.debug('Checking if tenant exists ({})...'.format(tenant)) return tenant in [t.name for t in keystone.tenants.list()] - @retry_on_exception(5, base_delay=10) def keystone_wait_for_propagation(self, sentry_relation_pairs, api_version): """Iterate over list of sentry and relation tuples and verify that @@ -326,7 +325,7 @@ class OpenStackAmuletUtils(AmuletUtils): rel = sentry.relation('identity-service', relation_name) self.log.debug('keystone relation data: {}'.format(rel)) - if rel['api_version'] != str(api_version): + if rel.get('api_version') != str(api_version): raise Exception("api_version not propagated through relation" " data yet ('{}' != '{}')." "".format(rel['api_version'], api_version)) @@ -348,15 +347,19 @@ class OpenStackAmuletUtils(AmuletUtils): config = {'preferred-api-version': api_version} deployment.d.configure('keystone', config) + deployment._auto_wait_for_status() self.keystone_wait_for_propagation(sentry_relation_pairs, api_version) def authenticate_cinder_admin(self, keystone_sentry, username, - password, tenant): + password, tenant, api_version=2): """Authenticates admin user with cinder.""" # NOTE(beisner): cinder python client doesn't accept tokens. keystone_ip = keystone_sentry.info['public-address'] ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8')) - return cinder_client.Client(username, password, tenant, ept) + _clients = { + 1: cinder_client.Client, + 2: cinder_clientv2.Client} + return _clients[api_version](username, password, tenant, ept) def authenticate_keystone(self, keystone_ip, username, password, api_version=False, admin_port=False, @@ -617,13 +620,25 @@ class OpenStackAmuletUtils(AmuletUtils): self.log.debug('Keypair ({}) already exists, ' 'using it.'.format(keypair_name)) return _keypair - except: + except Exception: self.log.debug('Keypair ({}) does not exist, ' 'creating it.'.format(keypair_name)) _keypair = nova.keypairs.create(name=keypair_name) return _keypair + def _get_cinder_obj_name(self, cinder_object): + """Retrieve name of cinder object. + + :param cinder_object: cinder snapshot or volume object + :returns: str cinder object name + """ + # v1 objects store name in 'display_name' attr but v2+ use 'name' + try: + return cinder_object.display_name + except AttributeError: + return cinder_object.name + def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1, img_id=None, src_vol_id=None, snap_id=None): """Create cinder volume, optionally from a glance image, OR @@ -674,6 +689,13 @@ class OpenStackAmuletUtils(AmuletUtils): source_volid=src_vol_id, snapshot_id=snap_id) vol_id = vol_new.id + except TypeError: + vol_new = cinder.volumes.create(name=vol_name, + imageRef=img_id, + size=vol_size, + source_volid=src_vol_id, + snapshot_id=snap_id) + vol_id = vol_new.id except Exception as e: msg = 'Failed to create volume: {}'.format(e) amulet.raise_status(amulet.FAIL, msg=msg) @@ -688,7 +710,7 @@ class OpenStackAmuletUtils(AmuletUtils): # Re-validate new volume self.log.debug('Validating volume attributes...') - val_vol_name = cinder.volumes.get(vol_id).display_name + val_vol_name = self._get_cinder_obj_name(cinder.volumes.get(vol_id)) val_vol_boot = cinder.volumes.get(vol_id).bootable val_vol_stat = cinder.volumes.get(vol_id).status val_vol_size = cinder.volumes.get(vol_id).size diff --git a/charmhelpers/contrib/openstack/context.py b/charmhelpers/contrib/openstack/context.py index b486b210..ece75df8 100644 --- a/charmhelpers/contrib/openstack/context.py +++ b/charmhelpers/contrib/openstack/context.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import collections import glob import json import math @@ -578,11 +579,14 @@ class HAProxyContext(OSContextGenerator): laddr = get_address_in_network(config(cfg_opt)) if laddr: netmask = get_netmask_for_address(laddr) - cluster_hosts[laddr] = {'network': "{}/{}".format(laddr, - netmask), - 'backends': {l_unit: laddr}} + cluster_hosts[laddr] = { + 'network': "{}/{}".format(laddr, + netmask), + 'backends': collections.OrderedDict([(l_unit, + laddr)]) + } for rid in relation_ids('cluster'): - for unit in related_units(rid): + for unit in sorted(related_units(rid)): _laddr = relation_get('{}-address'.format(addr_type), rid=rid, unit=unit) if _laddr: @@ -594,10 +598,13 @@ class HAProxyContext(OSContextGenerator): # match in the frontend cluster_hosts[addr] = {} netmask = get_netmask_for_address(addr) - cluster_hosts[addr] = {'network': "{}/{}".format(addr, netmask), - 'backends': {l_unit: addr}} + cluster_hosts[addr] = { + 'network': "{}/{}".format(addr, netmask), + 'backends': collections.OrderedDict([(l_unit, + addr)]) + } for rid in relation_ids('cluster'): - for unit in related_units(rid): + for unit in sorted(related_units(rid)): _laddr = relation_get('private-address', rid=rid, unit=unit) if _laddr: @@ -628,6 +635,8 @@ class HAProxyContext(OSContextGenerator): ctxt['local_host'] = '127.0.0.1' ctxt['haproxy_host'] = '0.0.0.0' + ctxt['ipv6_enabled'] = not is_ipv6_disabled() + ctxt['stat_port'] = '8888' db = kv() @@ -844,15 +853,6 @@ class NeutronContext(OSContextGenerator): for pkgs in self.packages: ensure_packages(pkgs) - def _save_flag_file(self): - if self.network_manager == 'quantum': - _file = '/etc/nova/quantum_plugin.conf' - else: - _file = '/etc/nova/neutron_plugin.conf' - - with open(_file, 'wb') as out: - out.write(self.plugin + '\n') - def ovs_ctxt(self): driver = neutron_plugin_attribute(self.plugin, 'driver', self.network_manager) @@ -997,7 +997,6 @@ class NeutronContext(OSContextGenerator): flags = config_flags_parser(alchemy_flags) ctxt['neutron_alchemy_flags'] = flags - self._save_flag_file() return ctxt @@ -1177,7 +1176,7 @@ class SubordinateConfigContext(OSContextGenerator): if sub_config and sub_config != '': try: sub_config = json.loads(sub_config) - except: + except Exception: log('Could not parse JSON from ' 'subordinate_configuration setting from %s' % rid, level=ERROR) diff --git a/charmhelpers/contrib/openstack/templates/ceph.conf b/charmhelpers/contrib/openstack/templates/ceph.conf index ed5c4f10..a11ce8ab 100644 --- a/charmhelpers/contrib/openstack/templates/ceph.conf +++ b/charmhelpers/contrib/openstack/templates/ceph.conf @@ -18,7 +18,7 @@ rbd default features = {{ rbd_features }} [client] {% if rbd_client_cache_settings -%} -{% for key, value in rbd_client_cache_settings.iteritems() -%} +{% for key, value in rbd_client_cache_settings.items() -%} {{ key }} = {{ value }} {% endfor -%} -{%- endif %} \ No newline at end of file +{%- endif %} diff --git a/charmhelpers/contrib/openstack/templates/haproxy.cfg b/charmhelpers/contrib/openstack/templates/haproxy.cfg index 2e660450..ebc8a68a 100644 --- a/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -48,7 +48,9 @@ listen stats {% for service, ports in service_ports.items() -%} frontend tcp-in_{{ service }} bind *:{{ ports[0] }} + {% if ipv6_enabled -%} bind :::{{ ports[0] }} + {% endif -%} {% for frontend in frontends -%} acl net_{{ frontend }} dst {{ frontends[frontend]['network'] }} use_backend {{ service }}_{{ frontend }} if net_{{ frontend }} diff --git a/charmhelpers/contrib/openstack/templating.py b/charmhelpers/contrib/openstack/templating.py index d8c1fc7f..77490e4d 100644 --- a/charmhelpers/contrib/openstack/templating.py +++ b/charmhelpers/contrib/openstack/templating.py @@ -272,6 +272,8 @@ class OSConfigRenderer(object): raise OSConfigException _out = self.render(config_file) + if six.PY3: + _out = _out.encode('UTF-8') with open(config_file, 'wb') as out: out.write(_out) diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index 6054a242..b073c77b 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -426,7 +426,7 @@ def get_os_codename_package(package, fatal=True): try: pkg = cache[package] - except: + except Exception: if not fatal: return None # the package is unknown to the current apt cache. @@ -618,7 +618,7 @@ def save_script_rc(script_path="scripts/scriptrc", **env_vars): juju_rc_path = "%s/%s" % (charm_dir(), script_path) if not os.path.exists(os.path.dirname(juju_rc_path)): os.mkdir(os.path.dirname(juju_rc_path)) - with open(juju_rc_path, 'wb') as rc_script: + with open(juju_rc_path, 'wt') as rc_script: rc_script.write( "#!/bin/bash\n") [rc_script.write('export %s=%s\n' % (u, p)) @@ -797,7 +797,7 @@ def git_default_repos(projects_yaml): service = service_name() core_project = service - for default, branch in GIT_DEFAULT_BRANCHES.iteritems(): + for default, branch in six.iteritems(GIT_DEFAULT_BRANCHES): if projects_yaml == default: # add the requirements repo first @@ -1618,7 +1618,7 @@ def do_action_openstack_upgrade(package, upgrade_callback, configs): upgrade_callback(configs=configs) action_set({'outcome': 'success, upgrade completed.'}) ret = True - except: + except Exception: action_set({'outcome': 'upgrade failed, see traceback.'}) action_set({'traceback': traceback.format_exc()}) action_fail('do_openstack_upgrade resulted in an ' @@ -1723,7 +1723,7 @@ def is_unit_paused_set(): kv = t[0] # transform something truth-y into a Boolean. return not(not(kv.get('unit-paused'))) - except: + except Exception: return False @@ -2051,7 +2051,7 @@ def update_json_file(filename, items): def snap_install_requested(): """ Determine if installing from snaps - If openstack-origin is of the form snap:track/channel + If openstack-origin is of the form snap:track/channel[/branch] and channel is in SNAPS_CHANNELS return True. """ origin = config('openstack-origin') or "" @@ -2060,9 +2060,9 @@ def snap_install_requested(): _src = origin[5:] if '/' in _src: - _track, channel = _src.split('/') + channel = _src.split('/')[1] else: - # Hanlde snap:track with no channel + # Handle snap:track with no channel channel = 'stable' return valid_snap_channel(channel) diff --git a/charmhelpers/contrib/python/debug.py b/charmhelpers/contrib/python/debug.py index 7d04dfa5..d2142c75 100644 --- a/charmhelpers/contrib/python/debug.py +++ b/charmhelpers/contrib/python/debug.py @@ -49,6 +49,6 @@ def set_trace(addr=DEFAULT_ADDR, port=DEFAULT_PORT): open_port(port) debugger = Rpdb(addr=addr, port=port) debugger.set_trace(sys._getframe().f_back) - except: + except Exception: _error("Cannot start a remote debug session on %s:%s" % (addr, port)) diff --git a/charmhelpers/contrib/storage/linux/ceph.py b/charmhelpers/contrib/storage/linux/ceph.py index e5a01b1b..39231612 100644 --- a/charmhelpers/contrib/storage/linux/ceph.py +++ b/charmhelpers/contrib/storage/linux/ceph.py @@ -370,9 +370,10 @@ def get_mon_map(service): Also raises CalledProcessError if our ceph command fails """ try: - mon_status = check_output( - ['ceph', '--id', service, - 'mon_status', '--format=json']) + mon_status = check_output(['ceph', '--id', service, + 'mon_status', '--format=json']) + if six.PY3: + mon_status = mon_status.decode('UTF-8') try: return json.loads(mon_status) except ValueError as v: @@ -457,7 +458,7 @@ def monitor_key_get(service, key): try: output = check_output( ['ceph', '--id', service, - 'config-key', 'get', str(key)]) + 'config-key', 'get', str(key)]).decode('UTF-8') return output except CalledProcessError as e: log("Monitor config-key get failed with message: {}".format( @@ -500,6 +501,8 @@ def get_erasure_profile(service, name): out = check_output(['ceph', '--id', service, 'osd', 'erasure-code-profile', 'get', name, '--format=json']) + if six.PY3: + out = out.decode('UTF-8') return json.loads(out) except (CalledProcessError, OSError, ValueError): return None @@ -686,7 +689,10 @@ def get_cache_mode(service, pool_name): """ validator(value=service, valid_type=six.string_types) validator(value=pool_name, valid_type=six.string_types) - out = check_output(['ceph', '--id', service, 'osd', 'dump', '--format=json']) + out = check_output(['ceph', '--id', service, + 'osd', 'dump', '--format=json']) + if six.PY3: + out = out.decode('UTF-8') try: osd_json = json.loads(out) for pool in osd_json['pools']: @@ -700,8 +706,9 @@ def get_cache_mode(service, pool_name): def pool_exists(service, name): """Check to see if a RADOS pool already exists.""" try: - out = check_output(['rados', '--id', service, - 'lspools']).decode('UTF-8') + out = check_output(['rados', '--id', service, 'lspools']) + if six.PY3: + out = out.decode('UTF-8') except CalledProcessError: return False @@ -714,9 +721,12 @@ def get_osds(service): """ version = ceph_version() if version and version >= '0.56': - return json.loads(check_output(['ceph', '--id', service, - 'osd', 'ls', - '--format=json']).decode('UTF-8')) + out = check_output(['ceph', '--id', service, + 'osd', 'ls', + '--format=json']) + if six.PY3: + out = out.decode('UTF-8') + return json.loads(out) return None @@ -734,7 +744,9 @@ def rbd_exists(service, pool, rbd_img): """Check to see if a RADOS block device exists.""" try: out = check_output(['rbd', 'list', '--id', - service, '--pool', pool]).decode('UTF-8') + service, '--pool', pool]) + if six.PY3: + out = out.decode('UTF-8') except CalledProcessError: return False @@ -859,7 +871,9 @@ def configure(service, key, auth, use_syslog): def image_mapped(name): """Determine whether a RADOS block device is mapped locally.""" try: - out = check_output(['rbd', 'showmapped']).decode('UTF-8') + out = check_output(['rbd', 'showmapped']) + if six.PY3: + out = out.decode('UTF-8') except CalledProcessError: return False @@ -1018,7 +1032,9 @@ def ceph_version(): """Retrieve the local version of ceph.""" if os.path.exists('/usr/bin/ceph'): cmd = ['ceph', '-v'] - output = check_output(cmd).decode('US-ASCII') + output = check_output(cmd) + if six.PY3: + output = output.decode('UTF-8') output = output.split() if len(output) > 3: return output[2] diff --git a/charmhelpers/contrib/storage/linux/lvm.py b/charmhelpers/contrib/storage/linux/lvm.py index 4719f53c..7f2a0604 100644 --- a/charmhelpers/contrib/storage/linux/lvm.py +++ b/charmhelpers/contrib/storage/linux/lvm.py @@ -74,10 +74,10 @@ def list_lvm_volume_group(block_device): ''' vg = None pvd = check_output(['pvdisplay', block_device]).splitlines() - for l in pvd: - l = l.decode('UTF-8') - if l.strip().startswith('VG Name'): - vg = ' '.join(l.strip().split()[2:]) + for lvm in pvd: + lvm = lvm.decode('UTF-8') + if lvm.strip().startswith('VG Name'): + vg = ' '.join(lvm.strip().split()[2:]) return vg diff --git a/charmhelpers/contrib/storage/linux/utils.py b/charmhelpers/contrib/storage/linux/utils.py index 3dc0df68..c9428894 100644 --- a/charmhelpers/contrib/storage/linux/utils.py +++ b/charmhelpers/contrib/storage/linux/utils.py @@ -64,6 +64,6 @@ def is_device_mounted(device): ''' try: out = check_output(['lsblk', '-P', device]).decode('UTF-8') - except: + except Exception: return False return bool(re.search(r'MOUNTPOINT=".+"', out)) diff --git a/charmhelpers/contrib/unison/__init__.py b/charmhelpers/contrib/unison/__init__.py index 83108883..61409b14 100644 --- a/charmhelpers/contrib/unison/__init__.py +++ b/charmhelpers/contrib/unison/__init__.py @@ -283,7 +283,7 @@ def sync_path_to_host(path, host, user, verbose=False, cmd=None, gid=None, try: log('Syncing local path %s to %s@%s:%s' % (path, user, host, path)) run_as_user(user, cmd, gid) - except: + except Exception: log('Error syncing remote files') if fatal: raise diff --git a/charmhelpers/core/hookenv.py b/charmhelpers/core/hookenv.py index fb96f2dd..5a88f798 100644 --- a/charmhelpers/core/hookenv.py +++ b/charmhelpers/core/hookenv.py @@ -22,6 +22,7 @@ from __future__ import print_function import copy from distutils.version import LooseVersion from functools import wraps +from collections import namedtuple import glob import os import json @@ -644,18 +645,31 @@ def is_relation_made(relation, keys='private-address'): return False +def _port_op(op_name, port, protocol="TCP"): + """Open or close a service network port""" + _args = [op_name] + icmp = protocol.upper() == "ICMP" + if icmp: + _args.append(protocol) + else: + _args.append('{}/{}'.format(port, protocol)) + try: + subprocess.check_call(_args) + except subprocess.CalledProcessError: + # Older Juju pre 2.3 doesn't support ICMP + # so treat it as a no-op if it fails. + if not icmp: + raise + + def open_port(port, protocol="TCP"): """Open a service network port""" - _args = ['open-port'] - _args.append('{}/{}'.format(port, protocol)) - subprocess.check_call(_args) + _port_op('open-port', port, protocol) def close_port(port, protocol="TCP"): """Close a service network port""" - _args = ['close-port'] - _args.append('{}/{}'.format(port, protocol)) - subprocess.check_call(_args) + _port_op('close-port', port, protocol) def open_ports(start, end, protocol="TCP"): @@ -1101,13 +1115,24 @@ def network_get(endpoint, relation_id=None): :param endpoint: string. The name of a relation endpoint :param relation_id: int. The ID of the relation for the current context. :return: dict. The loaded YAML output of the network-get query. - :raise: NotImplementedError if run on Juju < 2.0 + :raise: NotImplementedError if run on Juju < 2.1 """ cmd = ['network-get', endpoint, '--format', 'yaml'] if relation_id: cmd.append('-r') cmd.append(relation_id) - response = subprocess.check_output(cmd).decode('UTF-8').strip() + try: + response = subprocess.check_output( + cmd, + stderr=subprocess.STDOUT).decode('UTF-8').strip() + except CalledProcessError as e: + # Early versions of Juju 2.0.x required the --primary-address argument. + # We catch that condition here and raise NotImplementedError since + # the requested semantics are not available - the caller can then + # use the network_get_primary_address() method instead. + if '--primary-address is currently required' in e.output.decode('UTF-8'): + raise NotImplementedError + raise return yaml.safe_load(response) @@ -1140,3 +1165,42 @@ def meter_info(): """Get the meter status information, if running in the meter-status-changed hook.""" return os.environ.get('JUJU_METER_INFO') + + +def iter_units_for_relation_name(relation_name): + """Iterate through all units in a relation + + Generator that iterates through all the units in a relation and yields + a named tuple with rid and unit field names. + + Usage: + data = [(u.rid, u.unit) + for u in iter_units_for_relation_name(relation_name)] + + :param relation_name: string relation name + :yield: Named Tuple with rid and unit field names + """ + RelatedUnit = namedtuple('RelatedUnit', 'rid, unit') + for rid in relation_ids(relation_name): + for unit in related_units(rid): + yield RelatedUnit(rid, unit) + + +def ingress_address(rid=None, unit=None): + """ + Retrieve the ingress-address from a relation when available. Otherwise, + return the private-address. This function is to be used on the consuming + side of the relation. + + Usage: + addresses = [ingress_address(rid=u.rid, unit=u.unit) + for u in iter_units_for_relation_name(relation_name)] + + :param rid: string relation id + :param unit: string unit name + :side effect: calls relation_get + :return: string IP address + """ + settings = relation_get(rid=rid, unit=unit) + return (settings.get('ingress-address') or + settings.get('private-address')) diff --git a/charmhelpers/core/strutils.py b/charmhelpers/core/strutils.py index 685dabde..e8df0452 100644 --- a/charmhelpers/core/strutils.py +++ b/charmhelpers/core/strutils.py @@ -61,13 +61,19 @@ def bytes_from_string(value): if isinstance(value, six.string_types): value = six.text_type(value) else: - msg = "Unable to interpret non-string value '%s' as boolean" % (value) + msg = "Unable to interpret non-string value '%s' as bytes" % (value) raise ValueError(msg) matches = re.match("([0-9]+)([a-zA-Z]+)", value) - if not matches: - msg = "Unable to interpret string value '%s' as bytes" % (value) - raise ValueError(msg) - return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)]) + if matches: + size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)]) + else: + # Assume that value passed in is bytes + try: + size = int(value) + except ValueError: + msg = "Unable to interpret string value '%s' as bytes" % (value) + raise ValueError(msg) + return size class BasicStringComparator(object): diff --git a/charmhelpers/core/unitdata.py b/charmhelpers/core/unitdata.py index 54ec969f..7af875c2 100644 --- a/charmhelpers/core/unitdata.py +++ b/charmhelpers/core/unitdata.py @@ -358,7 +358,7 @@ class Storage(object): try: yield self.revision self.revision = None - except: + except Exception: self.flush(False) self.revision = None raise diff --git a/charmhelpers/fetch/ubuntu.py b/charmhelpers/fetch/ubuntu.py index 40e1cb5b..910e96a6 100644 --- a/charmhelpers/fetch/ubuntu.py +++ b/charmhelpers/fetch/ubuntu.py @@ -572,7 +572,7 @@ def get_upstream_version(package): cache = apt_cache() try: pkg = cache[package] - except: + except Exception: # the package is unknown to the current apt cache. return None diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index dab27cd6..a29ba13f 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -422,6 +422,11 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): def validate_keystone_users(self, client): """Verify all existing roles.""" u.log.debug('Checking keystone users...') + + if self._get_openstack_release() < self.xenial_pike: + cinder_user = 'cinder_cinderv2' + else: + cinder_user = 'cinderv3_cinderv2' base = [ {'name': 'demoUser', 'enabled': True, @@ -431,7 +436,7 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): 'enabled': True, 'id': u.not_null, 'email': 'juju@localhost'}, - {'name': 'cinder_cinderv2', + {'name': cinder_user, 'enabled': True, 'id': u.not_null, 'email': u'juju@localhost'} @@ -609,6 +614,9 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): 'volume': [endpoint_check], 'identity': [endpoint_check] } + if self._get_openstack_release() >= self.xenial_pike: + expected.pop('volume') + expected['volumev2'] = [endpoint_check] actual = self.keystone_v2.service_catalog.get_endpoints() ret = u.validate_svc_catalog_endpoint_data(expected, actual) @@ -704,6 +712,8 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): 'service_tenant_id': u.not_null, 'service_host': u.valid_ip } + if self._get_openstack_release() >= self.xenial_pike: + expected['service_username'] = 'cinderv3_cinderv2' for unit in self.keystone_sentries: ret = u.validate_relation_data(unit, relation, expected) if ret: @@ -728,6 +738,22 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): 'cinderv2_admin_url': u.valid_url, 'private-address': u.valid_ip, } + + if self._get_openstack_release() >= self.xenial_pike: + expected.pop('cinder_region') + expected.pop('cinder_service') + expected.pop('cinder_public_url') + expected.pop('cinder_admin_url') + expected.pop('cinder_internal_url') + expected.update({ + 'cinderv2_region': 'RegionOne', + 'cinderv3_region': 'RegionOne', + 'cinderv3_service': 'cinderv3', + 'cinderv3_region': 'RegionOne', + 'cinderv3_public_url': u.valid_url, + 'cinderv3_internal_url': u.valid_url, + 'cinderv3_admin_url': u.valid_url}) + ret = u.validate_relation_data(unit, relation, expected) if ret: message = u.relation_error('cinder identity-service', ret) diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 5c041d2c..5e33eb71 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -250,7 +250,14 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.log.debug('Waiting up to {}s for extended status on services: ' '{}'.format(timeout, services)) service_messages = {service: message for service in services} + + # Check for idleness + self.d.sentry.wait() + # Check for error states and bail early + self.d.sentry.wait_for_status(self.d.juju_env, services) + # Check for ready messages self.d.sentry.wait_for_messages(service_messages, timeout=timeout) + self.log.info('OK') def _get_openstack_release(self): @@ -303,20 +310,27 @@ class OpenStackAmuletDeployment(AmuletDeployment): test scenario, based on OpenStack release and whether ceph radosgw is flagged as present or not.""" - if self._get_openstack_release() >= self.trusty_kilo: - # Kilo or later - pools = [ - 'rbd', - 'cinder', - 'glance' - ] - else: + if self._get_openstack_release() <= self.trusty_juno: # Juno or earlier pools = [ 'data', 'metadata', 'rbd', - 'cinder', + 'cinder-ceph', + 'glance' + ] + elif (self.trust_kilo <= self._get_openstack_release() <= + self.zesty_ocata): + # Kilo through Ocata + pools = [ + 'rbd', + 'cinder-ceph', + 'glance' + ] + else: + # Pike and later + pools = [ + 'cinder-ceph', 'glance' ] diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index c8edbf65..b71b2b19 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -23,6 +23,7 @@ import urllib import urlparse import cinderclient.v1.client as cinder_client +import cinderclient.v2.client as cinder_clientv2 import glanceclient.v1.client as glance_client import heatclient.v1.client as heat_client from keystoneclient.v2_0 import client as keystone_client @@ -42,7 +43,6 @@ import swiftclient from charmhelpers.contrib.amulet.utils import ( AmuletUtils ) -from charmhelpers.core.decorators import retry_on_exception from charmhelpers.core.host import CompareHostReleases DEBUG = logging.DEBUG @@ -310,7 +310,6 @@ class OpenStackAmuletUtils(AmuletUtils): self.log.debug('Checking if tenant exists ({})...'.format(tenant)) return tenant in [t.name for t in keystone.tenants.list()] - @retry_on_exception(5, base_delay=10) def keystone_wait_for_propagation(self, sentry_relation_pairs, api_version): """Iterate over list of sentry and relation tuples and verify that @@ -326,7 +325,7 @@ class OpenStackAmuletUtils(AmuletUtils): rel = sentry.relation('identity-service', relation_name) self.log.debug('keystone relation data: {}'.format(rel)) - if rel['api_version'] != str(api_version): + if rel.get('api_version') != str(api_version): raise Exception("api_version not propagated through relation" " data yet ('{}' != '{}')." "".format(rel['api_version'], api_version)) @@ -348,15 +347,19 @@ class OpenStackAmuletUtils(AmuletUtils): config = {'preferred-api-version': api_version} deployment.d.configure('keystone', config) + deployment._auto_wait_for_status() self.keystone_wait_for_propagation(sentry_relation_pairs, api_version) def authenticate_cinder_admin(self, keystone_sentry, username, - password, tenant): + password, tenant, api_version=2): """Authenticates admin user with cinder.""" # NOTE(beisner): cinder python client doesn't accept tokens. keystone_ip = keystone_sentry.info['public-address'] ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8')) - return cinder_client.Client(username, password, tenant, ept) + _clients = { + 1: cinder_client.Client, + 2: cinder_clientv2.Client} + return _clients[api_version](username, password, tenant, ept) def authenticate_keystone(self, keystone_ip, username, password, api_version=False, admin_port=False, @@ -617,13 +620,25 @@ class OpenStackAmuletUtils(AmuletUtils): self.log.debug('Keypair ({}) already exists, ' 'using it.'.format(keypair_name)) return _keypair - except: + except Exception: self.log.debug('Keypair ({}) does not exist, ' 'creating it.'.format(keypair_name)) _keypair = nova.keypairs.create(name=keypair_name) return _keypair + def _get_cinder_obj_name(self, cinder_object): + """Retrieve name of cinder object. + + :param cinder_object: cinder snapshot or volume object + :returns: str cinder object name + """ + # v1 objects store name in 'display_name' attr but v2+ use 'name' + try: + return cinder_object.display_name + except AttributeError: + return cinder_object.name + def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1, img_id=None, src_vol_id=None, snap_id=None): """Create cinder volume, optionally from a glance image, OR @@ -674,6 +689,13 @@ class OpenStackAmuletUtils(AmuletUtils): source_volid=src_vol_id, snapshot_id=snap_id) vol_id = vol_new.id + except TypeError: + vol_new = cinder.volumes.create(name=vol_name, + imageRef=img_id, + size=vol_size, + source_volid=src_vol_id, + snapshot_id=snap_id) + vol_id = vol_new.id except Exception as e: msg = 'Failed to create volume: {}'.format(e) amulet.raise_status(amulet.FAIL, msg=msg) @@ -688,7 +710,7 @@ class OpenStackAmuletUtils(AmuletUtils): # Re-validate new volume self.log.debug('Validating volume attributes...') - val_vol_name = cinder.volumes.get(vol_id).display_name + val_vol_name = self._get_cinder_obj_name(cinder.volumes.get(vol_id)) val_vol_boot = cinder.volumes.get(vol_id).bootable val_vol_stat = cinder.volumes.get(vol_id).status val_vol_size = cinder.volumes.get(vol_id).size diff --git a/tests/charmhelpers/core/hookenv.py b/tests/charmhelpers/core/hookenv.py index fb96f2dd..5a88f798 100644 --- a/tests/charmhelpers/core/hookenv.py +++ b/tests/charmhelpers/core/hookenv.py @@ -22,6 +22,7 @@ from __future__ import print_function import copy from distutils.version import LooseVersion from functools import wraps +from collections import namedtuple import glob import os import json @@ -644,18 +645,31 @@ def is_relation_made(relation, keys='private-address'): return False +def _port_op(op_name, port, protocol="TCP"): + """Open or close a service network port""" + _args = [op_name] + icmp = protocol.upper() == "ICMP" + if icmp: + _args.append(protocol) + else: + _args.append('{}/{}'.format(port, protocol)) + try: + subprocess.check_call(_args) + except subprocess.CalledProcessError: + # Older Juju pre 2.3 doesn't support ICMP + # so treat it as a no-op if it fails. + if not icmp: + raise + + def open_port(port, protocol="TCP"): """Open a service network port""" - _args = ['open-port'] - _args.append('{}/{}'.format(port, protocol)) - subprocess.check_call(_args) + _port_op('open-port', port, protocol) def close_port(port, protocol="TCP"): """Close a service network port""" - _args = ['close-port'] - _args.append('{}/{}'.format(port, protocol)) - subprocess.check_call(_args) + _port_op('close-port', port, protocol) def open_ports(start, end, protocol="TCP"): @@ -1101,13 +1115,24 @@ def network_get(endpoint, relation_id=None): :param endpoint: string. The name of a relation endpoint :param relation_id: int. The ID of the relation for the current context. :return: dict. The loaded YAML output of the network-get query. - :raise: NotImplementedError if run on Juju < 2.0 + :raise: NotImplementedError if run on Juju < 2.1 """ cmd = ['network-get', endpoint, '--format', 'yaml'] if relation_id: cmd.append('-r') cmd.append(relation_id) - response = subprocess.check_output(cmd).decode('UTF-8').strip() + try: + response = subprocess.check_output( + cmd, + stderr=subprocess.STDOUT).decode('UTF-8').strip() + except CalledProcessError as e: + # Early versions of Juju 2.0.x required the --primary-address argument. + # We catch that condition here and raise NotImplementedError since + # the requested semantics are not available - the caller can then + # use the network_get_primary_address() method instead. + if '--primary-address is currently required' in e.output.decode('UTF-8'): + raise NotImplementedError + raise return yaml.safe_load(response) @@ -1140,3 +1165,42 @@ def meter_info(): """Get the meter status information, if running in the meter-status-changed hook.""" return os.environ.get('JUJU_METER_INFO') + + +def iter_units_for_relation_name(relation_name): + """Iterate through all units in a relation + + Generator that iterates through all the units in a relation and yields + a named tuple with rid and unit field names. + + Usage: + data = [(u.rid, u.unit) + for u in iter_units_for_relation_name(relation_name)] + + :param relation_name: string relation name + :yield: Named Tuple with rid and unit field names + """ + RelatedUnit = namedtuple('RelatedUnit', 'rid, unit') + for rid in relation_ids(relation_name): + for unit in related_units(rid): + yield RelatedUnit(rid, unit) + + +def ingress_address(rid=None, unit=None): + """ + Retrieve the ingress-address from a relation when available. Otherwise, + return the private-address. This function is to be used on the consuming + side of the relation. + + Usage: + addresses = [ingress_address(rid=u.rid, unit=u.unit) + for u in iter_units_for_relation_name(relation_name)] + + :param rid: string relation id + :param unit: string unit name + :side effect: calls relation_get + :return: string IP address + """ + settings = relation_get(rid=rid, unit=unit) + return (settings.get('ingress-address') or + settings.get('private-address')) diff --git a/tests/charmhelpers/core/strutils.py b/tests/charmhelpers/core/strutils.py index 685dabde..e8df0452 100644 --- a/tests/charmhelpers/core/strutils.py +++ b/tests/charmhelpers/core/strutils.py @@ -61,13 +61,19 @@ def bytes_from_string(value): if isinstance(value, six.string_types): value = six.text_type(value) else: - msg = "Unable to interpret non-string value '%s' as boolean" % (value) + msg = "Unable to interpret non-string value '%s' as bytes" % (value) raise ValueError(msg) matches = re.match("([0-9]+)([a-zA-Z]+)", value) - if not matches: - msg = "Unable to interpret string value '%s' as bytes" % (value) - raise ValueError(msg) - return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)]) + if matches: + size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)]) + else: + # Assume that value passed in is bytes + try: + size = int(value) + except ValueError: + msg = "Unable to interpret string value '%s' as bytes" % (value) + raise ValueError(msg) + return size class BasicStringComparator(object): diff --git a/tests/charmhelpers/core/unitdata.py b/tests/charmhelpers/core/unitdata.py index 54ec969f..7af875c2 100644 --- a/tests/charmhelpers/core/unitdata.py +++ b/tests/charmhelpers/core/unitdata.py @@ -358,7 +358,7 @@ class Storage(object): try: yield self.revision self.revision = None - except: + except Exception: self.flush(False) self.revision = None raise diff --git a/tests/gate-basic-xenial-pike b/tests/gate-basic-xenial-pike old mode 100644 new mode 100755 diff --git a/tox.ini b/tox.ini index 7c2936e3..6d44f4b9 100644 --- a/tox.ini +++ b/tox.ini @@ -60,7 +60,7 @@ basepython = python2.7 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = - bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-mitaka --no-destroy + bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-pike --no-destroy [testenv:func27-dfs] # Charm Functional Test diff --git a/unit_tests/test_keystone_contexts.py b/unit_tests/test_keystone_contexts.py index fda0bed6..e245b11b 100644 --- a/unit_tests/test_keystone_contexts.py +++ b/unit_tests/test_keystone_contexts.py @@ -186,6 +186,7 @@ class TestKeystoneContexts(CharmTestCase): 'service_ports': {'admin-port': ['12', '34'], 'public-port': ['12', '34']}, 'default_backend': '1.2.3.4', + 'ipv6_enabled': True, 'frontends': {'1.2.3.4': { 'network': '1.2.3.4/255.255.255.0', 'backends': {