diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py index 424b7f76..80d574dc 100644 --- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py +++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py @@ -125,7 +125,7 @@ class CheckException(Exception): class Check(object): - shortname_re = '[A-Za-z0-9-_]+$' + shortname_re = '[A-Za-z0-9-_.]+$' service_template = (""" #--------------------------------------------------- # This file is Juju managed diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 15f65960..d7e6debf 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -243,11 +243,11 @@ def is_ipv6_disabled(): try: result = subprocess.check_output( ['sysctl', 'net.ipv6.conf.all.disable_ipv6'], - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + universal_newlines=True) except subprocess.CalledProcessError: return True - if six.PY3: - result = result.decode('UTF-8') + return "net.ipv6.conf.all.disable_ipv6 = 1" in result diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index ea93159d..37528c47 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -97,6 +97,7 @@ from charmhelpers.contrib.openstack.utils import ( git_determine_usr_bin, git_determine_python_path, enable_memcache, + snap_install_requested, ) from charmhelpers.core.unitdata import kv @@ -244,6 +245,11 @@ class SharedDBContext(OSContextGenerator): 'database_password': rdata.get(password_setting), 'database_type': 'mysql' } + # Note(coreycb): We can drop mysql+pymysql if we want when the + # following review lands, though it seems mysql+pymysql would + # be preferred. https://review.openstack.org/#/c/462190/ + if snap_install_requested(): + ctxt['database_type'] = 'mysql+pymysql' if self.context_complete(ctxt): db_ssl(rdata, ctxt, self.ssl_dir) return ctxt @@ -510,6 +516,10 @@ class CephContext(OSContextGenerator): ctxt['auth'] = relation_get('auth', rid=rid, unit=unit) if not ctxt.get('key'): ctxt['key'] = relation_get('key', rid=rid, unit=unit) + if not ctxt.get('rbd_features'): + default_features = relation_get('rbd-features', rid=rid, unit=unit) + if default_features is not None: + ctxt['rbd_features'] = default_features ceph_addrs = relation_get('ceph-public-address', rid=rid, unit=unit) @@ -1397,6 +1407,18 @@ class NeutronAPIContext(OSContextGenerator): 'rel_key': 'dns-domain', 'default': None, }, + 'polling_interval': { + 'rel_key': 'polling-interval', + 'default': 2, + }, + 'rpc_response_timeout': { + 'rel_key': 'rpc-response-timeout', + 'default': 60, + }, + 'report_interval': { + 'rel_key': 'report-interval', + 'default': 30, + }, } ctxt = self.get_neutron_options({}) for rid in relation_ids('neutron-plugin-api'): diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf index 33ceee25..ed5c4f10 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf @@ -1,6 +1,6 @@ ############################################################################### # [ WARNING ] -# cinder configuration file maintained by Juju +# ceph configuration file maintained by Juju # local changes may be overwritten. ############################################################################### [global] @@ -12,6 +12,9 @@ mon host = {{ mon_hosts }} log to syslog = {{ use_syslog }} err to syslog = {{ use_syslog }} clog to syslog = {{ use_syslog }} +{% if rbd_features %} +rbd default features = {{ rbd_features }} +{% endif %} [client] {% if rbd_client_cache_settings -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index 54fba39d..edae7a0c 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -1,6 +1,6 @@ global - log {{ local_host }} local0 - log {{ local_host }} local1 notice + log /var/lib/haproxy/dev/log local0 + log /var/lib/haproxy/dev/log local1 notice maxconn 20000 user haproxy group haproxy diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-oslo-notifications b/hooks/charmhelpers/contrib/openstack/templates/section-oslo-notifications new file mode 100644 index 00000000..5dccd4bb --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/section-oslo-notifications @@ -0,0 +1,8 @@ +{% if transport_url -%} +[oslo_messaging_notifications] +driver = messagingv2 +transport_url = {{ transport_url }} +{% if notification_topics -%} +topics = {{ notification_topics }} +{% endif -%} +{% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 1eaab95f..5b88b33a 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -51,6 +51,7 @@ from charmhelpers.core.hookenv import ( status_set, hook_name, application_version_set, + cached, ) from charmhelpers.core.strutils import BasicStringComparator @@ -90,6 +91,13 @@ from charmhelpers.fetch import ( GPGKeyError, get_upstream_version ) + +from charmhelpers.fetch.snap import ( + snap_install, + snap_refresh, + SNAP_CHANNELS, +) + from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device from charmhelpers.contrib.openstack.exceptions import OSContextError @@ -327,8 +335,10 @@ def get_os_codename_install_source(src): return ca_rel # Best guess match based on deb string provided - if src.startswith('deb') or src.startswith('ppa'): - for k, v in six.iteritems(OPENSTACK_CODENAMES): + if (src.startswith('deb') or + src.startswith('ppa') or + src.startswith('snap')): + for v in OPENSTACK_CODENAMES.values(): if v in src: return v @@ -397,6 +407,19 @@ def get_swift_codename(version): def get_os_codename_package(package, fatal=True): '''Derive OpenStack release codename from an installed package.''' + + if snap_install_requested(): + cmd = ['snap', 'list', package] + try: + out = subprocess.check_output(cmd) + except subprocess.CalledProcessError as e: + return None + lines = out.split('\n') + for line in lines: + if package in line: + # Second item in list is Version + return line.split()[1] + import apt_pkg as apt cache = apt_cache() @@ -613,6 +636,9 @@ def openstack_upgrade_available(package): import apt_pkg as apt src = config('openstack-origin') cur_vers = get_os_version_package(package) + if not cur_vers: + # The package has not been installed yet do not attempt upgrade + return False if "swift" in package: codename = get_os_codename_install_source(src) avail_vers = get_os_version_codename_swift(codename) @@ -1863,6 +1889,30 @@ def pausable_restart_on_change(restart_map, stopstart=False, return wrap +def ordered(orderme): + """Converts the provided dictionary into a collections.OrderedDict. + + The items in the returned OrderedDict will be inserted based on the + natural sort order of the keys. Nested dictionaries will also be sorted + in order to ensure fully predictable ordering. + + :param orderme: the dict to order + :return: collections.OrderedDict + :raises: ValueError: if `orderme` isn't a dict instance. + """ + if not isinstance(orderme, dict): + raise ValueError('argument must be a dict type') + + result = OrderedDict() + for k, v in sorted(six.iteritems(orderme), key=lambda x: x[0]): + if isinstance(v, dict): + result[k] = ordered(v) + else: + result[k] = v + + return result + + def config_flags_parser(config_flags): """Parses config flags string into dict. @@ -1874,15 +1924,13 @@ def config_flags_parser(config_flags): example, a string in the format of 'key1=value1, key2=value2' will return a dict of: - {'key1': 'value1', - 'key2': 'value2'}. + {'key1': 'value1', 'key2': 'value2'}. 2. A string in the above format, but supporting a comma-delimited list of values for the same key. For example, a string in the format of 'key1=value1, key2=value3,value4,value5' will return a dict of: - {'key1', 'value1', - 'key2', 'value2,value3,value4'} + {'key1': 'value1', 'key2': 'value2,value3,value4'} 3. A string containing a colon character (:) prior to an equal character (=) will be treated as yaml and parsed as such. This can be @@ -1902,7 +1950,7 @@ def config_flags_parser(config_flags): equals = config_flags.find('=') if colon > 0: if colon < equals or equals < 0: - return yaml.safe_load(config_flags) + return ordered(yaml.safe_load(config_flags)) if config_flags.find('==') >= 0: juju_log("config_flags is not in expected format (key=value)", @@ -1915,7 +1963,7 @@ def config_flags_parser(config_flags): # split on '='. split = config_flags.strip(' =').split('=') limit = len(split) - flags = {} + flags = OrderedDict() for i in range(0, limit - 1): current = split[i] next = split[i + 1] @@ -1982,3 +2030,84 @@ def token_cache_pkgs(source=None, release=None): if enable_memcache(source=source, release=release): packages.extend(['memcached', 'python-memcache']) return packages + + +def update_json_file(filename, items): + """Updates the json `filename` with a given dict. + :param filename: json filename (i.e.: /etc/glance/policy.json) + :param items: dict of items to update + """ + with open(filename) as fd: + policy = json.load(fd) + policy.update(items) + with open(filename, "w") as fd: + fd.write(json.dumps(policy, indent=4)) + + +@cached +def snap_install_requested(): + """ Determine if installing from snaps + + If openstack-origin is of the form snap:channel-series-release + and channel is in SNAPS_CHANNELS return True. + """ + origin = config('openstack-origin') or "" + if not origin.startswith('snap:'): + return False + + _src = origin[5:] + channel, series, release = _src.split('-') + if channel.lower() in SNAP_CHANNELS: + return True + return False + + +def get_snaps_install_info_from_origin(snaps, src, mode='classic'): + """Generate a dictionary of snap install information from origin + + @param snaps: List of snaps + @param src: String of openstack-origin or source of the form + snap:channel-series-track + @param mode: String classic, devmode or jailmode + @returns: Dictionary of snaps with channels and modes + """ + + if not src.startswith('snap:'): + juju_log("Snap source is not a snap origin", 'WARN') + return {} + + _src = src[5:] + _channel, _series, _release = _src.split('-') + channel = '--channel={}/{}'.format(_release, _channel) + + return {snap: {'channel': channel, 'mode': mode} + for snap in snaps} + + +def install_os_snaps(snaps, refresh=False): + """Install OpenStack snaps from channel and with mode + + @param snaps: Dictionary of snaps with channels and modes of the form: + {'snap_name': {'channel': 'snap_channel', + 'mode': 'snap_mode'}} + Where channel a snapstore channel and mode is --classic, --devmode or + --jailmode. + @param post_snap_install: Callback function to run after snaps have been + installed + """ + + def _ensure_flag(flag): + if flag.startswith('--'): + return flag + return '--{}'.format(flag) + + if refresh: + for snap in snaps.keys(): + snap_refresh(snap, + _ensure_flag(snaps[snap]['channel']), + _ensure_flag(snaps[snap]['mode'])) + else: + for snap in snaps.keys(): + snap_install(snap, + _ensure_flag(snaps[snap]['channel']), + _ensure_flag(snaps[snap]['mode'])) diff --git a/hooks/charmhelpers/contrib/storage/linux/bcache.py b/hooks/charmhelpers/contrib/storage/linux/bcache.py new file mode 100644 index 00000000..605991e1 --- /dev/null +++ b/hooks/charmhelpers/contrib/storage/linux/bcache.py @@ -0,0 +1,74 @@ +# Copyright 2017 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import json + +from charmhelpers.core.hookenv import log + +stats_intervals = ['stats_day', 'stats_five_minute', + 'stats_hour', 'stats_total'] + +SYSFS = '/sys' + + +class Bcache(object): + """Bcache behaviour + """ + + def __init__(self, cachepath): + self.cachepath = cachepath + + @classmethod + def fromdevice(cls, devname): + return cls('{}/block/{}/bcache'.format(SYSFS, devname)) + + def __str__(self): + return self.cachepath + + def get_stats(self, interval): + """Get cache stats + """ + intervaldir = 'stats_{}'.format(interval) + path = "{}/{}".format(self.cachepath, intervaldir) + out = dict() + for elem in os.listdir(path): + out[elem] = open('{}/{}'.format(path, elem)).read().strip() + return out + + +def get_bcache_fs(): + """Return all cache sets + """ + cachesetroot = "{}/fs/bcache".format(SYSFS) + try: + dirs = os.listdir(cachesetroot) + except OSError: + log("No bcache fs found") + return [] + cacheset = set([Bcache('{}/{}'.format(cachesetroot, d)) for d in dirs if not d.startswith('register')]) + return cacheset + + +def get_stats_action(cachespec, interval): + """Action for getting bcache statistics for a given cachespec. + Cachespec can either be a device name, eg. 'sdb', which will retrieve + cache stats for the given device, or 'global', which will retrieve stats + for all cachesets + """ + if cachespec == 'global': + caches = get_bcache_fs() + else: + caches = [Bcache.fromdevice(cachespec)] + res = dict((c.cachepath, c.get_stats(interval)) for c in caches) + return json.dumps(res, indent=4, separators=(',', ': ')) diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 1f0540a1..e5a01b1b 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -1372,7 +1372,7 @@ class CephConfContext(object): return {} conf = config_flags_parser(conf) - if type(conf) != dict: + if not isinstance(conf, dict): log("Provided config-flags is not a dictionary - ignoring", level=WARNING) return {} diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index e44e22bf..fe7fd8c0 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -202,6 +202,27 @@ def service_name(): return local_unit().split('/')[0] +def principal_unit(): + """Returns the principal unit of this unit, otherwise None""" + # Juju 2.2 and above provides JUJU_PRINCIPAL_UNIT + principal_unit = os.environ.get('JUJU_PRINCIPAL_UNIT', None) + # If it's empty, then this unit is the principal + if principal_unit == '': + return os.environ['JUJU_UNIT_NAME'] + elif principal_unit is not None: + return principal_unit + # For Juju 2.1 and below, let's try work out the principle unit by + # the various charms' metadata.yaml. + for reltype in relation_types(): + for rid in relation_ids(reltype): + for unit in related_units(rid): + md = _metadata_unit(unit) + subordinate = md.pop('subordinate', None) + if not subordinate: + return unit + return None + + @cached def remote_service_name(relid=None): """The remote service name for a given relation-id (or the current relation)""" @@ -478,6 +499,21 @@ def metadata(): return yaml.safe_load(md) +def _metadata_unit(unit): + """Given the name of a unit (e.g. apache2/0), get the unit charm's + metadata.yaml. Very similar to metadata() but allows us to inspect + other units. Unit needs to be co-located, such as a subordinate or + principal/primary. + + :returns: metadata.yaml as a python object. + + """ + basedir = os.sep.join(charm_dir().split(os.sep)[:-2]) + unitdir = 'unit-{}'.format(unit.replace(os.sep, '-')) + with open(os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')) as md: + return yaml.safe_load(md) + + @cached def relation_types(): """Get a list of relation types supported by this charm""" diff --git a/hooks/charmhelpers/fetch/snap.py b/hooks/charmhelpers/fetch/snap.py index 23c707b0..112a54c3 100644 --- a/hooks/charmhelpers/fetch/snap.py +++ b/hooks/charmhelpers/fetch/snap.py @@ -18,15 +18,23 @@ If writing reactive charms, use the snap layer: https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html """ import subprocess -from os import environ +import os from time import sleep from charmhelpers.core.hookenv import log __author__ = 'Joseph Borg ' -SNAP_NO_LOCK = 1 # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved). +# The return code for "couldn't acquire lock" in Snap +# (hopefully this will be improved). +SNAP_NO_LOCK = 1 SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks. SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. +SNAP_CHANNELS = [ + 'edge', + 'beta', + 'candidate', + 'stable', +] class CouldNotAcquireLockException(Exception): @@ -47,13 +55,17 @@ def _snap_exec(commands): while return_code is None or return_code == SNAP_NO_LOCK: try: - return_code = subprocess.check_call(['snap'] + commands, env=environ) + return_code = subprocess.check_call(['snap'] + commands, + env=os.environ) except subprocess.CalledProcessError as e: retry_count += + 1 if retry_count > SNAP_NO_LOCK_RETRY_COUNT: - raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT) + raise CouldNotAcquireLockException( + 'Could not aquire lock after {} attempts' + .format(SNAP_NO_LOCK_RETRY_COUNT)) return_code = e.returncode - log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN') + log('Snap failed to acquire lock, trying again in {} seconds.' + .format(SNAP_NO_LOCK_RETRY_DELAY, level='WARN')) sleep(SNAP_NO_LOCK_RETRY_DELAY) return return_code diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py index 57b5fb61..545348ff 100644 --- a/hooks/charmhelpers/fetch/ubuntu.py +++ b/hooks/charmhelpers/fetch/ubuntu.py @@ -139,7 +139,7 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-updates/ocata': 'xenial-updates/ocata', 'ocata/proposed': 'xenial-proposed/ocata', 'xenial-ocata/proposed': 'xenial-proposed/ocata', - 'xenial-ocata/newton': 'xenial-proposed/ocata', + 'xenial-proposed/ocata': 'xenial-proposed/ocata', # Pike 'pike': 'xenial-updates/pike', 'xenial-pike': 'xenial-updates/pike', @@ -147,7 +147,7 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-updates/pike': 'xenial-updates/pike', 'pike/proposed': 'xenial-proposed/pike', 'xenial-pike/proposed': 'xenial-proposed/pike', - 'xenial-pike/newton': 'xenial-proposed/pike', + 'xenial-proposed/pike': 'xenial-proposed/pike', # Queens 'queens': 'xenial-updates/queens', 'xenial-queens': 'xenial-updates/queens', @@ -155,13 +155,13 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-updates/queens': 'xenial-updates/queens', 'queens/proposed': 'xenial-proposed/queens', 'xenial-queens/proposed': 'xenial-proposed/queens', - 'xenial-queens/newton': 'xenial-proposed/queens', + 'xenial-proposed/queens': 'xenial-proposed/queens', } APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries. -CMD_RETRY_COUNT = 30 # Retry a failing fatal command X times. +CMD_RETRY_COUNT = 3 # Retry a failing fatal command X times. def filter_installed_packages(packages): @@ -364,6 +364,7 @@ def add_source(source, key=None, fail_invalid=False): (r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging), (r"^cloud:(.*)-(.*)$", _add_cloud_distro_check), (r"^cloud:(.*)$", _add_cloud_pocket), + (r"^snap:.*-(.*)-(.*)$", _add_cloud_distro_check), ]) if source is None: source = '' diff --git a/tests/charmhelpers/contrib/network/ip.py b/tests/charmhelpers/contrib/network/ip.py index 15f65960..d7e6debf 100644 --- a/tests/charmhelpers/contrib/network/ip.py +++ b/tests/charmhelpers/contrib/network/ip.py @@ -243,11 +243,11 @@ def is_ipv6_disabled(): try: result = subprocess.check_output( ['sysctl', 'net.ipv6.conf.all.disable_ipv6'], - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + universal_newlines=True) except subprocess.CalledProcessError: return True - if six.PY3: - result = result.decode('UTF-8') + return "net.ipv6.conf.all.disable_ipv6 = 1" in result diff --git a/tests/charmhelpers/contrib/openstack/utils.py b/tests/charmhelpers/contrib/openstack/utils.py index 1eaab95f..5b88b33a 100644 --- a/tests/charmhelpers/contrib/openstack/utils.py +++ b/tests/charmhelpers/contrib/openstack/utils.py @@ -51,6 +51,7 @@ from charmhelpers.core.hookenv import ( status_set, hook_name, application_version_set, + cached, ) from charmhelpers.core.strutils import BasicStringComparator @@ -90,6 +91,13 @@ from charmhelpers.fetch import ( GPGKeyError, get_upstream_version ) + +from charmhelpers.fetch.snap import ( + snap_install, + snap_refresh, + SNAP_CHANNELS, +) + from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device from charmhelpers.contrib.openstack.exceptions import OSContextError @@ -327,8 +335,10 @@ def get_os_codename_install_source(src): return ca_rel # Best guess match based on deb string provided - if src.startswith('deb') or src.startswith('ppa'): - for k, v in six.iteritems(OPENSTACK_CODENAMES): + if (src.startswith('deb') or + src.startswith('ppa') or + src.startswith('snap')): + for v in OPENSTACK_CODENAMES.values(): if v in src: return v @@ -397,6 +407,19 @@ def get_swift_codename(version): def get_os_codename_package(package, fatal=True): '''Derive OpenStack release codename from an installed package.''' + + if snap_install_requested(): + cmd = ['snap', 'list', package] + try: + out = subprocess.check_output(cmd) + except subprocess.CalledProcessError as e: + return None + lines = out.split('\n') + for line in lines: + if package in line: + # Second item in list is Version + return line.split()[1] + import apt_pkg as apt cache = apt_cache() @@ -613,6 +636,9 @@ def openstack_upgrade_available(package): import apt_pkg as apt src = config('openstack-origin') cur_vers = get_os_version_package(package) + if not cur_vers: + # The package has not been installed yet do not attempt upgrade + return False if "swift" in package: codename = get_os_codename_install_source(src) avail_vers = get_os_version_codename_swift(codename) @@ -1863,6 +1889,30 @@ def pausable_restart_on_change(restart_map, stopstart=False, return wrap +def ordered(orderme): + """Converts the provided dictionary into a collections.OrderedDict. + + The items in the returned OrderedDict will be inserted based on the + natural sort order of the keys. Nested dictionaries will also be sorted + in order to ensure fully predictable ordering. + + :param orderme: the dict to order + :return: collections.OrderedDict + :raises: ValueError: if `orderme` isn't a dict instance. + """ + if not isinstance(orderme, dict): + raise ValueError('argument must be a dict type') + + result = OrderedDict() + for k, v in sorted(six.iteritems(orderme), key=lambda x: x[0]): + if isinstance(v, dict): + result[k] = ordered(v) + else: + result[k] = v + + return result + + def config_flags_parser(config_flags): """Parses config flags string into dict. @@ -1874,15 +1924,13 @@ def config_flags_parser(config_flags): example, a string in the format of 'key1=value1, key2=value2' will return a dict of: - {'key1': 'value1', - 'key2': 'value2'}. + {'key1': 'value1', 'key2': 'value2'}. 2. A string in the above format, but supporting a comma-delimited list of values for the same key. For example, a string in the format of 'key1=value1, key2=value3,value4,value5' will return a dict of: - {'key1', 'value1', - 'key2', 'value2,value3,value4'} + {'key1': 'value1', 'key2': 'value2,value3,value4'} 3. A string containing a colon character (:) prior to an equal character (=) will be treated as yaml and parsed as such. This can be @@ -1902,7 +1950,7 @@ def config_flags_parser(config_flags): equals = config_flags.find('=') if colon > 0: if colon < equals or equals < 0: - return yaml.safe_load(config_flags) + return ordered(yaml.safe_load(config_flags)) if config_flags.find('==') >= 0: juju_log("config_flags is not in expected format (key=value)", @@ -1915,7 +1963,7 @@ def config_flags_parser(config_flags): # split on '='. split = config_flags.strip(' =').split('=') limit = len(split) - flags = {} + flags = OrderedDict() for i in range(0, limit - 1): current = split[i] next = split[i + 1] @@ -1982,3 +2030,84 @@ def token_cache_pkgs(source=None, release=None): if enable_memcache(source=source, release=release): packages.extend(['memcached', 'python-memcache']) return packages + + +def update_json_file(filename, items): + """Updates the json `filename` with a given dict. + :param filename: json filename (i.e.: /etc/glance/policy.json) + :param items: dict of items to update + """ + with open(filename) as fd: + policy = json.load(fd) + policy.update(items) + with open(filename, "w") as fd: + fd.write(json.dumps(policy, indent=4)) + + +@cached +def snap_install_requested(): + """ Determine if installing from snaps + + If openstack-origin is of the form snap:channel-series-release + and channel is in SNAPS_CHANNELS return True. + """ + origin = config('openstack-origin') or "" + if not origin.startswith('snap:'): + return False + + _src = origin[5:] + channel, series, release = _src.split('-') + if channel.lower() in SNAP_CHANNELS: + return True + return False + + +def get_snaps_install_info_from_origin(snaps, src, mode='classic'): + """Generate a dictionary of snap install information from origin + + @param snaps: List of snaps + @param src: String of openstack-origin or source of the form + snap:channel-series-track + @param mode: String classic, devmode or jailmode + @returns: Dictionary of snaps with channels and modes + """ + + if not src.startswith('snap:'): + juju_log("Snap source is not a snap origin", 'WARN') + return {} + + _src = src[5:] + _channel, _series, _release = _src.split('-') + channel = '--channel={}/{}'.format(_release, _channel) + + return {snap: {'channel': channel, 'mode': mode} + for snap in snaps} + + +def install_os_snaps(snaps, refresh=False): + """Install OpenStack snaps from channel and with mode + + @param snaps: Dictionary of snaps with channels and modes of the form: + {'snap_name': {'channel': 'snap_channel', + 'mode': 'snap_mode'}} + Where channel a snapstore channel and mode is --classic, --devmode or + --jailmode. + @param post_snap_install: Callback function to run after snaps have been + installed + """ + + def _ensure_flag(flag): + if flag.startswith('--'): + return flag + return '--{}'.format(flag) + + if refresh: + for snap in snaps.keys(): + snap_refresh(snap, + _ensure_flag(snaps[snap]['channel']), + _ensure_flag(snaps[snap]['mode'])) + else: + for snap in snaps.keys(): + snap_install(snap, + _ensure_flag(snaps[snap]['channel']), + _ensure_flag(snaps[snap]['mode'])) diff --git a/tests/charmhelpers/contrib/storage/linux/bcache.py b/tests/charmhelpers/contrib/storage/linux/bcache.py new file mode 100644 index 00000000..605991e1 --- /dev/null +++ b/tests/charmhelpers/contrib/storage/linux/bcache.py @@ -0,0 +1,74 @@ +# Copyright 2017 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import json + +from charmhelpers.core.hookenv import log + +stats_intervals = ['stats_day', 'stats_five_minute', + 'stats_hour', 'stats_total'] + +SYSFS = '/sys' + + +class Bcache(object): + """Bcache behaviour + """ + + def __init__(self, cachepath): + self.cachepath = cachepath + + @classmethod + def fromdevice(cls, devname): + return cls('{}/block/{}/bcache'.format(SYSFS, devname)) + + def __str__(self): + return self.cachepath + + def get_stats(self, interval): + """Get cache stats + """ + intervaldir = 'stats_{}'.format(interval) + path = "{}/{}".format(self.cachepath, intervaldir) + out = dict() + for elem in os.listdir(path): + out[elem] = open('{}/{}'.format(path, elem)).read().strip() + return out + + +def get_bcache_fs(): + """Return all cache sets + """ + cachesetroot = "{}/fs/bcache".format(SYSFS) + try: + dirs = os.listdir(cachesetroot) + except OSError: + log("No bcache fs found") + return [] + cacheset = set([Bcache('{}/{}'.format(cachesetroot, d)) for d in dirs if not d.startswith('register')]) + return cacheset + + +def get_stats_action(cachespec, interval): + """Action for getting bcache statistics for a given cachespec. + Cachespec can either be a device name, eg. 'sdb', which will retrieve + cache stats for the given device, or 'global', which will retrieve stats + for all cachesets + """ + if cachespec == 'global': + caches = get_bcache_fs() + else: + caches = [Bcache.fromdevice(cachespec)] + res = dict((c.cachepath, c.get_stats(interval)) for c in caches) + return json.dumps(res, indent=4, separators=(',', ': ')) diff --git a/tests/charmhelpers/contrib/storage/linux/ceph.py b/tests/charmhelpers/contrib/storage/linux/ceph.py index 1f0540a1..e5a01b1b 100644 --- a/tests/charmhelpers/contrib/storage/linux/ceph.py +++ b/tests/charmhelpers/contrib/storage/linux/ceph.py @@ -1372,7 +1372,7 @@ class CephConfContext(object): return {} conf = config_flags_parser(conf) - if type(conf) != dict: + if not isinstance(conf, dict): log("Provided config-flags is not a dictionary - ignoring", level=WARNING) return {} diff --git a/tests/charmhelpers/core/hookenv.py b/tests/charmhelpers/core/hookenv.py index e44e22bf..fe7fd8c0 100644 --- a/tests/charmhelpers/core/hookenv.py +++ b/tests/charmhelpers/core/hookenv.py @@ -202,6 +202,27 @@ def service_name(): return local_unit().split('/')[0] +def principal_unit(): + """Returns the principal unit of this unit, otherwise None""" + # Juju 2.2 and above provides JUJU_PRINCIPAL_UNIT + principal_unit = os.environ.get('JUJU_PRINCIPAL_UNIT', None) + # If it's empty, then this unit is the principal + if principal_unit == '': + return os.environ['JUJU_UNIT_NAME'] + elif principal_unit is not None: + return principal_unit + # For Juju 2.1 and below, let's try work out the principle unit by + # the various charms' metadata.yaml. + for reltype in relation_types(): + for rid in relation_ids(reltype): + for unit in related_units(rid): + md = _metadata_unit(unit) + subordinate = md.pop('subordinate', None) + if not subordinate: + return unit + return None + + @cached def remote_service_name(relid=None): """The remote service name for a given relation-id (or the current relation)""" @@ -478,6 +499,21 @@ def metadata(): return yaml.safe_load(md) +def _metadata_unit(unit): + """Given the name of a unit (e.g. apache2/0), get the unit charm's + metadata.yaml. Very similar to metadata() but allows us to inspect + other units. Unit needs to be co-located, such as a subordinate or + principal/primary. + + :returns: metadata.yaml as a python object. + + """ + basedir = os.sep.join(charm_dir().split(os.sep)[:-2]) + unitdir = 'unit-{}'.format(unit.replace(os.sep, '-')) + with open(os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')) as md: + return yaml.safe_load(md) + + @cached def relation_types(): """Get a list of relation types supported by this charm""" diff --git a/tests/charmhelpers/fetch/snap.py b/tests/charmhelpers/fetch/snap.py index 23c707b0..112a54c3 100644 --- a/tests/charmhelpers/fetch/snap.py +++ b/tests/charmhelpers/fetch/snap.py @@ -18,15 +18,23 @@ If writing reactive charms, use the snap layer: https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html """ import subprocess -from os import environ +import os from time import sleep from charmhelpers.core.hookenv import log __author__ = 'Joseph Borg ' -SNAP_NO_LOCK = 1 # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved). +# The return code for "couldn't acquire lock" in Snap +# (hopefully this will be improved). +SNAP_NO_LOCK = 1 SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks. SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. +SNAP_CHANNELS = [ + 'edge', + 'beta', + 'candidate', + 'stable', +] class CouldNotAcquireLockException(Exception): @@ -47,13 +55,17 @@ def _snap_exec(commands): while return_code is None or return_code == SNAP_NO_LOCK: try: - return_code = subprocess.check_call(['snap'] + commands, env=environ) + return_code = subprocess.check_call(['snap'] + commands, + env=os.environ) except subprocess.CalledProcessError as e: retry_count += + 1 if retry_count > SNAP_NO_LOCK_RETRY_COUNT: - raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT) + raise CouldNotAcquireLockException( + 'Could not aquire lock after {} attempts' + .format(SNAP_NO_LOCK_RETRY_COUNT)) return_code = e.returncode - log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN') + log('Snap failed to acquire lock, trying again in {} seconds.' + .format(SNAP_NO_LOCK_RETRY_DELAY, level='WARN')) sleep(SNAP_NO_LOCK_RETRY_DELAY) return return_code diff --git a/tests/charmhelpers/fetch/ubuntu.py b/tests/charmhelpers/fetch/ubuntu.py index 57b5fb61..545348ff 100644 --- a/tests/charmhelpers/fetch/ubuntu.py +++ b/tests/charmhelpers/fetch/ubuntu.py @@ -139,7 +139,7 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-updates/ocata': 'xenial-updates/ocata', 'ocata/proposed': 'xenial-proposed/ocata', 'xenial-ocata/proposed': 'xenial-proposed/ocata', - 'xenial-ocata/newton': 'xenial-proposed/ocata', + 'xenial-proposed/ocata': 'xenial-proposed/ocata', # Pike 'pike': 'xenial-updates/pike', 'xenial-pike': 'xenial-updates/pike', @@ -147,7 +147,7 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-updates/pike': 'xenial-updates/pike', 'pike/proposed': 'xenial-proposed/pike', 'xenial-pike/proposed': 'xenial-proposed/pike', - 'xenial-pike/newton': 'xenial-proposed/pike', + 'xenial-proposed/pike': 'xenial-proposed/pike', # Queens 'queens': 'xenial-updates/queens', 'xenial-queens': 'xenial-updates/queens', @@ -155,13 +155,13 @@ CLOUD_ARCHIVE_POCKETS = { 'xenial-updates/queens': 'xenial-updates/queens', 'queens/proposed': 'xenial-proposed/queens', 'xenial-queens/proposed': 'xenial-proposed/queens', - 'xenial-queens/newton': 'xenial-proposed/queens', + 'xenial-proposed/queens': 'xenial-proposed/queens', } APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries. -CMD_RETRY_COUNT = 30 # Retry a failing fatal command X times. +CMD_RETRY_COUNT = 3 # Retry a failing fatal command X times. def filter_installed_packages(packages): @@ -364,6 +364,7 @@ def add_source(source, key=None, fail_invalid=False): (r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging), (r"^cloud:(.*)-(.*)$", _add_cloud_distro_check), (r"^cloud:(.*)$", _add_cloud_pocket), + (r"^snap:.*-(.*)-(.*)$", _add_cloud_distro_check), ]) if source is None: source = ''