diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index aa0b515..92325a9 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -41,10 +41,11 @@ from charmhelpers.core.hookenv import ( relation_get, config as config_get, INFO, - ERROR, + DEBUG, WARNING, unit_get, - is_leader as juju_is_leader + is_leader as juju_is_leader, + status_set, ) from charmhelpers.core.decorators import ( retry_on_exception, @@ -60,6 +61,10 @@ class HAIncompleteConfig(Exception): pass +class HAIncorrectConfig(Exception): + pass + + class CRMResourceNotFound(Exception): pass @@ -274,27 +279,71 @@ def get_hacluster_config(exclude_keys=None): Obtains all relevant configuration from charm configuration required for initiating a relation to hacluster: - ha-bindiface, ha-mcastport, vip + ha-bindiface, ha-mcastport, vip, os-internal-hostname, + os-admin-hostname, os-public-hostname param: exclude_keys: list of setting key(s) to be excluded. returns: dict: A dict containing settings keyed by setting name. - raises: HAIncompleteConfig if settings are missing. + raises: HAIncompleteConfig if settings are missing or incorrect. ''' - settings = ['ha-bindiface', 'ha-mcastport', 'vip'] + settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'os-internal-hostname', + 'os-admin-hostname', 'os-public-hostname'] conf = {} for setting in settings: if exclude_keys and setting in exclude_keys: continue conf[setting] = config_get(setting) - missing = [] - [missing.append(s) for s, v in six.iteritems(conf) if v is None] - if missing: - log('Insufficient config data to configure hacluster.', level=ERROR) - raise HAIncompleteConfig + + if not valid_hacluster_config(): + raise HAIncorrectConfig('Insufficient or incorrect config data to ' + 'configure hacluster.') return conf +def valid_hacluster_config(): + ''' + Check that either vip or dns-ha is set. If dns-ha then one of os-*-hostname + must be set. + + Note: ha-bindiface and ha-macastport both have defaults and will always + be set. We only care that either vip or dns-ha is set. + + :returns: boolean: valid config returns true. + raises: HAIncompatibileConfig if settings conflict. + raises: HAIncompleteConfig if settings are missing. + ''' + vip = config_get('vip') + dns = config_get('dns-ha') + if not(bool(vip) ^ bool(dns)): + msg = ('HA: Either vip or dns-ha must be set but not both in order to ' + 'use high availability') + status_set('blocked', msg) + raise HAIncorrectConfig(msg) + + # If dns-ha then one of os-*-hostname must be set + if dns: + dns_settings = ['os-internal-hostname', 'os-admin-hostname', + 'os-public-hostname'] + # At this point it is unknown if one or all of the possible + # network spaces are in HA. Validate at least one is set which is + # the minimum required. + for setting in dns_settings: + if config_get(setting): + log('DNS HA: At least one hostname is set {}: {}' + ''.format(setting, config_get(setting)), + level=DEBUG) + return True + + msg = ('DNS HA: At least one os-*-hostname(s) must be set to use ' + 'DNS HA') + status_set('blocked', msg) + raise HAIncompleteConfig(msg) + + log('VIP HA: VIP is set {}'.format(vip), level=DEBUG) + return True + + def canonical_url(configs, vip_setting='vip'): ''' Returns the correct HTTP URL to this host given the state of HTTPS diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 6bba07b..99d78f2 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -405,10 +405,10 @@ def is_ip(address): Returns True if address is a valid IP address. """ try: - # Test to see if already an IPv4 address - socket.inet_aton(address) + # Test to see if already an IPv4/IPv6 address + address = netaddr.IPAddress(address) return True - except socket.error: + except netaddr.AddrFormatError: return False diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index d21c9c7..6b917d0 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -43,9 +43,6 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.openstack = openstack self.source = source self.stable = stable - # Note(coreycb): this needs to be changed when new next branches come - # out. - self.current_next = "trusty" def get_logger(self, name="deployment-logger", level=logging.DEBUG): """Get a logger object that will log to stdout.""" @@ -72,38 +69,34 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.log.info('OpenStackAmuletDeployment: determine branch locations') - # Charms outside the lp:~openstack-charmers namespace - base_charms = ['mysql', 'mongodb', 'nrpe'] - - # Force these charms to current series even when using an older series. - # ie. Use trusty/nrpe even when series is precise, as the P charm - # does not possess the necessary external master config and hooks. - force_series_current = ['nrpe'] - - if self.series in ['precise', 'trusty']: - base_series = self.series - else: - base_series = self.current_next + # Charms outside the ~openstack-charmers + base_charms = { + 'mysql': ['precise', 'trusty'], + 'mongodb': ['precise', 'trusty'], + 'nrpe': ['precise', 'trusty'], + } for svc in other_services: - if svc['name'] in force_series_current: - base_series = self.current_next # If a location has been explicitly set, use it if svc.get('location'): continue - if self.stable: - temp = 'lp:charms/{}/{}' - svc['location'] = temp.format(base_series, - svc['name']) + if svc['name'] in base_charms: + # NOTE: not all charms have support for all series we + # want/need to test against, so fix to most recent + # that each base charm supports + target_series = self.series + if self.series not in base_charms[svc['name']]: + target_series = base_charms[svc['name']][-1] + svc['location'] = 'cs:{}/{}'.format(target_series, + svc['name']) + elif self.stable: + svc['location'] = 'cs:{}/{}'.format(self.series, + svc['name']) else: - if svc['name'] in base_charms: - temp = 'lp:charms/{}/{}' - svc['location'] = temp.format(base_series, - svc['name']) - else: - temp = 'lp:~openstack-charmers/charms/{}/{}/next' - svc['location'] = temp.format(self.current_next, - svc['name']) + svc['location'] = 'cs:~openstack-charmers-next/{}/{}'.format( + self.series, + svc['name'] + ) return other_services diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index c07b33d..5faa7ed 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -23,7 +23,6 @@ from base64 import b64decode from subprocess import check_call, CalledProcessError import six -import yaml from charmhelpers.fetch import ( apt_install, @@ -50,6 +49,7 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.sysctl import create as sysctl_create from charmhelpers.core.strutils import bool_from_string +from charmhelpers.contrib.openstack.exceptions import OSContextError from charmhelpers.core.host import ( get_bond_master, @@ -88,7 +88,10 @@ from charmhelpers.contrib.network.ip import ( is_address_in_network, is_bridge_member, ) -from charmhelpers.contrib.openstack.utils import get_host_ip +from charmhelpers.contrib.openstack.utils import ( + config_flags_parser, + get_host_ip, +) from charmhelpers.core.unitdata import kv try: @@ -101,10 +104,6 @@ CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' ADDRESS_TYPES = ['admin', 'internal', 'public'] -class OSContextError(Exception): - pass - - def ensure_packages(packages): """Install but do not upgrade required plugin packages.""" required = filter_installed_packages(packages) @@ -125,83 +124,6 @@ def context_complete(ctxt): return True -def config_flags_parser(config_flags): - """Parses config flags string into dict. - - This parsing method supports a few different formats for the config - flag values to be parsed: - - 1. A string in the simple format of key=value pairs, with the possibility - of specifying multiple key value pairs within the same string. For - example, a string in the format of 'key1=value1, key2=value2' will - return a dict of: - - {'key1': 'value1', - 'key2': 'value2'}. - - 2. A string in the above format, but supporting a comma-delimited list - of values for the same key. For example, a string in the format of - 'key1=value1, key2=value3,value4,value5' will return a dict of: - - {'key1', 'value1', - 'key2', 'value2,value3,value4'} - - 3. A string containing a colon character (:) prior to an equal - character (=) will be treated as yaml and parsed as such. This can be - used to specify more complex key value pairs. For example, - a string in the format of 'key1: subkey1=value1, subkey2=value2' will - return a dict of: - - {'key1', 'subkey1=value1, subkey2=value2'} - - The provided config_flags string may be a list of comma-separated values - which themselves may be comma-separated list of values. - """ - # If we find a colon before an equals sign then treat it as yaml. - # Note: limit it to finding the colon first since this indicates assignment - # for inline yaml. - colon = config_flags.find(':') - equals = config_flags.find('=') - if colon > 0: - if colon < equals or equals < 0: - return yaml.safe_load(config_flags) - - if config_flags.find('==') >= 0: - log("config_flags is not in expected format (key=value)", level=ERROR) - raise OSContextError - - # strip the following from each value. - post_strippers = ' ,' - # we strip any leading/trailing '=' or ' ' from the string then - # split on '='. - split = config_flags.strip(' =').split('=') - limit = len(split) - flags = {} - for i in range(0, limit - 1): - current = split[i] - next = split[i + 1] - vindex = next.rfind(',') - if (i == limit - 2) or (vindex < 0): - value = next - else: - value = next[:vindex] - - if i == 0: - key = current - else: - # if this not the first entry, expect an embedded key. - index = current.rfind(',') - if index < 0: - log("Invalid config value(s) at index %s" % (i), level=ERROR) - raise OSContextError - key = current[index + 1:] - - # Add to collection. - flags[key.strip(post_strippers)] = value.rstrip(post_strippers) - - return flags - - class OSContextGenerator(object): """Base class for all context generators.""" interfaces = [] diff --git a/hooks/charmhelpers/contrib/openstack/exceptions.py b/hooks/charmhelpers/contrib/openstack/exceptions.py new file mode 100644 index 0000000..ea4eb68 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/exceptions.py @@ -0,0 +1,6 @@ +class OSContextError(Exception): + """Raised when an error occurs during context generation. + + This exception is principally used in contrib.openstack.context + """ + pass diff --git a/hooks/charmhelpers/contrib/openstack/ha/__init__.py b/hooks/charmhelpers/contrib/openstack/ha/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hooks/charmhelpers/contrib/openstack/ha/utils.py b/hooks/charmhelpers/contrib/openstack/ha/utils.py new file mode 100644 index 0000000..3406423 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/ha/utils.py @@ -0,0 +1,111 @@ +# Copyright 2014-2016 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +# +# Copyright 2016 Canonical Ltd. +# +# Authors: +# Openstack Charmers < +# + +""" +Helpers for high availability. +""" + +import re + +from charmhelpers.core.hookenv import ( + log, + relation_set, + charm_name, + config, + status_set, + DEBUG, +) + +from charmhelpers.contrib.openstack.ip import ( + resolve_address, +) + + +class DNSHAException(Exception): + """Raised when an error occurs setting up DNS HA + """ + + pass + + +def update_dns_ha_resource_params(resources, resource_params, + relation_id=None, + crm_ocf='ocf:maas:dns'): + """ Check for os-*-hostname settings and update resource dictionaries for + the HA relation. + + @param resources: Pointer to dictionary of resources. + Usually instantiated in ha_joined(). + @param resource_params: Pointer to dictionary of resource parameters. + Usually instantiated in ha_joined() + @param relation_id: Relation ID of the ha relation + @param crm_ocf: Corosync Open Cluster Framework resource agent to use for + DNS HA + """ + + settings = ['os-admin-hostname', 'os-internal-hostname', + 'os-public-hostname'] + + # Check which DNS settings are set and update dictionaries + hostname_group = [] + for setting in settings: + hostname = config(setting) + if hostname is None: + log('DNS HA: Hostname setting {} is None. Ignoring.' + ''.format(setting), + DEBUG) + continue + m = re.search('os-(.+?)-hostname', setting) + if m: + networkspace = m.group(1) + else: + msg = ('Unexpected DNS hostname setting: {}. ' + 'Cannot determine network space name' + ''.format(setting)) + status_set('blocked', msg) + raise DNSHAException(msg) + + hostname_key = 'res_{}_{}_hostname'.format(charm_name(), networkspace) + if hostname_key in hostname_group: + log('DNS HA: Resource {}: {} already exists in ' + 'hostname group - skipping'.format(hostname_key, hostname), + DEBUG) + continue + + hostname_group.append(hostname_key) + resources[hostname_key] = crm_ocf + resource_params[hostname_key] = ( + 'params fqdn="{}" ip_address="{}" ' + ''.format(hostname, resolve_address(endpoint_type=networkspace, + override=False))) + + if len(hostname_group) >= 1: + log('DNS HA: Hostname group is set with {} as members. ' + 'Informing the ha relation'.format(' '.join(hostname_group)), + DEBUG) + relation_set(relation_id=relation_id, groups={ + 'grp_{}_hostnames'.format(charm_name()): ' '.join(hostname_group)}) + else: + msg = 'DNS HA: Hostname group has no members.' + status_set('blocked', msg) + raise DNSHAException(msg) diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py index 532a1dc..7875b99 100644 --- a/hooks/charmhelpers/contrib/openstack/ip.py +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -109,7 +109,7 @@ def _get_address_override(endpoint_type=PUBLIC): return addr_override.format(service_name=service_name()) -def resolve_address(endpoint_type=PUBLIC): +def resolve_address(endpoint_type=PUBLIC, override=True): """Return unit address depending on net config. If unit is clustered with vip(s) and has net splits defined, return vip on @@ -119,10 +119,13 @@ def resolve_address(endpoint_type=PUBLIC): split if one is configured, or a Juju 2.0 extra-binding has been used. :param endpoint_type: Network endpoing type + :param override: Accept hostname overrides or not """ - resolved_address = _get_address_override(endpoint_type) - if resolved_address: - return resolved_address + resolved_address = None + if override: + resolved_address = _get_address_override(endpoint_type) + if resolved_address: + return resolved_address vips = config('vip') if vips: diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index e64a106..53e5842 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -25,6 +25,7 @@ import sys import re import itertools import functools +import shutil import six import tempfile @@ -46,9 +47,11 @@ from charmhelpers.core.hookenv import ( charm_dir, DEBUG, INFO, + ERROR, related_units, relation_ids, relation_set, + service_name, status_set, hook_name ) @@ -82,6 +85,7 @@ from charmhelpers.core.host import ( from charmhelpers.fetch import apt_install, apt_cache, install_remote from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device +from charmhelpers.contrib.openstack.exceptions import OSContextError CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu" CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA' @@ -100,6 +104,8 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('vivid', 'kilo'), ('wily', 'liberty'), ('xenial', 'mitaka'), + ('yakkety', 'newton'), + ('zebra', 'ocata'), # TODO: upload with real Z name ]) @@ -114,6 +120,8 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2015.1', 'kilo'), ('2015.2', 'liberty'), ('2016.1', 'mitaka'), + ('2016.2', 'newton'), + ('2017.1', 'ocata'), ]) # The ugly duckling - must list releases oldest to newest @@ -138,50 +146,89 @@ SWIFT_CODENAMES = OrderedDict([ ['2.3.0', '2.4.0', '2.5.0']), ('mitaka', ['2.5.0', '2.6.0', '2.7.0']), + ('newton', + ['2.8.0']), ]) # >= Liberty version->codename mapping PACKAGE_CODENAMES = { 'nova-common': OrderedDict([ - ('12.0', 'liberty'), - ('13.0', 'mitaka'), + ('12', 'liberty'), + ('13', 'mitaka'), + ('14', 'newton'), + ('15', 'ocata'), ]), 'neutron-common': OrderedDict([ - ('7.0', 'liberty'), - ('8.0', 'mitaka'), - ('8.1', 'mitaka'), + ('7', 'liberty'), + ('8', 'mitaka'), + ('9', 'newton'), + ('10', 'ocata'), ]), 'cinder-common': OrderedDict([ - ('7.0', 'liberty'), - ('8.0', 'mitaka'), + ('7', 'liberty'), + ('8', 'mitaka'), + ('9', 'newton'), + ('10', 'ocata'), ]), 'keystone': OrderedDict([ - ('8.0', 'liberty'), - ('8.1', 'liberty'), - ('9.0', 'mitaka'), + ('8', 'liberty'), + ('9', 'mitaka'), + ('10', 'newton'), + ('11', 'ocata'), ]), 'horizon-common': OrderedDict([ - ('8.0', 'liberty'), - ('9.0', 'mitaka'), + ('8', 'liberty'), + ('9', 'mitaka'), + ('10', 'newton'), + ('11', 'ocata'), ]), 'ceilometer-common': OrderedDict([ - ('5.0', 'liberty'), - ('6.0', 'mitaka'), + ('5', 'liberty'), + ('6', 'mitaka'), + ('7', 'newton'), + ('8', 'ocata'), ]), 'heat-common': OrderedDict([ - ('5.0', 'liberty'), - ('6.0', 'mitaka'), + ('5', 'liberty'), + ('6', 'mitaka'), + ('7', 'newton'), + ('8', 'ocata'), ]), 'glance-common': OrderedDict([ - ('11.0', 'liberty'), - ('12.0', 'mitaka'), + ('11', 'liberty'), + ('12', 'mitaka'), + ('13', 'newton'), + ('14', 'ocata'), ]), 'openstack-dashboard': OrderedDict([ - ('8.0', 'liberty'), - ('9.0', 'mitaka'), + ('8', 'liberty'), + ('9', 'mitaka'), + ('10', 'newton'), + ('11', 'ocata'), ]), } +GIT_DEFAULT_REPOS = { + 'requirements': 'git://github.com/openstack/requirements', + 'cinder': 'git://github.com/openstack/cinder', + 'glance': 'git://github.com/openstack/glance', + 'horizon': 'git://github.com/openstack/horizon', + 'keystone': 'git://github.com/openstack/keystone', + 'neutron': 'git://github.com/openstack/neutron', + 'neutron-fwaas': 'git://github.com/openstack/neutron-fwaas', + 'neutron-lbaas': 'git://github.com/openstack/neutron-lbaas', + 'neutron-vpnaas': 'git://github.com/openstack/neutron-vpnaas', + 'nova': 'git://github.com/openstack/nova', +} + +GIT_DEFAULT_BRANCHES = { + 'icehouse': 'icehouse-eol', + 'kilo': 'stable/kilo', + 'liberty': 'stable/liberty', + 'mitaka': 'stable/mitaka', + 'master': 'master', +} + DEFAULT_LOOPBACK_SIZE = '5G' @@ -254,6 +301,7 @@ def get_os_version_codename_swift(codename): def get_swift_codename(version): '''Determine OpenStack codename that corresponds to swift version.''' codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v] + if len(codenames) > 1: # If more than one release codename contains this version we determine # the actual codename based on the highest available install source. @@ -265,6 +313,16 @@ def get_swift_codename(version): return codename elif len(codenames) == 1: return codenames[0] + + # NOTE: fallback - attempt to match with just major.minor version + match = re.match('^(\d+)\.(\d+)', version) + if match: + major_minor_version = match.group(0) + for codename, versions in six.iteritems(SWIFT_CODENAMES): + for release_version in versions: + if release_version.startswith(major_minor_version): + return codename + return None @@ -303,10 +361,13 @@ def get_os_codename_package(package, fatal=True): if match: vers = match.group(0) + # Generate a major version number for newer semantic + # versions of openstack projects + major_vers = vers.split('.')[0] # >= Liberty independent project versions if (package in PACKAGE_CODENAMES and - vers in PACKAGE_CODENAMES[package]): - return PACKAGE_CODENAMES[package][vers] + major_vers in PACKAGE_CODENAMES[package]): + return PACKAGE_CODENAMES[package][major_vers] else: # < Liberty co-ordinated project versions try: @@ -466,6 +527,9 @@ def configure_installation_source(rel): 'mitaka': 'trusty-updates/mitaka', 'mitaka/updates': 'trusty-updates/mitaka', 'mitaka/proposed': 'trusty-proposed/mitaka', + 'newton': 'xenial-updates/newton', + 'newton/updates': 'xenial-updates/newton', + 'newton/proposed': 'xenial-proposed/newton', } try: @@ -661,6 +725,53 @@ def git_install_requested(): requirements_dir = None +def git_default_repos(projects): + """ + Returns default repos if a default openstack-origin-git value is specified. + """ + service = service_name() + + for default, branch in GIT_DEFAULT_BRANCHES.iteritems(): + if projects == default: + + # add the requirements repo first + repo = { + 'name': 'requirements', + 'repository': GIT_DEFAULT_REPOS['requirements'], + 'branch': branch, + } + repos = [repo] + + # neutron and nova charms require some additional repos + if service == 'neutron': + for svc in ['neutron-fwaas', 'neutron-lbaas', 'neutron-vpnaas']: + repo = { + 'name': svc, + 'repository': GIT_DEFAULT_REPOS[svc], + 'branch': branch, + } + repos.append(repo) + elif service == 'nova': + repo = { + 'name': 'neutron', + 'repository': GIT_DEFAULT_REPOS['neutron'], + 'branch': branch, + } + repos.append(repo) + + # finally add the current service's repo + repo = { + 'name': service, + 'repository': GIT_DEFAULT_REPOS[service], + 'branch': branch, + } + repos.append(repo) + + return yaml.dump(dict(repositories=repos)) + + return projects + + def _git_yaml_load(projects_yaml): """ Load the specified yaml into a dictionary. @@ -858,6 +969,47 @@ def git_yaml_value(projects_yaml, key): return None +def git_generate_systemd_init_files(templates_dir): + """ + Generate systemd init files. + + Generates and installs systemd init units and script files based on the + *.init.in files contained in the templates_dir directory. + + This code is based on the openstack-pkg-tools package and its init + script generation, which is used by the OpenStack packages. + """ + for f in os.listdir(templates_dir): + if f.endswith(".init.in"): + init_in_file = f + init_file = f[:-8] + service_file = "{}.service".format(init_file) + + init_in_source = os.path.join(templates_dir, init_in_file) + init_source = os.path.join(templates_dir, init_file) + service_source = os.path.join(templates_dir, service_file) + + init_dest = os.path.join('/etc/init.d', init_file) + service_dest = os.path.join('/lib/systemd/system', service_file) + + shutil.copyfile(init_in_source, init_source) + with open(init_source, 'a') as outfile: + template = '/usr/share/openstack-pkg-tools/init-script-template' + with open(template) as infile: + outfile.write('\n\n{}'.format(infile.read())) + + cmd = ['pkgos-gen-systemd-unit', init_in_source] + subprocess.check_call(cmd) + + if os.path.exists(init_dest): + os.remove(init_dest) + if os.path.exists(service_dest): + os.remove(service_dest) + shutil.move(init_source, init_dest) + shutil.move(service_source, service_dest) + os.chmod(init_dest, 0o755) + + def os_workload_status(configs, required_interfaces, charm_func=None): """ Decorator to set workload status based on complete contexts @@ -1574,3 +1726,82 @@ def pausable_restart_on_change(restart_map, stopstart=False, restart_functions) return wrapped_f return wrap + + +def config_flags_parser(config_flags): + """Parses config flags string into dict. + + This parsing method supports a few different formats for the config + flag values to be parsed: + + 1. A string in the simple format of key=value pairs, with the possibility + of specifying multiple key value pairs within the same string. For + example, a string in the format of 'key1=value1, key2=value2' will + return a dict of: + + {'key1': 'value1', + 'key2': 'value2'}. + + 2. A string in the above format, but supporting a comma-delimited list + of values for the same key. For example, a string in the format of + 'key1=value1, key2=value3,value4,value5' will return a dict of: + + {'key1', 'value1', + 'key2', 'value2,value3,value4'} + + 3. A string containing a colon character (:) prior to an equal + character (=) will be treated as yaml and parsed as such. This can be + used to specify more complex key value pairs. For example, + a string in the format of 'key1: subkey1=value1, subkey2=value2' will + return a dict of: + + {'key1', 'subkey1=value1, subkey2=value2'} + + The provided config_flags string may be a list of comma-separated values + which themselves may be comma-separated list of values. + """ + # If we find a colon before an equals sign then treat it as yaml. + # Note: limit it to finding the colon first since this indicates assignment + # for inline yaml. + colon = config_flags.find(':') + equals = config_flags.find('=') + if colon > 0: + if colon < equals or equals < 0: + return yaml.safe_load(config_flags) + + if config_flags.find('==') >= 0: + juju_log("config_flags is not in expected format (key=value)", + level=ERROR) + raise OSContextError + + # strip the following from each value. + post_strippers = ' ,' + # we strip any leading/trailing '=' or ' ' from the string then + # split on '='. + split = config_flags.strip(' =').split('=') + limit = len(split) + flags = {} + for i in range(0, limit - 1): + current = split[i] + next = split[i + 1] + vindex = next.rfind(',') + if (i == limit - 2) or (vindex < 0): + value = next + else: + value = next[:vindex] + + if i == 0: + key = current + else: + # if this not the first entry, expect an embedded key. + index = current.rfind(',') + if index < 0: + juju_log("Invalid config value(s) at index %s" % (i), + level=ERROR) + raise OSContextError + key = current[index + 1:] + + # Add to collection. + flags[key.strip(post_strippers)] = value.rstrip(post_strippers) + + return flags diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index d008081..b2484e7 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -40,6 +40,7 @@ from subprocess import ( CalledProcessError, ) from charmhelpers.core.hookenv import ( + config, local_unit, relation_get, relation_ids, @@ -64,6 +65,7 @@ from charmhelpers.fetch import ( ) from charmhelpers.core.kernel import modprobe +from charmhelpers.contrib.openstack.utils import config_flags_parser KEYRING = '/etc/ceph/ceph.client.{}.keyring' KEYFILE = '/etc/ceph/ceph.client.{}.key' @@ -1204,3 +1206,42 @@ def send_request_if_needed(request, relation='ceph'): for rid in relation_ids(relation): log('Sending request {}'.format(request.request_id), level=DEBUG) relation_set(relation_id=rid, broker_req=request.request) + + +class CephConfContext(object): + """Ceph config (ceph.conf) context. + + Supports user-provided Ceph configuration settings. Use can provide a + dictionary as the value for the config-flags charm option containing + Ceph configuration settings keyede by their section in ceph.conf. + """ + def __init__(self, permitted_sections=None): + self.permitted_sections = permitted_sections or [] + + def __call__(self): + conf = config('config-flags') + if not conf: + return {} + + conf = config_flags_parser(conf) + if type(conf) != dict: + log("Provided config-flags is not a dictionary - ignoring", + level=WARNING) + return {} + + permitted = self.permitted_sections + if permitted: + diff = set(conf.keys()).difference(set(permitted)) + if diff: + log("Config-flags contains invalid keys '%s' - they will be " + "ignored" % (', '.join(diff)), level=WARNING) + + ceph_conf = {} + for key in conf: + if permitted and key not in permitted: + log("Ignoring key '%s'" % key, level=WARNING) + continue + + ceph_conf[key] = conf[key] + + return ceph_conf diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 64b2df5..e367e45 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -176,7 +176,7 @@ def init_is_systemd(): def adduser(username, password=None, shell='/bin/bash', system_user=False, - primary_group=None, secondary_groups=None): + primary_group=None, secondary_groups=None, uid=None): """Add a user to the system. Will log but otherwise succeed if the user already exists. @@ -187,15 +187,21 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False, :param bool system_user: Whether to create a login or system user :param str primary_group: Primary group for user; defaults to username :param list secondary_groups: Optional list of additional groups + :param int uid: UID for user being created :returns: The password database entry struct, as returned by `pwd.getpwnam` """ try: user_info = pwd.getpwnam(username) log('user {0} already exists!'.format(username)) + if uid: + user_info = pwd.getpwuid(int(uid)) + log('user with uid {0} already exists!'.format(uid)) except KeyError: log('creating user {0}'.format(username)) cmd = ['useradd'] + if uid: + cmd.extend(['--uid', str(uid)]) if system_user or password is None: cmd.append('--system') else: @@ -230,14 +236,58 @@ def user_exists(username): return user_exists -def add_group(group_name, system_group=False): - """Add a group to the system""" +def uid_exists(uid): + """Check if a uid exists""" + try: + pwd.getpwuid(uid) + uid_exists = True + except KeyError: + uid_exists = False + return uid_exists + + +def group_exists(groupname): + """Check if a group exists""" + try: + grp.getgrnam(groupname) + group_exists = True + except KeyError: + group_exists = False + return group_exists + + +def gid_exists(gid): + """Check if a gid exists""" + try: + grp.getgrgid(gid) + gid_exists = True + except KeyError: + gid_exists = False + return gid_exists + + +def add_group(group_name, system_group=False, gid=None): + """Add a group to the system + + Will log but otherwise succeed if the group already exists. + + :param str group_name: group to create + :param bool system_group: Create system group + :param int gid: GID for user being created + + :returns: The password database entry struct, as returned by `grp.getgrnam` + """ try: group_info = grp.getgrnam(group_name) log('group {0} already exists!'.format(group_name)) + if gid: + group_info = grp.getgrgid(gid) + log('group with gid {0} already exists!'.format(gid)) except KeyError: log('creating group {0}'.format(group_name)) cmd = ['addgroup'] + if gid: + cmd.extend(['--gid', str(gid)]) if system_group: cmd.append('--system') else: diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index db0d86a..68b0f94 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -106,6 +106,14 @@ CLOUD_ARCHIVE_POCKETS = { 'mitaka/proposed': 'trusty-proposed/mitaka', 'trusty-mitaka/proposed': 'trusty-proposed/mitaka', 'trusty-proposed/mitaka': 'trusty-proposed/mitaka', + # Newton + 'newton': 'xenial-updates/newton', + 'xenial-newton': 'xenial-updates/newton', + 'xenial-newton/updates': 'xenial-updates/newton', + 'xenial-updates/newton': 'xenial-updates/newton', + 'newton/proposed': 'xenial-proposed/newton', + 'xenial-newton/proposed': 'xenial-proposed/newton', + 'xenial-proposed/newton': 'xenial-proposed/newton', } # The order of this list is very important. Handlers should be listed in from @@ -390,16 +398,13 @@ def install_remote(source, *args, **kwargs): # We ONLY check for True here because can_handle may return a string # explaining why it can't handle a given source. handlers = [h for h in plugins() if h.can_handle(source) is True] - installed_to = None for handler in handlers: try: - installed_to = handler.install(source, *args, **kwargs) + return handler.install(source, *args, **kwargs) except UnhandledSource as e: log('Install source attempt unsuccessful: {}'.format(e), level='WARNING') - if not installed_to: - raise UnhandledSource("No handler found for source {}".format(source)) - return installed_to + raise UnhandledSource("No handler found for source {}".format(source)) def install_from_config(config_var_name): diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py index cafd27f..b743753 100644 --- a/hooks/charmhelpers/fetch/bzrurl.py +++ b/hooks/charmhelpers/fetch/bzrurl.py @@ -42,15 +42,23 @@ class BzrUrlFetchHandler(BaseFetchHandler): else: return True - def branch(self, source, dest): + def branch(self, source, dest, revno=None): if not self.can_handle(source): raise UnhandledSource("Cannot handle {}".format(source)) + cmd_opts = [] + if revno: + cmd_opts += ['-r', str(revno)] if os.path.exists(dest): - check_call(['bzr', 'pull', '--overwrite', '-d', dest, source]) + cmd = ['bzr', 'pull'] + cmd += cmd_opts + cmd += ['--overwrite', '-d', dest, source] else: - check_call(['bzr', 'branch', source, dest]) + cmd = ['bzr', 'branch'] + cmd += cmd_opts + cmd += [source, dest] + check_call(cmd) - def install(self, source, dest=None): + def install(self, source, dest=None, revno=None): url_parts = self.parse_url(source) branch_name = url_parts.path.strip("/").split("/")[-1] if dest: @@ -59,10 +67,11 @@ class BzrUrlFetchHandler(BaseFetchHandler): dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", branch_name) - if not os.path.exists(dest_dir): - mkdir(dest_dir, perms=0o755) + if dest and not os.path.exists(dest): + mkdir(dest, perms=0o755) + try: - self.branch(source, dest_dir) + self.branch(source, dest_dir, revno) except OSError as e: raise UnhandledSource(e.strerror) return dest_dir diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index d21c9c7..6b917d0 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -43,9 +43,6 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.openstack = openstack self.source = source self.stable = stable - # Note(coreycb): this needs to be changed when new next branches come - # out. - self.current_next = "trusty" def get_logger(self, name="deployment-logger", level=logging.DEBUG): """Get a logger object that will log to stdout.""" @@ -72,38 +69,34 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.log.info('OpenStackAmuletDeployment: determine branch locations') - # Charms outside the lp:~openstack-charmers namespace - base_charms = ['mysql', 'mongodb', 'nrpe'] - - # Force these charms to current series even when using an older series. - # ie. Use trusty/nrpe even when series is precise, as the P charm - # does not possess the necessary external master config and hooks. - force_series_current = ['nrpe'] - - if self.series in ['precise', 'trusty']: - base_series = self.series - else: - base_series = self.current_next + # Charms outside the ~openstack-charmers + base_charms = { + 'mysql': ['precise', 'trusty'], + 'mongodb': ['precise', 'trusty'], + 'nrpe': ['precise', 'trusty'], + } for svc in other_services: - if svc['name'] in force_series_current: - base_series = self.current_next # If a location has been explicitly set, use it if svc.get('location'): continue - if self.stable: - temp = 'lp:charms/{}/{}' - svc['location'] = temp.format(base_series, - svc['name']) + if svc['name'] in base_charms: + # NOTE: not all charms have support for all series we + # want/need to test against, so fix to most recent + # that each base charm supports + target_series = self.series + if self.series not in base_charms[svc['name']]: + target_series = base_charms[svc['name']][-1] + svc['location'] = 'cs:{}/{}'.format(target_series, + svc['name']) + elif self.stable: + svc['location'] = 'cs:{}/{}'.format(self.series, + svc['name']) else: - if svc['name'] in base_charms: - temp = 'lp:charms/{}/{}' - svc['location'] = temp.format(base_series, - svc['name']) - else: - temp = 'lp:~openstack-charmers/charms/{}/{}/next' - svc['location'] = temp.format(self.current_next, - svc['name']) + svc['location'] = 'cs:~openstack-charmers-next/{}/{}'.format( + self.series, + svc['name'] + ) return other_services