diff --git a/hooks/charmhelpers/contrib/hardening/templating.py b/hooks/charmhelpers/contrib/hardening/templating.py index 2174c64..5b6765f 100644 --- a/hooks/charmhelpers/contrib/hardening/templating.py +++ b/hooks/charmhelpers/contrib/hardening/templating.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import six from charmhelpers.core.hookenv import ( log, @@ -26,7 +27,10 @@ except ImportError: from charmhelpers.fetch import apt_install from charmhelpers.fetch import apt_update apt_update(fatal=True) - apt_install('python-jinja2', fatal=True) + if six.PY2: + apt_install('python-jinja2', fatal=True) + else: + apt_install('python3-jinja2', fatal=True) from jinja2 import FileSystemLoader, Environment diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index e141fc1..54c76a7 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -20,25 +20,37 @@ import socket from functools import partial -from charmhelpers.core.hookenv import unit_get from charmhelpers.fetch import apt_install, apt_update from charmhelpers.core.hookenv import ( + config, log, + network_get_primary_address, + unit_get, WARNING, ) +from charmhelpers.core.host import ( + lsb_release, +) + try: import netifaces except ImportError: apt_update(fatal=True) - apt_install('python-netifaces', fatal=True) + if six.PY2: + apt_install('python-netifaces', fatal=True) + else: + apt_install('python3-netifaces', fatal=True) import netifaces try: import netaddr except ImportError: apt_update(fatal=True) - apt_install('python-netaddr', fatal=True) + if six.PY2: + apt_install('python-netaddr', fatal=True) + else: + apt_install('python3-netaddr', fatal=True) import netaddr @@ -414,7 +426,10 @@ def ns_query(address): try: import dns.resolver except ImportError: - apt_install('python-dnspython', fatal=True) + if six.PY2: + apt_install('python-dnspython', fatal=True) + else: + apt_install('python3-dnspython', fatal=True) import dns.resolver if isinstance(address, dns.name.Name): @@ -462,7 +477,10 @@ def get_hostname(address, fqdn=True): try: import dns.reversename except ImportError: - apt_install("python-dnspython", fatal=True) + if six.PY2: + apt_install("python-dnspython", fatal=True) + else: + apt_install("python3-dnspython", fatal=True) import dns.reversename rev = dns.reversename.from_address(address) @@ -499,3 +517,40 @@ def port_has_listener(address, port): cmd = ['nc', '-z', address, str(port)] result = subprocess.call(cmd) return not(bool(result)) + + +def assert_charm_supports_ipv6(): + """Check whether we are able to support charms ipv6.""" + if lsb_release()['DISTRIB_CODENAME'].lower() < "trusty": + raise Exception("IPv6 is not supported in the charms for Ubuntu " + "versions less than Trusty 14.04") + + +def get_relation_ip(interface, config_override=None): + """Return this unit's IP for the given relation. + + Allow for an arbitrary interface to use with network-get to select an IP. + Handle all address selection options including configuration parameter + override and IPv6. + + Usage: get_relation_ip('amqp', config_override='access-network') + + @param interface: string name of the relation. + @param config_override: string name of the config option for network + override. Supports legacy network override configuration parameters. + @raises Exception if prefer-ipv6 is configured but IPv6 unsupported. + @returns IPv6 or IPv4 address + """ + + fallback = get_host_ip(unit_get('private-address')) + if config('prefer-ipv6'): + assert_charm_supports_ipv6() + return get_ipv6_addr()[0] + elif config_override and config(config_override): + return get_address_in_network(config(config_override), + fallback) + else: + try: + return network_get_primary_address(interface) + except NotImplementedError: + return fallback diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 401c032..1f4cf42 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -32,6 +32,7 @@ from keystoneclient.v3 import client as keystone_client_v3 from novaclient import exceptions import novaclient.client as nova_client +import novaclient import pika import swiftclient @@ -434,9 +435,14 @@ class OpenStackAmuletUtils(AmuletUtils): self.log.debug('Authenticating nova user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', endpoint_type='publicURL') - return nova_client.Client(NOVA_CLIENT_VERSION, - username=user, api_key=password, - project_id=tenant, auth_url=ep) + if novaclient.__version__[0] >= "7": + return nova_client.Client(NOVA_CLIENT_VERSION, + username=user, password=password, + project_name=tenant, auth_url=ep) + else: + return nova_client.Client(NOVA_CLIENT_VERSION, + username=user, api_key=password, + project_id=tenant, auth_url=ep) def authenticate_swift_user(self, keystone, user, password, tenant): """Authenticates a regular user with swift api.""" diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 4231633..6cdbbbb 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -100,7 +100,10 @@ from charmhelpers.core.unitdata import kv try: import psutil except ImportError: - apt_install('python-psutil', fatal=True) + if six.PY2: + apt_install('python-psutil', fatal=True) + else: + apt_install('python3-psutil', fatal=True) import psutil CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' @@ -392,16 +395,20 @@ class AMQPContext(OSContextGenerator): for rid in relation_ids(self.rel_name): ha_vip_only = False self.related = True + transport_hosts = None + rabbitmq_port = '5672' for unit in related_units(rid): if relation_get('clustered', rid=rid, unit=unit): ctxt['clustered'] = True vip = relation_get('vip', rid=rid, unit=unit) vip = format_ipv6_addr(vip) or vip ctxt['rabbitmq_host'] = vip + transport_hosts = [vip] else: host = relation_get('private-address', rid=rid, unit=unit) host = format_ipv6_addr(host) or host ctxt['rabbitmq_host'] = host + transport_hosts = [host] ctxt.update({ 'rabbitmq_user': username, @@ -413,6 +420,7 @@ class AMQPContext(OSContextGenerator): ssl_port = relation_get('ssl_port', rid=rid, unit=unit) if ssl_port: ctxt['rabbit_ssl_port'] = ssl_port + rabbitmq_port = ssl_port ssl_ca = relation_get('ssl_ca', rid=rid, unit=unit) if ssl_ca: @@ -450,6 +458,20 @@ class AMQPContext(OSContextGenerator): rabbitmq_hosts.append(host) ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts)) + transport_hosts = rabbitmq_hosts + + if transport_hosts: + transport_url_hosts = '' + for host in transport_hosts: + if transport_url_hosts: + format_string = ",{}:{}@{}:{}" + else: + format_string = "{}:{}@{}:{}" + transport_url_hosts += format_string.format( + ctxt['rabbitmq_user'], ctxt['rabbitmq_password'], + host, rabbitmq_port) + ctxt['transport_url'] = "rabbit://{}/{}".format( + transport_url_hosts, vhost) oslo_messaging_flags = conf.get('oslo-messaging-flags', None) if oslo_messaging_flags: @@ -481,13 +503,16 @@ class CephContext(OSContextGenerator): ctxt['auth'] = relation_get('auth', rid=rid, unit=unit) if not ctxt.get('key'): ctxt['key'] = relation_get('key', rid=rid, unit=unit) - ceph_pub_addr = relation_get('ceph-public-address', rid=rid, + + ceph_addrs = relation_get('ceph-public-address', rid=rid, + unit=unit) + if ceph_addrs: + for addr in ceph_addrs.split(' '): + mon_hosts.append(format_ipv6_addr(addr) or addr) + else: + priv_addr = relation_get('private-address', rid=rid, unit=unit) - unit_priv_addr = relation_get('private-address', rid=rid, - unit=unit) - ceph_addr = ceph_pub_addr or unit_priv_addr - ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr - mon_hosts.append(ceph_addr) + mon_hosts.append(format_ipv6_addr(priv_addr) or priv_addr) ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts)) diff --git a/hooks/charmhelpers/contrib/openstack/ha/utils.py b/hooks/charmhelpers/contrib/openstack/ha/utils.py index 1f5310b..254a90e 100644 --- a/hooks/charmhelpers/contrib/openstack/ha/utils.py +++ b/hooks/charmhelpers/contrib/openstack/ha/utils.py @@ -126,3 +126,14 @@ def assert_charm_supports_dns_ha(): status_set('blocked', msg) raise DNSHAException(msg) return True + + +def expect_ha(): + """ Determine if the unit expects to be in HA + + Check for VIP or dns-ha settings which indicate the unit should expect to + be related to hacluster. + + @returns boolean + """ + return config('vip') or config('dns-ha') diff --git a/hooks/charmhelpers/contrib/openstack/keystone.py b/hooks/charmhelpers/contrib/openstack/keystone.py new file mode 100644 index 0000000..a15a03f --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/keystone.py @@ -0,0 +1,178 @@ +#!/usr/bin/python +# +# Copyright 2017 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six +from charmhelpers.fetch import apt_install +from charmhelpers.contrib.openstack.context import IdentityServiceContext +from charmhelpers.core.hookenv import ( + log, + ERROR, +) + + +def get_api_suffix(api_version): + """Return the formatted api suffix for the given version + @param api_version: version of the keystone endpoint + @returns the api suffix formatted according to the given api + version + """ + return 'v2.0' if api_version in (2, "2.0") else 'v3' + + +def format_endpoint(schema, addr, port, api_version): + """Return a formatted keystone endpoint + @param schema: http or https + @param addr: ipv4/ipv6 host of the keystone service + @param port: port of the keystone service + @param api_version: 2 or 3 + @returns a fully formatted keystone endpoint + """ + return '{}://{}:{}/{}/'.format(schema, addr, port, + get_api_suffix(api_version)) + + +def get_keystone_manager(endpoint, api_version, **kwargs): + """Return a keystonemanager for the correct API version + + @param endpoint: the keystone endpoint to point client at + @param api_version: version of the keystone api the client should use + @param kwargs: token or username/tenant/password information + @returns keystonemanager class used for interrogating keystone + """ + if api_version == 2: + return KeystoneManager2(endpoint, **kwargs) + if api_version == 3: + return KeystoneManager3(endpoint, **kwargs) + raise ValueError('No manager found for api version {}'.format(api_version)) + + +def get_keystone_manager_from_identity_service_context(): + """Return a keystonmanager generated from a + instance of charmhelpers.contrib.openstack.context.IdentityServiceContext + @returns keystonamenager instance + """ + context = IdentityServiceContext()() + if not context: + msg = "Identity service context cannot be generated" + log(msg, level=ERROR) + raise ValueError(msg) + + endpoint = format_endpoint(context['service_protocol'], + context['service_host'], + context['service_port'], + context['api_version']) + + if context['api_version'] in (2, "2.0"): + api_version = 2 + else: + api_version = 3 + + return get_keystone_manager(endpoint, api_version, + username=context['admin_user'], + password=context['admin_password'], + tenant_name=context['admin_tenant_name']) + + +class KeystoneManager(object): + + def resolve_service_id(self, service_name=None, service_type=None): + """Find the service_id of a given service""" + services = [s._info for s in self.api.services.list()] + + service_name = service_name.lower() + for s in services: + name = s['name'].lower() + if service_type and service_name: + if (service_name == name and service_type == s['type']): + return s['id'] + elif service_name and service_name == name: + return s['id'] + elif service_type and service_type == s['type']: + return s['id'] + return None + + def service_exists(self, service_name=None, service_type=None): + """Determine if the given service exists on the service list""" + return self.resolve_service_id(service_name, service_type) is not None + + +class KeystoneManager2(KeystoneManager): + + def __init__(self, endpoint, **kwargs): + try: + from keystoneclient.v2_0 import client + from keystoneclient.auth.identity import v2 + from keystoneclient import session + except ImportError: + if six.PY2: + apt_install(["python-keystoneclient"], fatal=True) + else: + apt_install(["python3-keystoneclient"], fatal=True) + + from keystoneclient.v2_0 import client + from keystoneclient.auth.identity import v2 + from keystoneclient import session + + self.api_version = 2 + + token = kwargs.get("token", None) + if token: + api = client.Client(endpoint=endpoint, token=token) + else: + auth = v2.Password(username=kwargs.get("username"), + password=kwargs.get("password"), + tenant_name=kwargs.get("tenant_name"), + auth_url=endpoint) + sess = session.Session(auth=auth) + api = client.Client(session=sess) + + self.api = api + + +class KeystoneManager3(KeystoneManager): + + def __init__(self, endpoint, **kwargs): + try: + from keystoneclient.v3 import client + from keystoneclient.auth import token_endpoint + from keystoneclient import session + from keystoneclient.auth.identity import v3 + except ImportError: + if six.PY2: + apt_install(["python-keystoneclient"], fatal=True) + else: + apt_install(["python3-keystoneclient"], fatal=True) + + from keystoneclient.v3 import client + from keystoneclient.auth import token_endpoint + from keystoneclient import session + from keystoneclient.auth.identity import v3 + + self.api_version = 3 + + token = kwargs.get("token", None) + if token: + auth = token_endpoint.Token(endpoint=endpoint, + token=token) + sess = session.Session(auth=auth) + else: + auth = v3.Password(auth_url=endpoint, + user_id=kwargs.get("username"), + password=kwargs.get("password"), + project_id=kwargs.get("tenant_name")) + sess = session.Session(auth=auth) + + self.api = client.Client(session=sess) diff --git a/hooks/charmhelpers/contrib/openstack/templating.py b/hooks/charmhelpers/contrib/openstack/templating.py index 8958895..934baf5 100644 --- a/hooks/charmhelpers/contrib/openstack/templating.py +++ b/hooks/charmhelpers/contrib/openstack/templating.py @@ -28,7 +28,10 @@ try: from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions except ImportError: apt_update(fatal=True) - apt_install('python-jinja2', fatal=True) + if six.PY2: + apt_install('python-jinja2', fatal=True) + else: + apt_install('python3-jinja2', fatal=True) from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions @@ -207,7 +210,10 @@ class OSConfigRenderer(object): # if this code is running, the object is created pre-install hook. # jinja2 shouldn't get touched until the module is reloaded on next # hook execution, with proper jinja2 bits successfully imported. - apt_install('python-jinja2') + if six.PY2: + apt_install('python-jinja2') + else: + apt_install('python3-jinja2') def register(self, config_file, contexts): """ diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 80219d6..7e8ecff 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -153,7 +153,7 @@ SWIFT_CODENAMES = OrderedDict([ ('newton', ['2.8.0', '2.9.0', '2.10.0']), ('ocata', - ['2.11.0', '2.12.0']), + ['2.11.0', '2.12.0', '2.13.0']), ]) # >= Liberty version->codename mapping diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index e29bd1b..6e95028 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -16,6 +16,7 @@ # limitations under the License. import os +import six import subprocess import sys @@ -39,7 +40,10 @@ def pip_execute(*args, **kwargs): from pip import main as _pip_execute except ImportError: apt_update() - apt_install('python-pip') + if six.PY2: + apt_install('python-pip') + else: + apt_install('python3-pip') from pip import main as _pip_execute _pip_execute(*args, **kwargs) finally: @@ -136,7 +140,10 @@ def pip_list(): def pip_create_virtualenv(path=None): """Create an isolated Python environment.""" - apt_install('python-virtualenv') + if six.PY2: + apt_install('python-virtualenv') + else: + apt_install('python3-virtualenv') if path: venv_path = path diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index edbb72f..05edfa5 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -306,6 +306,8 @@ SYSTEMD_SYSTEM = '/run/systemd/system' def init_is_systemd(): """Return True if the host system uses systemd, False otherwise.""" + if lsb_release()['DISTRIB_CODENAME'] == 'trusty': + return False return os.path.isdir(SYSTEMD_SYSTEM) diff --git a/hooks/charmhelpers/fetch/snap.py b/hooks/charmhelpers/fetch/snap.py new file mode 100644 index 0000000..23c707b --- /dev/null +++ b/hooks/charmhelpers/fetch/snap.py @@ -0,0 +1,122 @@ +# Copyright 2014-2017 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Charm helpers snap for classic charms. + +If writing reactive charms, use the snap layer: +https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html +""" +import subprocess +from os import environ +from time import sleep +from charmhelpers.core.hookenv import log + +__author__ = 'Joseph Borg ' + +SNAP_NO_LOCK = 1 # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved). +SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks. +SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. + + +class CouldNotAcquireLockException(Exception): + pass + + +def _snap_exec(commands): + """ + Execute snap commands. + + :param commands: List commands + :return: Integer exit code + """ + assert type(commands) == list + + retry_count = 0 + return_code = None + + while return_code is None or return_code == SNAP_NO_LOCK: + try: + return_code = subprocess.check_call(['snap'] + commands, env=environ) + except subprocess.CalledProcessError as e: + retry_count += + 1 + if retry_count > SNAP_NO_LOCK_RETRY_COUNT: + raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT) + return_code = e.returncode + log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN') + sleep(SNAP_NO_LOCK_RETRY_DELAY) + + return return_code + + +def snap_install(packages, *flags): + """ + Install a snap package. + + :param packages: String or List String package name + :param flags: List String flags to pass to install command + :return: Integer return code from snap + """ + if type(packages) is not list: + packages = [packages] + + flags = list(flags) + + message = 'Installing snap(s) "%s"' % ', '.join(packages) + if flags: + message += ' with option(s) "%s"' % ', '.join(flags) + + log(message, level='INFO') + return _snap_exec(['install'] + flags + packages) + + +def snap_remove(packages, *flags): + """ + Remove a snap package. + + :param packages: String or List String package name + :param flags: List String flags to pass to remove command + :return: Integer return code from snap + """ + if type(packages) is not list: + packages = [packages] + + flags = list(flags) + + message = 'Removing snap(s) "%s"' % ', '.join(packages) + if flags: + message += ' with options "%s"' % ', '.join(flags) + + log(message, level='INFO') + return _snap_exec(['remove'] + flags + packages) + + +def snap_refresh(packages, *flags): + """ + Refresh / Update snap package. + + :param packages: String or List String package name + :param flags: List String flags to pass to refresh command + :return: Integer return code from snap + """ + if type(packages) is not list: + packages = [packages] + + flags = list(flags) + + message = 'Refreshing snap(s) "%s"' % ', '.join(packages) + if flags: + message += ' with options "%s"' % ', '.join(flags) + + log(message, level='INFO') + return _snap_exec(['refresh'] + flags + packages) diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py index 39b9b80..82ac80f 100644 --- a/hooks/charmhelpers/fetch/ubuntu.py +++ b/hooks/charmhelpers/fetch/ubuntu.py @@ -116,8 +116,8 @@ CLOUD_ARCHIVE_POCKETS = { } APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. -APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks. -APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. +CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries. +CMD_RETRY_COUNT = 30 # Retry a failing fatal command X times. def filter_installed_packages(packages): @@ -249,7 +249,8 @@ def add_source(source, key=None): source.startswith('http') or source.startswith('deb ') or source.startswith('cloud-archive:')): - subprocess.check_call(['add-apt-repository', '--yes', source]) + cmd = ['add-apt-repository', '--yes', source] + _run_with_retries(cmd) elif source.startswith('cloud:'): install(filter_installed_packages(['ubuntu-cloud-keyring']), fatal=True) @@ -286,41 +287,60 @@ def add_source(source, key=None): key]) -def _run_apt_command(cmd, fatal=False): - """Run an APT command. - - Checks the output and retries if the fatal flag is set - to True. +def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,), + retry_message="", cmd_env=None): + """Run a command and retry until success or max_retries is reached. :param: cmd: str: The apt command to run. + :param: max_retries: int: The number of retries to attempt on a fatal + command. Defaults to CMD_RETRY_COUNT. + :param: retry_exitcodes: tuple: Optional additional exit codes to retry. + Defaults to retry on exit code 1. + :param: retry_message: str: Optional log prefix emitted during retries. + :param: cmd_env: dict: Environment variables to add to the command run. + """ + + env = os.environ.copy() + if cmd_env: + env.update(cmd_env) + + if not retry_message: + retry_message = "Failed executing '{}'".format(" ".join(cmd)) + retry_message += ". Will retry in {} seconds".format(CMD_RETRY_DELAY) + + retry_count = 0 + result = None + + retry_results = (None,) + retry_exitcodes + while result in retry_results: + try: + result = subprocess.check_call(cmd, env=env) + except subprocess.CalledProcessError as e: + retry_count = retry_count + 1 + if retry_count > max_retries: + raise + result = e.returncode + log(retry_message) + time.sleep(CMD_RETRY_DELAY) + + +def _run_apt_command(cmd, fatal=False): + """Run an apt command with optional retries. + :param: fatal: bool: Whether the command's output should be checked and retried. """ - env = os.environ.copy() - - if 'DEBIAN_FRONTEND' not in env: - env['DEBIAN_FRONTEND'] = 'noninteractive' + # Provide DEBIAN_FRONTEND=noninteractive if not present in the environment. + cmd_env = { + 'DEBIAN_FRONTEND': os.environ.get('DEBIAN_FRONTEND', 'noninteractive')} if fatal: - retry_count = 0 - result = None - - # If the command is considered "fatal", we need to retry if the apt - # lock was not acquired. - - while result is None or result == APT_NO_LOCK: - try: - result = subprocess.check_call(cmd, env=env) - except subprocess.CalledProcessError as e: - retry_count = retry_count + 1 - if retry_count > APT_NO_LOCK_RETRY_COUNT: - raise - result = e.returncode - log("Couldn't acquire DPKG lock. Will retry in {} seconds." - "".format(APT_NO_LOCK_RETRY_DELAY)) - time.sleep(APT_NO_LOCK_RETRY_DELAY) - + _run_with_retries( + cmd, cmd_env=cmd_env, retry_exitcodes=(1, APT_NO_LOCK,), + retry_message="Couldn't acquire DPKG lock") else: + env = os.environ.copy() + env.update(cmd_env) subprocess.call(cmd, env=env) diff --git a/metadata.yaml b/metadata.yaml index 59f673a..c2170aa 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -9,6 +9,7 @@ tags: - openstack series: - xenial + - zesty - trusty - yakkety extra-bindings: diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 54b55b4..a01b9ff 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -78,6 +78,12 @@ class HeatBasicDeployment(OpenStackAmuletDeployment): {'name': 'nova-cloud-controller'}, {'name': 'nova-compute'} ] + if self._get_openstack_release() >= self.xenial_ocata: + other_services.extend([ + {'name': 'neutron-gateway'}, + {'name': 'neutron-api'}, + {'name': 'neutron-openvswitch'}, + ]) super(HeatBasicDeployment, self)._add_services(this_service, other_services) @@ -103,6 +109,19 @@ class HeatBasicDeployment(OpenStackAmuletDeployment): 'glance:shared-db': 'percona-cluster:shared-db', 'glance:amqp': 'rabbitmq-server:amqp' } + if self._get_openstack_release() >= self.xenial_ocata: + relations.update({ + 'neutron-gateway:amqp': 'rabbitmq-server:amqp', + 'nova-cloud-controller:quantum-network-service': + 'neutron-gateway:quantum-network-service', + 'neutron-api:shared-db': 'percona-cluster:shared-db', + 'neutron-api:amqp': 'rabbitmq-server:amqp', + 'neutron-api:neutron-api': 'nova-cloud-controller:neutron-api', + 'neutron-api:identity-service': 'keystone:identity-service', + 'nova-compute:neutron-plugin': 'neutron-openvswitch:' + 'neutron-plugin', + 'rabbitmq-server:amqp': 'neutron-openvswitch:amqp', + }) super(HeatBasicDeployment, self)._add_relations(relations) def _configure_services(self): @@ -118,10 +137,23 @@ class HeatBasicDeployment(OpenStackAmuletDeployment): 'sst-password': 'ChangeMe123', } + heat_config = { + 'debug': True, + 'verbose': True, + } + + nova_cc_config = { + 'api-rate-limit-rules': "( POST, '*', .*, 9999, MINUTE );", + } + if self._get_openstack_release() >= self.xenial_ocata: + nova_cc_config['network-manager'] = 'Neutron' + configs = { + 'nova-cloud-controller': nova_cc_config, 'nova-compute': nova_config, 'keystone': keystone_config, 'percona-cluster': pxc_config, + 'heat': heat_config, } super(HeatBasicDeployment, self)._configure_services(configs) @@ -500,8 +532,8 @@ class HeatBasicDeployment(OpenStackAmuletDeployment): expected = { 'DEFAULT': { 'use_syslog': 'False', - 'debug': 'False', - 'verbose': 'False', + 'debug': 'True', + 'verbose': 'True', 'log_dir': '/var/log/heat', 'instance_driver': 'heat.engine.nova', 'plugin_dirs': '/usr/lib64/heat,/usr/lib/heat', diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index f9e4c3a..8a6b764 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -785,37 +785,30 @@ class AmuletUtils(object): generating test messages which need to be unique-ish.""" return '[{}-{}]'.format(uuid.uuid4(), time.time()) -# amulet juju action helpers: + # amulet juju action helpers: def run_action(self, unit_sentry, action, _check_output=subprocess.check_output, params=None): - """Run the named action on a given unit sentry. + """Translate to amulet's built in run_action(). Deprecated. + + Run the named action on a given unit sentry. params a dict of parameters to use - _check_output parameter is used for dependency injection. + _check_output parameter is no longer used @return action_id. """ - unit_id = unit_sentry.info["unit_name"] - command = ["juju", "action", "do", "--format=json", unit_id, action] - if params is not None: - for key, value in params.iteritems(): - command.append("{}={}".format(key, value)) - self.log.info("Running command: %s\n" % " ".join(command)) - output = _check_output(command, universal_newlines=True) - data = json.loads(output) - action_id = data[u'Action queued with id'] - return action_id + self.log.warn('charmhelpers.contrib.amulet.utils.run_action has been ' + 'deprecated for amulet.run_action') + return unit_sentry.run_action(action, action_args=params) def wait_on_action(self, action_id, _check_output=subprocess.check_output): """Wait for a given action, returning if it completed or not. - _check_output parameter is used for dependency injection. + action_id a string action uuid + _check_output parameter is no longer used """ - command = ["juju", "action", "fetch", "--format=json", "--wait=0", - action_id] - output = _check_output(command, universal_newlines=True) - data = json.loads(output) + data = amulet.actions.get_action_output(action_id, full_output=True) return data.get(u"status") == "completed" def status_get(self, unit): diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index 401c032..1f4cf42 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -32,6 +32,7 @@ from keystoneclient.v3 import client as keystone_client_v3 from novaclient import exceptions import novaclient.client as nova_client +import novaclient import pika import swiftclient @@ -434,9 +435,14 @@ class OpenStackAmuletUtils(AmuletUtils): self.log.debug('Authenticating nova user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', endpoint_type='publicURL') - return nova_client.Client(NOVA_CLIENT_VERSION, - username=user, api_key=password, - project_id=tenant, auth_url=ep) + if novaclient.__version__[0] >= "7": + return nova_client.Client(NOVA_CLIENT_VERSION, + username=user, password=password, + project_name=tenant, auth_url=ep) + else: + return nova_client.Client(NOVA_CLIENT_VERSION, + username=user, api_key=password, + project_id=tenant, auth_url=ep) def authenticate_swift_user(self, keystone, user, password, tenant): """Authenticates a regular user with swift api.""" diff --git a/tests/charmhelpers/core/host.py b/tests/charmhelpers/core/host.py index edbb72f..05edfa5 100644 --- a/tests/charmhelpers/core/host.py +++ b/tests/charmhelpers/core/host.py @@ -306,6 +306,8 @@ SYSTEMD_SYSTEM = '/run/systemd/system' def init_is_systemd(): """Return True if the host system uses systemd, False otherwise.""" + if lsb_release()['DISTRIB_CODENAME'] == 'trusty': + return False return os.path.isdir(SYSTEMD_SYSTEM) diff --git a/tests/gate-basic-xenial-ocata b/tests/gate-basic-xenial-ocata old mode 100644 new mode 100755