From 7f59c00a59886e0fbf09edec575692c5eb71be87 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 7 Mar 2017 11:29:18 -0800 Subject: [PATCH] Enable Ocata Amulet Tests - Add Zesty as a supported series to metadata.yaml. - Turn on Xenial-Ocata Amulet test definitions. - Sync charm helpers to get Juju 2.x amulet compatibility. - Keeping Zesty-Ocata Amulet test definitions turned off until the metadata.yaml changes propagate to the charm store. Change-Id: I906b0008fb15939334b993e25f977c0e8e3ed96c --- .../charmhelpers/contrib/charmsupport/nrpe.py | 32 +++- .../contrib/hardening/templating.py | 6 +- hooks/charmhelpers/contrib/network/ip.py | 65 ++++++- .../contrib/network/ovs/__init__.py | 67 ++++++- .../contrib/openstack/amulet/utils.py | 12 +- .../charmhelpers/contrib/openstack/context.py | 39 +++- .../contrib/openstack/ha/utils.py | 11 ++ .../contrib/openstack/keystone.py | 178 ++++++++++++++++++ .../contrib/openstack/templating.py | 10 +- hooks/charmhelpers/contrib/openstack/utils.py | 2 +- hooks/charmhelpers/contrib/python/packages.py | 11 +- hooks/charmhelpers/core/host.py | 2 + hooks/charmhelpers/fetch/snap.py | 122 ++++++++++++ hooks/charmhelpers/fetch/ubuntu.py | 80 +++++--- metadata.yaml | 1 + tests/gate-basic-xenial-ocata | 0 16 files changed, 576 insertions(+), 62 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/keystone.py create mode 100644 hooks/charmhelpers/fetch/snap.py mode change 100644 => 100755 tests/gate-basic-xenial-ocata diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py index 1410512..9646b83 100644 --- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py +++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py @@ -227,6 +227,7 @@ class NRPE(object): nagios_logdir = '/var/log/nagios' nagios_exportdir = '/var/lib/nagios/export' nrpe_confdir = '/etc/nagios/nrpe.d' + homedir = '/var/lib/nagios' # home dir provided by nagios-nrpe-server def __init__(self, hostname=None, primary=True): super(NRPE, self).__init__() @@ -338,13 +339,14 @@ def get_nagios_unit_name(relation_name='nrpe-external-master'): return unit -def add_init_service_checks(nrpe, services, unit_name): +def add_init_service_checks(nrpe, services, unit_name, immediate_check=True): """ Add checks for each service in list :param NRPE nrpe: NRPE object to add check to :param list services: List of services to check :param str unit_name: Unit name to use in check description + :param bool immediate_check: For sysv init, run the service check immediately """ for svc in services: # Don't add a check for these services from neutron-gateway @@ -368,21 +370,31 @@ def add_init_service_checks(nrpe, services, unit_name): ) elif os.path.exists(sysv_init): cronpath = '/etc/cron.d/nagios-service-check-%s' % svc - cron_file = ('*/5 * * * * root ' - '/usr/local/lib/nagios/plugins/check_exit_status.pl ' - '-s /etc/init.d/%s status > ' - '/var/lib/nagios/service-check-%s.txt\n' % (svc, - svc) - ) + checkpath = '%s/service-check-%s.txt' % (nrpe.homedir, svc) + croncmd = ( + '/usr/local/lib/nagios/plugins/check_exit_status.pl ' + '-s /etc/init.d/%s status' % svc + ) + cron_file = '*/5 * * * * root %s > %s\n' % (croncmd, checkpath) f = open(cronpath, 'w') f.write(cron_file) f.close() nrpe.add_check( shortname=svc, - description='process check {%s}' % unit_name, - check_cmd='check_status_file.py -f ' - '/var/lib/nagios/service-check-%s.txt' % svc, + description='service check {%s}' % unit_name, + check_cmd='check_status_file.py -f %s' % checkpath, ) + # if /var/lib/nagios doesn't exist open(checkpath, 'w') will fail + # (LP: #1670223). + if immediate_check and os.path.isdir(nrpe.homedir): + f = open(checkpath, 'w') + subprocess.call( + croncmd.split(), + stdout=f, + stderr=subprocess.STDOUT + ) + f.close() + os.chmod(checkpath, 0o644) def copy_nrpe_checks(): diff --git a/hooks/charmhelpers/contrib/hardening/templating.py b/hooks/charmhelpers/contrib/hardening/templating.py index 2174c64..5b6765f 100644 --- a/hooks/charmhelpers/contrib/hardening/templating.py +++ b/hooks/charmhelpers/contrib/hardening/templating.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import six from charmhelpers.core.hookenv import ( log, @@ -26,7 +27,10 @@ except ImportError: from charmhelpers.fetch import apt_install from charmhelpers.fetch import apt_update apt_update(fatal=True) - apt_install('python-jinja2', fatal=True) + if six.PY2: + apt_install('python-jinja2', fatal=True) + else: + apt_install('python3-jinja2', fatal=True) from jinja2 import FileSystemLoader, Environment diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index e141fc1..54c76a7 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -20,25 +20,37 @@ import socket from functools import partial -from charmhelpers.core.hookenv import unit_get from charmhelpers.fetch import apt_install, apt_update from charmhelpers.core.hookenv import ( + config, log, + network_get_primary_address, + unit_get, WARNING, ) +from charmhelpers.core.host import ( + lsb_release, +) + try: import netifaces except ImportError: apt_update(fatal=True) - apt_install('python-netifaces', fatal=True) + if six.PY2: + apt_install('python-netifaces', fatal=True) + else: + apt_install('python3-netifaces', fatal=True) import netifaces try: import netaddr except ImportError: apt_update(fatal=True) - apt_install('python-netaddr', fatal=True) + if six.PY2: + apt_install('python-netaddr', fatal=True) + else: + apt_install('python3-netaddr', fatal=True) import netaddr @@ -414,7 +426,10 @@ def ns_query(address): try: import dns.resolver except ImportError: - apt_install('python-dnspython', fatal=True) + if six.PY2: + apt_install('python-dnspython', fatal=True) + else: + apt_install('python3-dnspython', fatal=True) import dns.resolver if isinstance(address, dns.name.Name): @@ -462,7 +477,10 @@ def get_hostname(address, fqdn=True): try: import dns.reversename except ImportError: - apt_install("python-dnspython", fatal=True) + if six.PY2: + apt_install("python-dnspython", fatal=True) + else: + apt_install("python3-dnspython", fatal=True) import dns.reversename rev = dns.reversename.from_address(address) @@ -499,3 +517,40 @@ def port_has_listener(address, port): cmd = ['nc', '-z', address, str(port)] result = subprocess.call(cmd) return not(bool(result)) + + +def assert_charm_supports_ipv6(): + """Check whether we are able to support charms ipv6.""" + if lsb_release()['DISTRIB_CODENAME'].lower() < "trusty": + raise Exception("IPv6 is not supported in the charms for Ubuntu " + "versions less than Trusty 14.04") + + +def get_relation_ip(interface, config_override=None): + """Return this unit's IP for the given relation. + + Allow for an arbitrary interface to use with network-get to select an IP. + Handle all address selection options including configuration parameter + override and IPv6. + + Usage: get_relation_ip('amqp', config_override='access-network') + + @param interface: string name of the relation. + @param config_override: string name of the config option for network + override. Supports legacy network override configuration parameters. + @raises Exception if prefer-ipv6 is configured but IPv6 unsupported. + @returns IPv6 or IPv4 address + """ + + fallback = get_host_ip(unit_get('private-address')) + if config('prefer-ipv6'): + assert_charm_supports_ipv6() + return get_ipv6_addr()[0] + elif config_override and config(config_override): + return get_address_in_network(config(config_override), + fallback) + else: + try: + return network_get_primary_address(interface) + except NotImplementedError: + return fallback diff --git a/hooks/charmhelpers/contrib/network/ovs/__init__.py b/hooks/charmhelpers/contrib/network/ovs/__init__.py index 1bc68dc..f044b60 100644 --- a/hooks/charmhelpers/contrib/network/ovs/__init__.py +++ b/hooks/charmhelpers/contrib/network/ovs/__init__.py @@ -15,13 +15,30 @@ ''' Helpers for interacting with OpenvSwitch ''' import subprocess import os +import six + +from charmhelpers.fetch import apt_install + + from charmhelpers.core.hookenv import ( - log, WARNING + log, WARNING, INFO, DEBUG ) from charmhelpers.core.host import ( service ) +BRIDGE_TEMPLATE = """\ +# This veth pair is required when neutron data-port is mapped to an existing linux bridge. lp:1635067 + +auto {linuxbridge_port} +iface {linuxbridge_port} inet manual + pre-up ip link add name {linuxbridge_port} type veth peer name {ovsbridge_port} + pre-up ip link set {ovsbridge_port} master {bridge} + pre-up ip link set {ovsbridge_port} up + up ip link set {linuxbridge_port} up + down ip link del {linuxbridge_port} +""" + def add_bridge(name, datapath_type=None): ''' Add the named bridge to openvswitch ''' @@ -60,6 +77,54 @@ def del_bridge_port(name, port): subprocess.check_call(["ip", "link", "set", port, "promisc", "off"]) +def add_ovsbridge_linuxbridge(name, bridge): + ''' Add linux bridge to the named openvswitch bridge + :param name: Name of ovs bridge to be added to Linux bridge + :param bridge: Name of Linux bridge to be added to ovs bridge + :returns: True if veth is added between ovs bridge and linux bridge, + False otherwise''' + try: + import netifaces + except ImportError: + if six.PY2: + apt_install('python-netifaces', fatal=True) + else: + apt_install('python3-netifaces', fatal=True) + import netifaces + + ovsbridge_port = "veth-" + name + linuxbridge_port = "veth-" + bridge + log('Adding linuxbridge {} to ovsbridge {}'.format(bridge, name), + level=INFO) + interfaces = netifaces.interfaces() + for interface in interfaces: + if interface == ovsbridge_port or interface == linuxbridge_port: + log('Interface {} already exists'.format(interface), level=INFO) + return + + with open('/etc/network/interfaces.d/{}.cfg'.format( + linuxbridge_port), 'w') as config: + config.write(BRIDGE_TEMPLATE.format(linuxbridge_port=linuxbridge_port, + ovsbridge_port=ovsbridge_port, + bridge=bridge)) + + subprocess.check_call(["ifup", linuxbridge_port]) + add_bridge_port(name, linuxbridge_port) + + +def is_linuxbridge_interface(port): + ''' Check if the interface is a linuxbridge bridge + :param port: Name of an interface to check whether it is a Linux bridge + :returns: True if port is a Linux bridge''' + + if os.path.exists('/sys/class/net/' + port + '/bridge'): + log('Interface {} is a Linux bridge'.format(port), level=DEBUG) + return True + else: + log('Interface {} is not a Linux bridge'.format(port), level=DEBUG) + return False + + def set_manager(manager): ''' Set the controller for the local openvswitch ''' log('Setting manager for local ovs to {}'.format(manager)) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 401c032..1f4cf42 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -32,6 +32,7 @@ from keystoneclient.v3 import client as keystone_client_v3 from novaclient import exceptions import novaclient.client as nova_client +import novaclient import pika import swiftclient @@ -434,9 +435,14 @@ class OpenStackAmuletUtils(AmuletUtils): self.log.debug('Authenticating nova user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', endpoint_type='publicURL') - return nova_client.Client(NOVA_CLIENT_VERSION, - username=user, api_key=password, - project_id=tenant, auth_url=ep) + if novaclient.__version__[0] >= "7": + return nova_client.Client(NOVA_CLIENT_VERSION, + username=user, password=password, + project_name=tenant, auth_url=ep) + else: + return nova_client.Client(NOVA_CLIENT_VERSION, + username=user, api_key=password, + project_id=tenant, auth_url=ep) def authenticate_swift_user(self, keystone, user, password, tenant): """Authenticates a regular user with swift api.""" diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 4231633..6cdbbbb 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -100,7 +100,10 @@ from charmhelpers.core.unitdata import kv try: import psutil except ImportError: - apt_install('python-psutil', fatal=True) + if six.PY2: + apt_install('python-psutil', fatal=True) + else: + apt_install('python3-psutil', fatal=True) import psutil CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' @@ -392,16 +395,20 @@ class AMQPContext(OSContextGenerator): for rid in relation_ids(self.rel_name): ha_vip_only = False self.related = True + transport_hosts = None + rabbitmq_port = '5672' for unit in related_units(rid): if relation_get('clustered', rid=rid, unit=unit): ctxt['clustered'] = True vip = relation_get('vip', rid=rid, unit=unit) vip = format_ipv6_addr(vip) or vip ctxt['rabbitmq_host'] = vip + transport_hosts = [vip] else: host = relation_get('private-address', rid=rid, unit=unit) host = format_ipv6_addr(host) or host ctxt['rabbitmq_host'] = host + transport_hosts = [host] ctxt.update({ 'rabbitmq_user': username, @@ -413,6 +420,7 @@ class AMQPContext(OSContextGenerator): ssl_port = relation_get('ssl_port', rid=rid, unit=unit) if ssl_port: ctxt['rabbit_ssl_port'] = ssl_port + rabbitmq_port = ssl_port ssl_ca = relation_get('ssl_ca', rid=rid, unit=unit) if ssl_ca: @@ -450,6 +458,20 @@ class AMQPContext(OSContextGenerator): rabbitmq_hosts.append(host) ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts)) + transport_hosts = rabbitmq_hosts + + if transport_hosts: + transport_url_hosts = '' + for host in transport_hosts: + if transport_url_hosts: + format_string = ",{}:{}@{}:{}" + else: + format_string = "{}:{}@{}:{}" + transport_url_hosts += format_string.format( + ctxt['rabbitmq_user'], ctxt['rabbitmq_password'], + host, rabbitmq_port) + ctxt['transport_url'] = "rabbit://{}/{}".format( + transport_url_hosts, vhost) oslo_messaging_flags = conf.get('oslo-messaging-flags', None) if oslo_messaging_flags: @@ -481,13 +503,16 @@ class CephContext(OSContextGenerator): ctxt['auth'] = relation_get('auth', rid=rid, unit=unit) if not ctxt.get('key'): ctxt['key'] = relation_get('key', rid=rid, unit=unit) - ceph_pub_addr = relation_get('ceph-public-address', rid=rid, + + ceph_addrs = relation_get('ceph-public-address', rid=rid, + unit=unit) + if ceph_addrs: + for addr in ceph_addrs.split(' '): + mon_hosts.append(format_ipv6_addr(addr) or addr) + else: + priv_addr = relation_get('private-address', rid=rid, unit=unit) - unit_priv_addr = relation_get('private-address', rid=rid, - unit=unit) - ceph_addr = ceph_pub_addr or unit_priv_addr - ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr - mon_hosts.append(ceph_addr) + mon_hosts.append(format_ipv6_addr(priv_addr) or priv_addr) ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts)) diff --git a/hooks/charmhelpers/contrib/openstack/ha/utils.py b/hooks/charmhelpers/contrib/openstack/ha/utils.py index 1f5310b..254a90e 100644 --- a/hooks/charmhelpers/contrib/openstack/ha/utils.py +++ b/hooks/charmhelpers/contrib/openstack/ha/utils.py @@ -126,3 +126,14 @@ def assert_charm_supports_dns_ha(): status_set('blocked', msg) raise DNSHAException(msg) return True + + +def expect_ha(): + """ Determine if the unit expects to be in HA + + Check for VIP or dns-ha settings which indicate the unit should expect to + be related to hacluster. + + @returns boolean + """ + return config('vip') or config('dns-ha') diff --git a/hooks/charmhelpers/contrib/openstack/keystone.py b/hooks/charmhelpers/contrib/openstack/keystone.py new file mode 100644 index 0000000..a15a03f --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/keystone.py @@ -0,0 +1,178 @@ +#!/usr/bin/python +# +# Copyright 2017 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six +from charmhelpers.fetch import apt_install +from charmhelpers.contrib.openstack.context import IdentityServiceContext +from charmhelpers.core.hookenv import ( + log, + ERROR, +) + + +def get_api_suffix(api_version): + """Return the formatted api suffix for the given version + @param api_version: version of the keystone endpoint + @returns the api suffix formatted according to the given api + version + """ + return 'v2.0' if api_version in (2, "2.0") else 'v3' + + +def format_endpoint(schema, addr, port, api_version): + """Return a formatted keystone endpoint + @param schema: http or https + @param addr: ipv4/ipv6 host of the keystone service + @param port: port of the keystone service + @param api_version: 2 or 3 + @returns a fully formatted keystone endpoint + """ + return '{}://{}:{}/{}/'.format(schema, addr, port, + get_api_suffix(api_version)) + + +def get_keystone_manager(endpoint, api_version, **kwargs): + """Return a keystonemanager for the correct API version + + @param endpoint: the keystone endpoint to point client at + @param api_version: version of the keystone api the client should use + @param kwargs: token or username/tenant/password information + @returns keystonemanager class used for interrogating keystone + """ + if api_version == 2: + return KeystoneManager2(endpoint, **kwargs) + if api_version == 3: + return KeystoneManager3(endpoint, **kwargs) + raise ValueError('No manager found for api version {}'.format(api_version)) + + +def get_keystone_manager_from_identity_service_context(): + """Return a keystonmanager generated from a + instance of charmhelpers.contrib.openstack.context.IdentityServiceContext + @returns keystonamenager instance + """ + context = IdentityServiceContext()() + if not context: + msg = "Identity service context cannot be generated" + log(msg, level=ERROR) + raise ValueError(msg) + + endpoint = format_endpoint(context['service_protocol'], + context['service_host'], + context['service_port'], + context['api_version']) + + if context['api_version'] in (2, "2.0"): + api_version = 2 + else: + api_version = 3 + + return get_keystone_manager(endpoint, api_version, + username=context['admin_user'], + password=context['admin_password'], + tenant_name=context['admin_tenant_name']) + + +class KeystoneManager(object): + + def resolve_service_id(self, service_name=None, service_type=None): + """Find the service_id of a given service""" + services = [s._info for s in self.api.services.list()] + + service_name = service_name.lower() + for s in services: + name = s['name'].lower() + if service_type and service_name: + if (service_name == name and service_type == s['type']): + return s['id'] + elif service_name and service_name == name: + return s['id'] + elif service_type and service_type == s['type']: + return s['id'] + return None + + def service_exists(self, service_name=None, service_type=None): + """Determine if the given service exists on the service list""" + return self.resolve_service_id(service_name, service_type) is not None + + +class KeystoneManager2(KeystoneManager): + + def __init__(self, endpoint, **kwargs): + try: + from keystoneclient.v2_0 import client + from keystoneclient.auth.identity import v2 + from keystoneclient import session + except ImportError: + if six.PY2: + apt_install(["python-keystoneclient"], fatal=True) + else: + apt_install(["python3-keystoneclient"], fatal=True) + + from keystoneclient.v2_0 import client + from keystoneclient.auth.identity import v2 + from keystoneclient import session + + self.api_version = 2 + + token = kwargs.get("token", None) + if token: + api = client.Client(endpoint=endpoint, token=token) + else: + auth = v2.Password(username=kwargs.get("username"), + password=kwargs.get("password"), + tenant_name=kwargs.get("tenant_name"), + auth_url=endpoint) + sess = session.Session(auth=auth) + api = client.Client(session=sess) + + self.api = api + + +class KeystoneManager3(KeystoneManager): + + def __init__(self, endpoint, **kwargs): + try: + from keystoneclient.v3 import client + from keystoneclient.auth import token_endpoint + from keystoneclient import session + from keystoneclient.auth.identity import v3 + except ImportError: + if six.PY2: + apt_install(["python-keystoneclient"], fatal=True) + else: + apt_install(["python3-keystoneclient"], fatal=True) + + from keystoneclient.v3 import client + from keystoneclient.auth import token_endpoint + from keystoneclient import session + from keystoneclient.auth.identity import v3 + + self.api_version = 3 + + token = kwargs.get("token", None) + if token: + auth = token_endpoint.Token(endpoint=endpoint, + token=token) + sess = session.Session(auth=auth) + else: + auth = v3.Password(auth_url=endpoint, + user_id=kwargs.get("username"), + password=kwargs.get("password"), + project_id=kwargs.get("tenant_name")) + sess = session.Session(auth=auth) + + self.api = client.Client(session=sess) diff --git a/hooks/charmhelpers/contrib/openstack/templating.py b/hooks/charmhelpers/contrib/openstack/templating.py index 8958895..934baf5 100644 --- a/hooks/charmhelpers/contrib/openstack/templating.py +++ b/hooks/charmhelpers/contrib/openstack/templating.py @@ -28,7 +28,10 @@ try: from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions except ImportError: apt_update(fatal=True) - apt_install('python-jinja2', fatal=True) + if six.PY2: + apt_install('python-jinja2', fatal=True) + else: + apt_install('python3-jinja2', fatal=True) from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions @@ -207,7 +210,10 @@ class OSConfigRenderer(object): # if this code is running, the object is created pre-install hook. # jinja2 shouldn't get touched until the module is reloaded on next # hook execution, with proper jinja2 bits successfully imported. - apt_install('python-jinja2') + if six.PY2: + apt_install('python-jinja2') + else: + apt_install('python3-jinja2') def register(self, config_file, contexts): """ diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 80219d6..7e8ecff 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -153,7 +153,7 @@ SWIFT_CODENAMES = OrderedDict([ ('newton', ['2.8.0', '2.9.0', '2.10.0']), ('ocata', - ['2.11.0', '2.12.0']), + ['2.11.0', '2.12.0', '2.13.0']), ]) # >= Liberty version->codename mapping diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index e29bd1b..6e95028 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -16,6 +16,7 @@ # limitations under the License. import os +import six import subprocess import sys @@ -39,7 +40,10 @@ def pip_execute(*args, **kwargs): from pip import main as _pip_execute except ImportError: apt_update() - apt_install('python-pip') + if six.PY2: + apt_install('python-pip') + else: + apt_install('python3-pip') from pip import main as _pip_execute _pip_execute(*args, **kwargs) finally: @@ -136,7 +140,10 @@ def pip_list(): def pip_create_virtualenv(path=None): """Create an isolated Python environment.""" - apt_install('python-virtualenv') + if six.PY2: + apt_install('python-virtualenv') + else: + apt_install('python3-virtualenv') if path: venv_path = path diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index edbb72f..05edfa5 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -306,6 +306,8 @@ SYSTEMD_SYSTEM = '/run/systemd/system' def init_is_systemd(): """Return True if the host system uses systemd, False otherwise.""" + if lsb_release()['DISTRIB_CODENAME'] == 'trusty': + return False return os.path.isdir(SYSTEMD_SYSTEM) diff --git a/hooks/charmhelpers/fetch/snap.py b/hooks/charmhelpers/fetch/snap.py new file mode 100644 index 0000000..23c707b --- /dev/null +++ b/hooks/charmhelpers/fetch/snap.py @@ -0,0 +1,122 @@ +# Copyright 2014-2017 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Charm helpers snap for classic charms. + +If writing reactive charms, use the snap layer: +https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html +""" +import subprocess +from os import environ +from time import sleep +from charmhelpers.core.hookenv import log + +__author__ = 'Joseph Borg ' + +SNAP_NO_LOCK = 1 # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved). +SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks. +SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. + + +class CouldNotAcquireLockException(Exception): + pass + + +def _snap_exec(commands): + """ + Execute snap commands. + + :param commands: List commands + :return: Integer exit code + """ + assert type(commands) == list + + retry_count = 0 + return_code = None + + while return_code is None or return_code == SNAP_NO_LOCK: + try: + return_code = subprocess.check_call(['snap'] + commands, env=environ) + except subprocess.CalledProcessError as e: + retry_count += + 1 + if retry_count > SNAP_NO_LOCK_RETRY_COUNT: + raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT) + return_code = e.returncode + log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN') + sleep(SNAP_NO_LOCK_RETRY_DELAY) + + return return_code + + +def snap_install(packages, *flags): + """ + Install a snap package. + + :param packages: String or List String package name + :param flags: List String flags to pass to install command + :return: Integer return code from snap + """ + if type(packages) is not list: + packages = [packages] + + flags = list(flags) + + message = 'Installing snap(s) "%s"' % ', '.join(packages) + if flags: + message += ' with option(s) "%s"' % ', '.join(flags) + + log(message, level='INFO') + return _snap_exec(['install'] + flags + packages) + + +def snap_remove(packages, *flags): + """ + Remove a snap package. + + :param packages: String or List String package name + :param flags: List String flags to pass to remove command + :return: Integer return code from snap + """ + if type(packages) is not list: + packages = [packages] + + flags = list(flags) + + message = 'Removing snap(s) "%s"' % ', '.join(packages) + if flags: + message += ' with options "%s"' % ', '.join(flags) + + log(message, level='INFO') + return _snap_exec(['remove'] + flags + packages) + + +def snap_refresh(packages, *flags): + """ + Refresh / Update snap package. + + :param packages: String or List String package name + :param flags: List String flags to pass to refresh command + :return: Integer return code from snap + """ + if type(packages) is not list: + packages = [packages] + + flags = list(flags) + + message = 'Refreshing snap(s) "%s"' % ', '.join(packages) + if flags: + message += ' with options "%s"' % ', '.join(flags) + + log(message, level='INFO') + return _snap_exec(['refresh'] + flags + packages) diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py index 39b9b80..82ac80f 100644 --- a/hooks/charmhelpers/fetch/ubuntu.py +++ b/hooks/charmhelpers/fetch/ubuntu.py @@ -116,8 +116,8 @@ CLOUD_ARCHIVE_POCKETS = { } APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. -APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks. -APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. +CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries. +CMD_RETRY_COUNT = 30 # Retry a failing fatal command X times. def filter_installed_packages(packages): @@ -249,7 +249,8 @@ def add_source(source, key=None): source.startswith('http') or source.startswith('deb ') or source.startswith('cloud-archive:')): - subprocess.check_call(['add-apt-repository', '--yes', source]) + cmd = ['add-apt-repository', '--yes', source] + _run_with_retries(cmd) elif source.startswith('cloud:'): install(filter_installed_packages(['ubuntu-cloud-keyring']), fatal=True) @@ -286,41 +287,60 @@ def add_source(source, key=None): key]) -def _run_apt_command(cmd, fatal=False): - """Run an APT command. - - Checks the output and retries if the fatal flag is set - to True. +def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,), + retry_message="", cmd_env=None): + """Run a command and retry until success or max_retries is reached. :param: cmd: str: The apt command to run. + :param: max_retries: int: The number of retries to attempt on a fatal + command. Defaults to CMD_RETRY_COUNT. + :param: retry_exitcodes: tuple: Optional additional exit codes to retry. + Defaults to retry on exit code 1. + :param: retry_message: str: Optional log prefix emitted during retries. + :param: cmd_env: dict: Environment variables to add to the command run. + """ + + env = os.environ.copy() + if cmd_env: + env.update(cmd_env) + + if not retry_message: + retry_message = "Failed executing '{}'".format(" ".join(cmd)) + retry_message += ". Will retry in {} seconds".format(CMD_RETRY_DELAY) + + retry_count = 0 + result = None + + retry_results = (None,) + retry_exitcodes + while result in retry_results: + try: + result = subprocess.check_call(cmd, env=env) + except subprocess.CalledProcessError as e: + retry_count = retry_count + 1 + if retry_count > max_retries: + raise + result = e.returncode + log(retry_message) + time.sleep(CMD_RETRY_DELAY) + + +def _run_apt_command(cmd, fatal=False): + """Run an apt command with optional retries. + :param: fatal: bool: Whether the command's output should be checked and retried. """ - env = os.environ.copy() - - if 'DEBIAN_FRONTEND' not in env: - env['DEBIAN_FRONTEND'] = 'noninteractive' + # Provide DEBIAN_FRONTEND=noninteractive if not present in the environment. + cmd_env = { + 'DEBIAN_FRONTEND': os.environ.get('DEBIAN_FRONTEND', 'noninteractive')} if fatal: - retry_count = 0 - result = None - - # If the command is considered "fatal", we need to retry if the apt - # lock was not acquired. - - while result is None or result == APT_NO_LOCK: - try: - result = subprocess.check_call(cmd, env=env) - except subprocess.CalledProcessError as e: - retry_count = retry_count + 1 - if retry_count > APT_NO_LOCK_RETRY_COUNT: - raise - result = e.returncode - log("Couldn't acquire DPKG lock. Will retry in {} seconds." - "".format(APT_NO_LOCK_RETRY_DELAY)) - time.sleep(APT_NO_LOCK_RETRY_DELAY) - + _run_with_retries( + cmd, cmd_env=cmd_env, retry_exitcodes=(1, APT_NO_LOCK,), + retry_message="Couldn't acquire DPKG lock") else: + env = os.environ.copy() + env.update(cmd_env) subprocess.call(cmd, env=env) diff --git a/metadata.yaml b/metadata.yaml index 0391611..da6ae2b 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -12,6 +12,7 @@ tags: - openstack series: - xenial + - zesty - trusty provides: cloud-compute: diff --git a/tests/gate-basic-xenial-ocata b/tests/gate-basic-xenial-ocata old mode 100644 new mode 100755