Sync charm-helpers

Change-Id: I04fd37042b1510193a9b2f50c36cf97d880813cc

hooks/charmhelpers/__init__.py

@@ -23,22 +23,22 @@ import subprocess
 import sys
 
 try:
-    import six  # flake8: noqa
+    import six  # NOQA:F401
 except ImportError:
     if sys.version_info.major == 2:
         subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
     else:
         subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
-    import six  # flake8: noqa
+    import six  # NOQA:F401
 
 try:
-    import yaml  # flake8: noqa
+    import yaml  # NOQA:F401
 except ImportError:
     if sys.version_info.major == 2:
         subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
     else:
         subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
-    import yaml  # flake8: noqa
+    import yaml  # NOQA:F401
 
 
 # Holds a list of mapping of mangled function names that have been deprecated

hooks/charmhelpers/contrib/hahelpers/apache.py

@@ -23,8 +23,8 @@
 #
 
 import os
-import subprocess
 
+from charmhelpers.core import host
 from charmhelpers.core.hookenv import (
     config as config_get,
     relation_get,
@@ -83,14 +83,4 @@ def retrieve_ca_cert(cert_file):
 
 
 def install_ca_cert(ca_cert):
-    if ca_cert:
+    host.install_ca_cert(ca_cert, 'keystone_juju_ca_cert')
-        cert_file = ('/usr/local/share/ca-certificates/'
-                     'keystone_juju_ca_cert.crt')
-        old_cert = retrieve_ca_cert(cert_file)
-        if old_cert and old_cert == ca_cert:
-            log("CA cert is the same as installed version", level=INFO)
-        else:
-            log("Installing new CA cert", level=INFO)
-            with open(cert_file, 'wb') as crt:
-                crt.write(ca_cert)
-            subprocess.check_call(['update-ca-certificates', '--fresh'])

hooks/charmhelpers/contrib/openstack/amulet/utils.py

@@ -24,7 +24,8 @@ import urlparse
 
 import cinderclient.v1.client as cinder_client
 import cinderclient.v2.client as cinder_clientv2
-import glanceclient.v1.client as glance_client
+import glanceclient.v1 as glance_client
+import glanceclient.v2 as glance_clientv2
 import heatclient.v1.client as heat_client
 from keystoneclient.v2_0 import client as keystone_client
 from keystoneauth1.identity import (
@@ -617,13 +618,13 @@ class OpenStackAmuletUtils(AmuletUtils):
         return self.authenticate_keystone(keystone_ip, user, password,
                                           project_name=tenant)
 
-    def authenticate_glance_admin(self, keystone):
+    def authenticate_glance_admin(self, keystone, force_v1_client=False):
         """Authenticates admin user with glance."""
         self.log.debug('Authenticating glance admin...')
         ep = keystone.service_catalog.url_for(service_type='image',
                                               interface='adminURL')
-        if keystone.session:
+        if not force_v1_client and keystone.session:
-            return glance_client.Client(ep, session=keystone.session)
+            return glance_clientv2.Client("2", session=keystone.session)
         else:
             return glance_client.Client(ep, token=keystone.auth_token)
 
@@ -679,18 +680,30 @@ class OpenStackAmuletUtils(AmuletUtils):
             nova.flavors.create(name, ram, vcpus, disk, flavorid,
                                 ephemeral, swap, rxtx_factor, is_public)
 
-    def create_cirros_image(self, glance, image_name):
+    def glance_create_image(self, glance, image_name, image_url,
-        """Download the latest cirros image and upload it to glance,
+                            download_dir='tests',
-        validate and return a resource pointer.
+                            hypervisor_type=None,
+                            disk_format='qcow2',
+                            architecture='x86_64',
+                            container_format='bare'):
+        """Download an image and upload it to glance, validate its status
+        and return an image object pointer. KVM defaults, can override for
+        LXD.
 
-        :param glance: pointer to authenticated glance connection
+        :param glance: pointer to authenticated glance api connection
         :param image_name: display name for new image
+        :param image_url: url to retrieve
+        :param download_dir: directory to store downloaded image file
+        :param hypervisor_type: glance image hypervisor property
+        :param disk_format: glance image disk format
+        :param architecture: glance image architecture property
+        :param container_format: glance image container format
         :returns: glance image pointer
         """
-        self.log.debug('Creating glance cirros image '
+        self.log.debug('Creating glance image ({}) from '
-                       '({})...'.format(image_name))
+                       '{}...'.format(image_name, image_url))
 
-        # Download cirros image
+        # Download image
         http_proxy = os.getenv('AMULET_HTTP_PROXY')
         self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
         if http_proxy:
@@ -699,22 +712,34 @@ class OpenStackAmuletUtils(AmuletUtils):
         else:
             opener = urllib.FancyURLopener()
 
-        f = opener.open('http://download.cirros-cloud.net/version/released')
+        abs_file_name = os.path.join(download_dir, image_name)
-        version = f.read().strip()
+        if not os.path.exists(abs_file_name):
-        cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
+            opener.retrieve(image_url, abs_file_name)
-        local_path = os.path.join('tests', cirros_img)
-
-        if not os.path.exists(local_path):
-            cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
-                                                  version, cirros_img)
-            opener.retrieve(cirros_url, local_path)
-        f.close()
 
         # Create glance image
-        with open(local_path) as f:
+        glance_properties = {
-            image = glance.images.create(name=image_name, is_public=True,
+            'architecture': architecture,
-                                         disk_format='qcow2',
+        }
-                                         container_format='bare', data=f)
+        if hypervisor_type:
+            glance_properties['hypervisor_type'] = hypervisor_type
+        # Create glance image
+        if float(glance.version) < 2.0:
+            with open(abs_file_name) as f:
+                image = glance.images.create(
+                    name=image_name,
+                    is_public=True,
+                    disk_format=disk_format,
+                    container_format=container_format,
+                    properties=glance_properties,
+                    data=f)
+        else:
+            image = glance.images.create(
+                name=image_name,
+                visibility="public",
+                disk_format=disk_format,
+                container_format=container_format)
+            glance.images.upload(image.id, open(abs_file_name, 'rb'))
+            glance.images.update(image.id, **glance_properties)
 
         # Wait for image to reach active status
         img_id = image.id
@@ -729,24 +754,68 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Validating image attributes...')
         val_img_name = glance.images.get(img_id).name
         val_img_stat = glance.images.get(img_id).status
-        val_img_pub = glance.images.get(img_id).is_public
         val_img_cfmt = glance.images.get(img_id).container_format
         val_img_dfmt = glance.images.get(img_id).disk_format
+
+        if float(glance.version) < 2.0:
+            val_img_pub = glance.images.get(img_id).is_public
+        else:
+            val_img_pub = glance.images.get(img_id).visibility == "public"
+
         msg_attr = ('Image attributes - name:{} public:{} id:{} stat:{} '
                     'container fmt:{} disk fmt:{}'.format(
                         val_img_name, val_img_pub, img_id,
                         val_img_stat, val_img_cfmt, val_img_dfmt))
 
         if val_img_name == image_name and val_img_stat == 'active' \
-                and val_img_pub is True and val_img_cfmt == 'bare' \
+                and val_img_pub is True and val_img_cfmt == container_format \
-                and val_img_dfmt == 'qcow2':
+                and val_img_dfmt == disk_format:
             self.log.debug(msg_attr)
         else:
-            msg = ('Volume validation failed, {}'.format(msg_attr))
+            msg = ('Image validation failed, {}'.format(msg_attr))
             amulet.raise_status(amulet.FAIL, msg=msg)
 
         return image
 
+    def create_cirros_image(self, glance, image_name, hypervisor_type=None):
+        """Download the latest cirros image and upload it to glance,
+        validate and return a resource pointer.
+
+        :param glance: pointer to authenticated glance connection
+        :param image_name: display name for new image
+        :param hypervisor_type: glance image hypervisor property
+        :returns: glance image pointer
+        """
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'glance_create_image instead of '
+                      'create_cirros_image.')
+
+        self.log.debug('Creating glance cirros image '
+                       '({})...'.format(image_name))
+
+        # Get cirros image URL
+        http_proxy = os.getenv('AMULET_HTTP_PROXY')
+        self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
+        if http_proxy:
+            proxies = {'http': http_proxy}
+            opener = urllib.FancyURLopener(proxies)
+        else:
+            opener = urllib.FancyURLopener()
+
+        f = opener.open('http://download.cirros-cloud.net/version/released')
+        version = f.read().strip()
+        cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
+        cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
+                                              version, cirros_img)
+        f.close()
+
+        return self.glance_create_image(
+            glance,
+            image_name,
+            cirros_url,
+            hypervisor_type=hypervisor_type)
+
     def delete_image(self, glance, image):
         """Delete the specified image."""
 
@@ -998,6 +1067,9 @@ class OpenStackAmuletUtils(AmuletUtils):
                                cmd, code, output))
             amulet.raise_status(amulet.FAIL, msg=msg)
 
+        # For mimic ceph osd lspools output
+        output = output.replace("\n", ",")
+
         # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance,
         for pool in str(output).split(','):
             pool_id_name = pool.split(' ')

hooks/charmhelpers/contrib/openstack/cert_utils.py

@@ -25,7 +25,9 @@ from charmhelpers.core.hookenv import (
     local_unit,
     network_get_primary_address,
     config,
+    related_units,
     relation_get,
+    relation_ids,
     unit_get,
     NoNetworkBinding,
     log,
@@ -225,3 +227,49 @@ def process_certificates(service_name, relation_id, unit,
         create_ip_cert_links(
             ssl_dir,
             custom_hostname_link=custom_hostname_link)
+
+
+def get_requests_for_local_unit(relation_name=None):
+    """Extract any certificates data targeted at this unit down relation_name.
+
+    :param relation_name: str Name of relation to check for data.
+    :returns: List of bundles of certificates.
+    :rtype: List of dicts
+    """
+    local_name = local_unit().replace('/', '_')
+    raw_certs_key = '{}.processed_requests'.format(local_name)
+    relation_name = relation_name or 'certificates'
+    bundles = []
+    for rid in relation_ids(relation_name):
+        for unit in related_units(rid):
+            data = relation_get(rid=rid, unit=unit)
+            if data.get(raw_certs_key):
+                bundles.append({
+                    'ca': data['ca'],
+                    'chain': data.get('chain'),
+                    'certs': json.loads(data[raw_certs_key])})
+    return bundles
+
+
+def get_bundle_for_cn(cn, relation_name=None):
+    """Extract certificates for the given cn.
+
+    :param cn: str Canonical Name on certificate.
+    :param relation_name: str Relation to check for certificates down.
+    :returns: Dictionary of certificate data,
+    :rtype: dict.
+    """
+    entries = get_requests_for_local_unit(relation_name)
+    cert_bundle = {}
+    for entry in entries:
+        for _cn, bundle in entry['certs'].items():
+            if _cn == cn:
+                cert_bundle = {
+                    'cert': bundle['cert'],
+                    'key': bundle['key'],
+                    'chain': entry['chain'],
+                    'ca': entry['ca']}
+                break
+        if cert_bundle:
+            break
+    return cert_bundle

hooks/charmhelpers/contrib/openstack/context.py

@@ -642,7 +642,7 @@ class HAProxyContext(OSContextGenerator):
             return {}
 
         l_unit = local_unit().replace('/', '-')
-        cluster_hosts = {}
+        cluster_hosts = collections.OrderedDict()
 
         # NOTE(jamespage): build out map of configured network endpoints
         # and associated backends
@@ -1519,6 +1519,10 @@ class NeutronAPIContext(OSContextGenerator):
                 'rel_key': 'enable-qos',
                 'default': False,
             },
+            'enable_nsg_logging': {
+                'rel_key': 'enable-nsg-logging',
+                'default': False,
+            },
         }
         ctxt = self.get_neutron_options({})
         for rid in relation_ids('neutron-plugin-api'):
@@ -1530,10 +1534,15 @@ class NeutronAPIContext(OSContextGenerator):
                 if 'l2-population' in rdata:
                     ctxt.update(self.get_neutron_options(rdata))
 
+        extension_drivers = []
+
         if ctxt['enable_qos']:
-            ctxt['extension_drivers'] = 'qos'
+            extension_drivers.append('qos')
-        else:
+
-            ctxt['extension_drivers'] = ''
+        if ctxt['enable_nsg_logging']:
+            extension_drivers.append('log')
+
+        ctxt['extension_drivers'] = ','.join(extension_drivers)
 
         return ctxt
 
@@ -1893,7 +1902,7 @@ class EnsureDirContext(OSContextGenerator):
     Some software requires a user to create a target directory to be
     scanned for drop-in files with a specific format. This is why this
     context is needed to do that before rendering a template.
-   '''
+    '''
 
     def __init__(self, dirname, **kwargs):
         '''Used merely to ensure that a given directory exists.'''
@@ -1903,3 +1912,23 @@ class EnsureDirContext(OSContextGenerator):
     def __call__(self):
         mkdir(self.dirname, **self.kwargs)
         return {}
+
+
+class VersionsContext(OSContextGenerator):
+    """Context to return the openstack and operating system versions.
+
+    """
+    def __init__(self, pkg='python-keystone'):
+        """Initialise context.
+
+        :param pkg: Package to extrapolate openstack version from.
+        :type pkg: str
+        """
+        self.pkg = pkg
+
+    def __call__(self):
+        ostack = os_release(self.pkg, base='icehouse')
+        osystem = lsb_release()['DISTRIB_CODENAME'].lower()
+        return {
+            'openstack_release': ostack,
+            'operating_system_release': osystem}

hooks/charmhelpers/contrib/openstack/ha/utils.py

@@ -28,6 +28,7 @@ import json
 import re
 
 from charmhelpers.core.hookenv import (
+    expected_related_units,
     log,
     relation_set,
     charm_name,
@@ -110,12 +111,17 @@ def assert_charm_supports_dns_ha():
 def expect_ha():
     """ Determine if the unit expects to be in HA
 
-    Check for VIP or dns-ha settings which indicate the unit should expect to
+    Check juju goal-state if ha relation is expected, check for VIP or dns-ha
-    be related to hacluster.
+    settings which indicate the unit should expect to be related to hacluster.
 
     @returns boolean
     """
-    return config('vip') or config('dns-ha')
+    ha_related_units = []
+    try:
+        ha_related_units = list(expected_related_units(reltype='ha'))
+    except (NotImplementedError, KeyError):
+        pass
+    return len(ha_related_units) > 0 or config('vip') or config('dns-ha')
 
 
 def generate_ha_relation_data(service):

hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka

@@ -1,12 +1,14 @@
 {% if auth_host -%}
 [keystone_authtoken]
-auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
-auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
 auth_type = password
 {% if api_version == "3" -%}
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/v3
+auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v3
 project_domain_name = {{ admin_domain_name }}
 user_domain_name = {{ admin_domain_name }}
 {% else -%}
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
+auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
 project_domain_name = default
 user_domain_name = default
 {% endif -%}

hooks/charmhelpers/contrib/openstack/utils.py

@@ -186,7 +186,7 @@ SWIFT_CODENAMES = OrderedDict([
     ('queens',
         ['2.16.0', '2.17.0']),
     ('rocky',
-        ['2.18.0']),
+        ['2.18.0', '2.19.0']),
 ])
 
 # >= Liberty version->codename mapping
@@ -375,7 +375,7 @@ def get_swift_codename(version):
         return codenames[0]
 
     # NOTE: fallback - attempt to match with just major.minor version
-    match = re.match('^(\d+)\.(\d+)', version)
+    match = re.match(r'^(\d+)\.(\d+)', version)
     if match:
         major_minor_version = match.group(0)
         for codename, versions in six.iteritems(SWIFT_CODENAMES):
@@ -395,7 +395,7 @@ def get_os_codename_package(package, fatal=True):
             out = subprocess.check_output(cmd)
             if six.PY3:
                 out = out.decode('UTF-8')
-        except subprocess.CalledProcessError as e:
+        except subprocess.CalledProcessError:
             return None
         lines = out.split('\n')
         for line in lines:
@@ -427,11 +427,11 @@ def get_os_codename_package(package, fatal=True):
     vers = apt.upstream_version(pkg.current_ver.ver_str)
     if 'swift' in pkg.name:
         # Fully x.y.z match for swift versions
-        match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
+        match = re.match(r'^(\d+)\.(\d+)\.(\d+)', vers)
     else:
         # x.y match only for 20XX.X
         # and ignore patch level for other packages
-        match = re.match('^(\d+)\.(\d+)', vers)
+        match = re.match(r'^(\d+)\.(\d+)', vers)
 
     if match:
         vers = match.group(0)
@@ -1450,20 +1450,33 @@ def pausable_restart_on_change(restart_map, stopstart=False,
 
     see core.utils.restart_on_change() for more details.
 
+    Note restart_map can be a callable, in which case, restart_map is only
+    evaluated at runtime.  This means that it is lazy and the underlying
+    function won't be called if the decorated function is never called.  Note,
+    retains backwards compatibility for passing a non-callable dictionary.
+
     @param f: the function to decorate
-    @param restart_map: the restart map {conf_file: [services]}
+    @param restart_map: (optionally callable, which then returns the
+        restart_map) the restart map {conf_file: [services]}
     @param stopstart: DEFAULT false; whether to stop, start or just restart
     @returns decorator to use a restart_on_change with pausability
     """
     def wrap(f):
+        # py27 compatible nonlocal variable.  When py3 only, replace with
+        # nonlocal keyword
+        __restart_map_cache = {'cache': None}
+
         @functools.wraps(f)
         def wrapped_f(*args, **kwargs):
             if is_unit_paused_set():
                 return f(*args, **kwargs)
+            if __restart_map_cache['cache'] is None:
+                __restart_map_cache['cache'] = restart_map() \
+                    if callable(restart_map) else restart_map
             # otherwise, normal restart_on_change functionality
             return restart_on_change_helper(
-                (lambda: f(*args, **kwargs)), restart_map, stopstart,
+                (lambda: f(*args, **kwargs)), __restart_map_cache['cache'],
-                restart_functions)
+                stopstart, restart_functions)
         return wrapped_f
     return wrap
 
@@ -1733,3 +1746,31 @@ def is_unit_upgrading_set():
             return not(not(kv.get('unit-upgrading')))
     except Exception:
         return False
+
+
+def series_upgrade_prepare(pause_unit_helper=None, configs=None):
+    """ Run common series upgrade prepare tasks.
+
+    :param pause_unit_helper: function: Function to pause unit
+    :param configs: OSConfigRenderer object: Configurations
+    :returns None:
+    """
+    set_unit_upgrading()
+    if pause_unit_helper and configs:
+        if not is_unit_paused_set():
+            pause_unit_helper(configs)
+
+
+def series_upgrade_complete(resume_unit_helper=None, configs=None):
+    """ Run common series upgrade complete tasks.
+
+    :param resume_unit_helper: function: Function to resume unit
+    :param configs: OSConfigRenderer object: Configurations
+    :returns None:
+    """
+    clear_unit_paused()
+    clear_unit_upgrading()
+    if configs:
+        configs.write_all()
+        if resume_unit_helper:
+            resume_unit_helper(configs)

hooks/charmhelpers/contrib/storage/linux/loopback.py

@@ -39,7 +39,7 @@ def loopback_devices():
     devs = [d.strip().split(' ') for d in
             check_output(cmd).splitlines() if d != '']
     for dev, _, f in devs:
-        loopbacks[dev.replace(':', '')] = re.search('\((\S+)\)', f).groups()[0]
+        loopbacks[dev.replace(':', '')] = re.search(r'\((\S+)\)', f).groups()[0]
     return loopbacks
 
 

hooks/charmhelpers/core/hookenv.py

@@ -48,6 +48,7 @@ INFO = "INFO"
 DEBUG = "DEBUG"
 TRACE = "TRACE"
 MARKER = object()
+SH_MAX_ARG = 131071
 
 cache = {}
 
@@ -98,7 +99,7 @@ def log(message, level=None):
         command += ['-l', level]
     if not isinstance(message, six.string_types):
         message = repr(message)
-    command += [message]
+    command += [message[:SH_MAX_ARG]]
     # Missing juju-log should not cause failures in unit tests
     # Send log output to stderr
     try:
@@ -509,6 +510,67 @@ def related_units(relid=None):
         subprocess.check_output(units_cmd_line).decode('UTF-8')) or []
 
 
+def expected_peer_units():
+    """Get a generator for units we expect to join peer relation based on
+    goal-state.
+
+    The local unit is excluded from the result to make it easy to gauge
+    completion of all peers joining the relation with existing hook tools.
+
+    Example usage:
+    log('peer {} of {} joined peer relation'
+        .format(len(related_units()),
+                len(list(expected_peer_units()))))
+
+    This function will raise NotImplementedError if used with juju versions
+    without goal-state support.
+
+    :returns: iterator
+    :rtype: types.GeneratorType
+    :raises: NotImplementedError
+    """
+    if not has_juju_version("2.4.0"):
+        # goal-state first appeared in 2.4.0.
+        raise NotImplementedError("goal-state")
+    _goal_state = goal_state()
+    return (key for key in _goal_state['units']
+            if '/' in key and key != local_unit())
+
+
+def expected_related_units(reltype=None):
+    """Get a generator for units we expect to join relation based on
+    goal-state.
+
+    Note that you can not use this function for the peer relation, take a look
+    at expected_peer_units() for that.
+
+    This function will raise KeyError if you request information for a
+    relation type for which juju goal-state does not have information.  It will
+    raise NotImplementedError if used with juju versions without goal-state
+    support.
+
+    Example usage:
+    log('participant {} of {} joined relation {}'
+        .format(len(related_units()),
+                len(list(expected_related_units())),
+                relation_type()))
+
+    :param reltype: Relation type to list data for, default is to list data for
+                    the realtion type we are currently executing a hook for.
+    :type reltype: str
+    :returns: iterator
+    :rtype: types.GeneratorType
+    :raises: KeyError, NotImplementedError
+    """
+    if not has_juju_version("2.4.4"):
+        # goal-state existed in 2.4.0, but did not list individual units to
+        # join a relation in 2.4.1 through 2.4.3. (LP: #1794739)
+        raise NotImplementedError("goal-state relation unit count")
+    reltype = reltype or relation_type()
+    _goal_state = goal_state()
+    return (key for key in _goal_state['relations'][reltype] if '/' in key)
+
+
 @cached
 def relation_for_unit(unit=None, rid=None):
     """Get the json represenation of a unit's relation"""
@@ -997,6 +1059,7 @@ def application_version_set(version):
 
 
 @translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+@cached
 def goal_state():
     """Juju goal state values"""
     cmd = ['goal-state', '--format=json']

hooks/charmhelpers/core/host.py

@@ -34,13 +34,13 @@ import six
 
 from contextlib import contextmanager
 from collections import OrderedDict
-from .hookenv import log, DEBUG, local_unit
+from .hookenv import log, INFO, DEBUG, local_unit, charm_name
 from .fstab import Fstab
 from charmhelpers.osplatform import get_platform
 
 __platform__ = get_platform()
 if __platform__ == "ubuntu":
-    from charmhelpers.core.host_factory.ubuntu import (
+    from charmhelpers.core.host_factory.ubuntu import (  # NOQA:F401
         service_available,
         add_new_group,
         lsb_release,
@@ -48,7 +48,7 @@ if __platform__ == "ubuntu":
         CompareHostReleases,
     )  # flake8: noqa -- ignore F401 for this import
 elif __platform__ == "centos":
-    from charmhelpers.core.host_factory.centos import (
+    from charmhelpers.core.host_factory.centos import (  # NOQA:F401
         service_available,
         add_new_group,
         lsb_release,
@@ -58,6 +58,7 @@ elif __platform__ == "centos":
 
 UPDATEDB_PATH = '/etc/updatedb.conf'
 
+
 def service_start(service_name, **kwargs):
     """Start a system service.
 
@@ -287,8 +288,8 @@ def service_running(service_name, **kwargs):
                 for key, value in six.iteritems(kwargs):
                     parameter = '%s=%s' % (key, value)
                     cmd.append(parameter)
-                output = subprocess.check_output(cmd,
+                output = subprocess.check_output(
-                    stderr=subprocess.STDOUT).decode('UTF-8')
+                    cmd, stderr=subprocess.STDOUT).decode('UTF-8')
             except subprocess.CalledProcessError:
                 return False
             else:
@@ -442,7 +443,7 @@ def add_user_to_group(username, group):
 
 
 def chage(username, lastday=None, expiredate=None, inactive=None,
-           mindays=None, maxdays=None, root=None, warndays=None):
+          mindays=None, maxdays=None, root=None, warndays=None):
     """Change user password expiry information
 
     :param str username: User to update
@@ -482,8 +483,10 @@ def chage(username, lastday=None, expiredate=None, inactive=None,
     cmd.append(username)
     subprocess.check_call(cmd)
 
+
 remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1')
 
+
 def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
     """Replicate the contents of a path"""
     options = options or ['--delete', '--executability']
@@ -535,13 +538,15 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
     # lets see if we can grab the file and compare the context, to avoid doing
     # a write.
     existing_content = None
-    existing_uid, existing_gid = None, None
+    existing_uid, existing_gid, existing_perms = None, None, None
     try:
         with open(path, 'rb') as target:
             existing_content = target.read()
         stat = os.stat(path)
-        existing_uid, existing_gid = stat.st_uid, stat.st_gid
+        existing_uid, existing_gid, existing_perms = (
-    except:
+            stat.st_uid, stat.st_gid, stat.st_mode
+        )
+    except Exception:
         pass
     if content != existing_content:
         log("Writing file {} {}:{} {:o}".format(path, owner, group, perms),
@@ -554,7 +559,7 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
             target.write(content)
         return
     # the contents were the same, but we might still need to change the
-    # ownership.
+    # ownership or permissions.
     if existing_uid != uid:
         log("Changing uid on already existing content: {} -> {}"
             .format(existing_uid, uid), level=DEBUG)
@@ -563,6 +568,10 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
         log("Changing gid on already existing content: {} -> {}"
             .format(existing_gid, gid), level=DEBUG)
         os.chown(path, -1, gid)
+    if existing_perms != perms:
+        log("Changing permissions on existing content: {} -> {}"
+            .format(existing_perms, perms), level=DEBUG)
+        os.chmod(path, perms)
 
 
 def fstab_remove(mp):
@@ -827,7 +836,7 @@ def list_nics(nic_type=None):
         ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
         ip_output = (line.strip() for line in ip_output if line)
 
-        key = re.compile('^[0-9]+:\s+(.+):')
+        key = re.compile(r'^[0-9]+:\s+(.+):')
         for line in ip_output:
             matched = re.search(key, line)
             if matched:
@@ -1040,3 +1049,27 @@ def modulo_distribution(modulo=3, wait=30, non_zero_wait=False):
         return modulo * wait
     else:
         return calculated_wait_time
+
+
+def install_ca_cert(ca_cert, name=None):
+    """
+    Install the given cert as a trusted CA.
+
+    The ``name`` is the stem of the filename where the cert is written, and if
+    not provided, it will default to ``juju-{charm_name}``.
+
+    If the cert is empty or None, or is unchanged, nothing is done.
+    """
+    if not ca_cert:
+        return
+    if not isinstance(ca_cert, bytes):
+        ca_cert = ca_cert.encode('utf8')
+    if not name:
+        name = 'juju-{}'.format(charm_name())
+    cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name)
+    new_hash = hashlib.md5(ca_cert).hexdigest()
+    if file_hash(cert_file) == new_hash:
+        return
+    log("Installing new CA cert at: {}".format(cert_file), level=INFO)
+    write_file(cert_file, ca_cert)
+    subprocess.check_call(['update-ca-certificates', '--fresh'])

hooks/charmhelpers/core/kernel.py

@@ -26,12 +26,12 @@ from charmhelpers.core.hookenv import (
 
 __platform__ = get_platform()
 if __platform__ == "ubuntu":
-    from charmhelpers.core.kernel_factory.ubuntu import (
+    from charmhelpers.core.kernel_factory.ubuntu import (  # NOQA:F401
         persistent_modprobe,
         update_initramfs,
     )  # flake8: noqa -- ignore F401 for this import
 elif __platform__ == "centos":
-    from charmhelpers.core.kernel_factory.centos import (
+    from charmhelpers.core.kernel_factory.centos import (  # NOQA:F401
         persistent_modprobe,
         update_initramfs,
     )  # flake8: noqa -- ignore F401 for this import

hooks/charmhelpers/fetch/__init__.py

@@ -84,6 +84,7 @@ module = "charmhelpers.fetch.%s" % __platform__
 fetch = importlib.import_module(module)
 
 filter_installed_packages = fetch.filter_installed_packages
+filter_missing_packages = fetch.filter_missing_packages
 install = fetch.apt_install
 upgrade = fetch.apt_upgrade
 update = _fetch_update = fetch.apt_update
@@ -96,6 +97,7 @@ if __platform__ == "ubuntu":
     apt_update = fetch.apt_update
     apt_upgrade = fetch.apt_upgrade
     apt_purge = fetch.apt_purge
+    apt_autoremove = fetch.apt_autoremove
     apt_mark = fetch.apt_mark
     apt_hold = fetch.apt_hold
     apt_unhold = fetch.apt_unhold

hooks/charmhelpers/fetch/bzrurl.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import os
-from subprocess import check_call
+from subprocess import STDOUT, check_output
 from charmhelpers.fetch import (
     BaseFetchHandler,
     UnhandledSource,
@@ -55,7 +55,7 @@ class BzrUrlFetchHandler(BaseFetchHandler):
             cmd = ['bzr', 'branch']
             cmd += cmd_opts
             cmd += [source, dest]
-        check_call(cmd)
+        check_output(cmd, stderr=STDOUT)
 
     def install(self, source, dest=None, revno=None):
         url_parts = self.parse_url(source)

hooks/charmhelpers/fetch/giturl.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import os
-from subprocess import check_call, CalledProcessError
+from subprocess import check_output, CalledProcessError, STDOUT
 from charmhelpers.fetch import (
     BaseFetchHandler,
     UnhandledSource,
@@ -50,7 +50,7 @@ class GitUrlFetchHandler(BaseFetchHandler):
             cmd = ['git', 'clone', source, dest, '--branch', branch]
             if depth:
                 cmd.extend(['--depth', depth])
-        check_call(cmd)
+        check_output(cmd, stderr=STDOUT)
 
     def install(self, source, branch="master", dest=None, depth=None):
         url_parts = self.parse_url(source)

hooks/charmhelpers/fetch/ubuntu.py

@@ -189,6 +189,18 @@ def filter_installed_packages(packages):
     return _pkgs
 
 
+def filter_missing_packages(packages):
+    """Return a list of packages that are installed.
+
+    :param packages: list of packages to evaluate.
+    :returns list: Packages that are installed.
+    """
+    return list(
+        set(packages) -
+        set(filter_installed_packages(packages))
+    )
+
+
 def apt_cache(in_memory=True, progress=None):
     """Build and return an apt cache."""
     from apt import apt_pkg
@@ -248,6 +260,14 @@ def apt_purge(packages, fatal=False):
     _run_apt_command(cmd, fatal)
 
 
+def apt_autoremove(purge=True, fatal=False):
+    """Purge one or more packages."""
+    cmd = ['apt-get', '--assume-yes', 'autoremove']
+    if purge:
+        cmd.append('--purge')
+    _run_apt_command(cmd, fatal)
+
+
 def apt_mark(packages, mark, fatal=False):
     """Flag one or more packages using apt-mark."""
     log("Marking {} as {}".format(packages, mark))
@@ -274,7 +294,7 @@ def apt_unhold(packages, fatal=False):
 def import_key(key):
     """Import an ASCII Armor key.
 
-    /!\ A Radix64 format keyid is also supported for backwards
+    A Radix64 format keyid is also supported for backwards
     compatibility, but should never be used; the key retrieval
     mechanism is insecure and subject to man-in-the-middle attacks
     voiding all signature checks using that key.
@@ -434,6 +454,9 @@ def _add_apt_repository(spec):
 
     :param spec: the parameter to pass to add_apt_repository
     """
+    if '{series}' in spec:
+        series = lsb_release()['DISTRIB_CODENAME']
+        spec = spec.replace('{series}', series)
     _run_with_retries(['add-apt-repository', '--yes', spec])