rebase

.bzrignore

@@ -1,3 +1,5 @@
 .coverage
 bin
 tags
+.tox
+.testrepository

.testr.conf

@@ -0,0 +1,8 @@
+[DEFAULT]
+test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
+             OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
+             OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \
+             ${PYTHON:-python} -m subunit.run discover -t ./ ./unit_tests $LISTOPT $IDOPTION
+
+test_id_option=--load-list $IDFILE
+test_list_option=--list

Makefile

@@ -8,10 +8,11 @@ lint:
 
 test:
 	@# Bundletester expects unit tests here.
-	@$(PYTHON) /usr/bin/nosetests --nologcapture --with-coverage unit_tests
+	@$(PYTHON) /usr/bin/nosetests -v --nologcapture --with-coverage unit_tests
 
 functional_test:
 	@echo Starting Amulet tests...
+	@tests/setup/00-setup
 	@juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700
 
 bin/charm_helpers_sync.py:

actions/openstack_upgrade.py

@@ -1,29 +1,15 @@
 #!/usr/bin/python
 
-import traceback
-
-
-from charmhelpers.core.hookenv import (
-    action_set,
-    action_fail,
-    config
-)
-
-from hooks.glance_relations import config_changed
-
 from charmhelpers.contrib.openstack.utils import (
-    juju_log,
+    do_action_openstack_upgrade,
-    git_install_requested,
-    openstack_upgrade_available
 )
 
-from hooks.glance_utils import (
+from hooks.glance_relations import (
-    do_openstack_upgrade,
+    config_changed,
-    register_configs
+    CONFIGS
 )
 
-
+from hooks.glance_utils import do_openstack_upgrade
-CONFIGS = register_configs()
 
 
 def openstack_upgrade():
@@ -33,29 +19,10 @@ def openstack_upgrade():
     For backwards compatibility a config flag must be set for this
     code to run, otherwise a full service level upgrade will fire
     on config-changed."""
-
+    if (do_action_openstack_upgrade('glance-common',
-    if git_install_requested():
+                                    do_openstack_upgrade,
-        action_set({'outcome': 'installed from source, skipped upgrade.'})
+                                    CONFIGS)):
-    else:
+        config_changed()
-        if openstack_upgrade_available('glance-common'):
-            if config('action-managed-upgrade'):
-                juju_log('Upgrading OpenStack release')
-
-                try:
-                    do_openstack_upgrade(CONFIGS)
-                    action_set({'outcome': 'success, upgrade completed.'})
-                except:
-                    action_set({'outcome': 'upgrade failed, see traceback.'})
-                    action_set({'traceback': traceback.format_exc()})
-                    action_fail('do_openstack_upgrade resulted in an '
-                                'unexpected error')
-
-                config_changed()
-            else:
-                action_set({'outcome': 'action-managed-upgrade config is '
-                                       'False, skipped upgrade.'})
-        else:
-            action_set({'outcome': 'no upgrade available.'})
 
 if __name__ == '__main__':
     openstack_upgrade()

charm-helpers-hooks.yaml

@@ -7,7 +7,8 @@ include:
     - contrib.openstack|inc=*
     - contrib.hahelpers
     - contrib.storage.linux.ceph
-    - payload.execd
+    - payload
     - contrib.network.ip
     - contrib.python.packages
     - contrib.charmsupport
+    - core.kernel

charmhelpers/cli/__init__.py

@@ -20,7 +20,7 @@ import sys
 
 from six.moves import zip
 
-from charmhelpers.core import unitdata
+import charmhelpers.core.unitdata
 
 
 class OutputFormatter(object):
@@ -163,8 +163,8 @@ class CommandLine(object):
         if getattr(arguments.func, '_cli_no_output', False):
             output = ''
         self.formatter.format_output(output, arguments.format)
-        if unitdata._KV:
+        if charmhelpers.core.unitdata._KV:
-            unitdata._KV.flush()
+            charmhelpers.core.unitdata._KV.flush()
 
 
 cmdline = CommandLine()

charmhelpers/contrib/charmsupport/nrpe.py

@@ -148,6 +148,13 @@ define service {{
         self.description = description
         self.check_cmd = self._locate_cmd(check_cmd)
 
+    def _get_check_filename(self):
+        return os.path.join(NRPE.nrpe_confdir, '{}.cfg'.format(self.command))
+
+    def _get_service_filename(self, hostname):
+        return os.path.join(NRPE.nagios_exportdir,
+                            'service__{}_{}.cfg'.format(hostname, self.command))
+
     def _locate_cmd(self, check_cmd):
         search_path = (
             '/usr/lib/nagios/plugins',
@@ -163,9 +170,21 @@ define service {{
         log('Check command not found: {}'.format(parts[0]))
         return ''
 
+    def _remove_service_files(self):
+        if not os.path.exists(NRPE.nagios_exportdir):
+            return
+        for f in os.listdir(NRPE.nagios_exportdir):
+            if f.endswith('_{}.cfg'.format(self.command)):
+                os.remove(os.path.join(NRPE.nagios_exportdir, f))
+
+    def remove(self, hostname):
+        nrpe_check_file = self._get_check_filename()
+        if os.path.exists(nrpe_check_file):
+            os.remove(nrpe_check_file)
+        self._remove_service_files()
+
     def write(self, nagios_context, hostname, nagios_servicegroups):
-        nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format(
+        nrpe_check_file = self._get_check_filename()
-            self.command)
         with open(nrpe_check_file, 'w') as nrpe_check_config:
             nrpe_check_config.write("# check {}\n".format(self.shortname))
             nrpe_check_config.write("command[{}]={}\n".format(
@@ -180,9 +199,7 @@ define service {{
 
     def write_service_config(self, nagios_context, hostname,
                              nagios_servicegroups):
-        for f in os.listdir(NRPE.nagios_exportdir):
+        self._remove_service_files()
-            if re.search('.*{}.cfg'.format(self.command), f):
-                os.remove(os.path.join(NRPE.nagios_exportdir, f))
 
         templ_vars = {
             'nagios_hostname': hostname,
@@ -192,8 +209,7 @@ define service {{
             'command': self.command,
         }
         nrpe_service_text = Check.service_template.format(**templ_vars)
-        nrpe_service_file = '{}/service__{}_{}.cfg'.format(
+        nrpe_service_file = self._get_service_filename(hostname)
-            NRPE.nagios_exportdir, hostname, self.command)
         with open(nrpe_service_file, 'w') as nrpe_service_config:
             nrpe_service_config.write(str(nrpe_service_text))
 
@@ -218,12 +234,32 @@ class NRPE(object):
         if hostname:
             self.hostname = hostname
         else:
-            self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
+            nagios_hostname = get_nagios_hostname()
+            if nagios_hostname:
+                self.hostname = nagios_hostname
+            else:
+                self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
         self.checks = []
 
     def add_check(self, *args, **kwargs):
         self.checks.append(Check(*args, **kwargs))
 
+    def remove_check(self, *args, **kwargs):
+        if kwargs.get('shortname') is None:
+            raise ValueError('shortname of check must be specified')
+
+        # Use sensible defaults if they're not specified - these are not
+        # actually used during removal, but they're required for constructing
+        # the Check object; check_disk is chosen because it's part of the
+        # nagios-plugins-basic package.
+        if kwargs.get('check_cmd') is None:
+            kwargs['check_cmd'] = 'check_disk'
+        if kwargs.get('description') is None:
+            kwargs['description'] = ''
+
+        check = Check(*args, **kwargs)
+        check.remove(self.hostname)
+
     def write(self):
         try:
             nagios_uid = pwd.getpwnam('nagios').pw_uid
@@ -260,7 +296,7 @@ def get_nagios_hostcontext(relation_name='nrpe-external-master'):
     :param str relation_name: Name of relation nrpe sub joined to
     """
     for rel in relations_of_type(relation_name):
-        if 'nagios_hostname' in rel:
+        if 'nagios_host_context' in rel:
             return rel['nagios_host_context']
 
 
@@ -301,11 +337,13 @@ def add_init_service_checks(nrpe, services, unit_name):
         upstart_init = '/etc/init/%s.conf' % svc
         sysv_init = '/etc/init.d/%s' % svc
         if os.path.exists(upstart_init):
-            nrpe.add_check(
+            # Don't add a check for these services from neutron-gateway
-                shortname=svc,
+            if svc not in ['ext-port', 'os-charm-phy-nic-mtu']:
-                description='process check {%s}' % unit_name,
+                nrpe.add_check(
-                check_cmd='check_upstart_job %s' % svc
+                    shortname=svc,
-            )
+                    description='process check {%s}' % unit_name,
+                    check_cmd='check_upstart_job %s' % svc
+                )
         elif os.path.exists(sysv_init):
             cronpath = '/etc/cron.d/nagios-service-check-%s' % svc
             cron_file = ('*/5 * * * * root '

charmhelpers/contrib/network/ip.py

@@ -23,7 +23,7 @@ import socket
 from functools import partial
 
 from charmhelpers.core.hookenv import unit_get
-from charmhelpers.fetch import apt_install
+from charmhelpers.fetch import apt_install, apt_update
 from charmhelpers.core.hookenv import (
     log,
     WARNING,
@@ -32,13 +32,15 @@ from charmhelpers.core.hookenv import (
 try:
     import netifaces
 except ImportError:
-    apt_install('python-netifaces')
+    apt_update(fatal=True)
+    apt_install('python-netifaces', fatal=True)
     import netifaces
 
 try:
     import netaddr
 except ImportError:
-    apt_install('python-netaddr')
+    apt_update(fatal=True)
+    apt_install('python-netaddr', fatal=True)
     import netaddr
 
 
@@ -51,7 +53,7 @@ def _validate_cidr(network):
 
 
 def no_ip_found_error_out(network):
-    errmsg = ("No IP address found in network: %s" % network)
+    errmsg = ("No IP address found in network(s): %s" % network)
     raise ValueError(errmsg)
 
 
@@ -59,7 +61,7 @@ def get_address_in_network(network, fallback=None, fatal=False):
     """Get an IPv4 or IPv6 address within the network from the host.
 
     :param network (str): CIDR presentation format. For example,
-        '192.168.1.0/24'.
+        '192.168.1.0/24'. Supports multiple networks as a space-delimited list.
     :param fallback (str): If no address is found, return fallback.
     :param fatal (boolean): If no address is found, fallback is not
         set and fatal is True then exit(1).
@@ -73,24 +75,26 @@ def get_address_in_network(network, fallback=None, fatal=False):
         else:
             return None
 
-    _validate_cidr(network)
+    networks = network.split() or [network]
-    network = netaddr.IPNetwork(network)
+    for network in networks:
-    for iface in netifaces.interfaces():
+        _validate_cidr(network)
-        addresses = netifaces.ifaddresses(iface)
+        network = netaddr.IPNetwork(network)
-        if network.version == 4 and netifaces.AF_INET in addresses:
+        for iface in netifaces.interfaces():
-            addr = addresses[netifaces.AF_INET][0]['addr']
+            addresses = netifaces.ifaddresses(iface)
-            netmask = addresses[netifaces.AF_INET][0]['netmask']
+            if network.version == 4 and netifaces.AF_INET in addresses:
-            cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
+                addr = addresses[netifaces.AF_INET][0]['addr']
-            if cidr in network:
+                netmask = addresses[netifaces.AF_INET][0]['netmask']
-                return str(cidr.ip)
+                cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
+                if cidr in network:
+                    return str(cidr.ip)
 
-        if network.version == 6 and netifaces.AF_INET6 in addresses:
+            if network.version == 6 and netifaces.AF_INET6 in addresses:
-            for addr in addresses[netifaces.AF_INET6]:
+                for addr in addresses[netifaces.AF_INET6]:
-                if not addr['addr'].startswith('fe80'):
+                    if not addr['addr'].startswith('fe80'):
-                    cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
+                        cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
-                                                        addr['netmask']))
+                                                            addr['netmask']))
-                    if cidr in network:
+                        if cidr in network:
-                        return str(cidr.ip)
+                            return str(cidr.ip)
 
     if fallback is not None:
         return fallback

charmhelpers/contrib/openstack/amulet/deployment.py

@@ -14,12 +14,18 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
+import re
+import sys
 import six
 from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
 
+DEBUG = logging.DEBUG
+ERROR = logging.ERROR
+
 
 class OpenStackAmuletDeployment(AmuletDeployment):
     """OpenStack amulet deployment.
@@ -28,9 +34,12 @@ class OpenStackAmuletDeployment(AmuletDeployment):
        that is specifically for use by OpenStack charms.
        """
 
-    def __init__(self, series=None, openstack=None, source=None, stable=True):
+    def __init__(self, series=None, openstack=None, source=None,
+                 stable=True, log_level=DEBUG):
         """Initialize the deployment environment."""
         super(OpenStackAmuletDeployment, self).__init__(series)
+        self.log = self.get_logger(level=log_level)
+        self.log.info('OpenStackAmuletDeployment:  init')
         self.openstack = openstack
         self.source = source
         self.stable = stable
@@ -38,26 +47,55 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         # out.
         self.current_next = "trusty"
 
+    def get_logger(self, name="deployment-logger", level=logging.DEBUG):
+        """Get a logger object that will log to stdout."""
+        log = logging
+        logger = log.getLogger(name)
+        fmt = log.Formatter("%(asctime)s %(funcName)s "
+                            "%(levelname)s: %(message)s")
+
+        handler = log.StreamHandler(stream=sys.stdout)
+        handler.setLevel(level)
+        handler.setFormatter(fmt)
+
+        logger.addHandler(handler)
+        logger.setLevel(level)
+
+        return logger
+
     def _determine_branch_locations(self, other_services):
         """Determine the branch locations for the other services.
 
            Determine if the local branch being tested is derived from its
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
+
+        self.log.info('OpenStackAmuletDeployment:  determine branch locations')
+
+        # Charms outside the lp:~openstack-charmers namespace
         base_charms = ['mysql', 'mongodb', 'nrpe']
 
+        # Force these charms to current series even when using an older series.
+        # ie. Use trusty/nrpe even when series is precise, as the P charm
+        # does not possess the necessary external master config and hooks.
+        force_series_current = ['nrpe']
+
         if self.series in ['precise', 'trusty']:
             base_series = self.series
         else:
             base_series = self.current_next
 
-        if self.stable:
+        for svc in other_services:
-            for svc in other_services:
+            if svc['name'] in force_series_current:
+                base_series = self.current_next
+            # If a location has been explicitly set, use it
+            if svc.get('location'):
+                continue
+            if self.stable:
                 temp = 'lp:charms/{}/{}'
                 svc['location'] = temp.format(base_series,
                                               svc['name'])
-        else:
+            else:
-            for svc in other_services:
                 if svc['name'] in base_charms:
                     temp = 'lp:charms/{}/{}'
                     svc['location'] = temp.format(base_series,
@@ -66,10 +104,13 @@ class OpenStackAmuletDeployment(AmuletDeployment):
                     temp = 'lp:~openstack-charmers/charms/{}/{}/next'
                     svc['location'] = temp.format(self.current_next,
                                                   svc['name'])
+
         return other_services
 
     def _add_services(self, this_service, other_services):
         """Add services to the deployment and set openstack-origin/source."""
+        self.log.info('OpenStackAmuletDeployment:  adding services')
+
         other_services = self._determine_branch_locations(other_services)
 
         super(OpenStackAmuletDeployment, self)._add_services(this_service,
@@ -77,29 +118,102 @@ class OpenStackAmuletDeployment(AmuletDeployment):
 
         services = other_services
         services.append(this_service)
+
+        # Charms which should use the source config option
         use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
                       'ceph-osd', 'ceph-radosgw']
-        # Most OpenStack subordinate charms do not expose an origin option
+
-        # as that is controlled by the principle.
+        # Charms which can not use openstack-origin, ie. many subordinates
-        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
+        no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe',
+                     'openvswitch-odl', 'neutron-api-odl', 'odl-controller']
 
         if self.openstack:
             for svc in services:
-                if svc['name'] not in use_source + ignore:
+                if svc['name'] not in use_source + no_origin:
                     config = {'openstack-origin': self.openstack}
                     self.d.configure(svc['name'], config)
 
         if self.source:
             for svc in services:
-                if svc['name'] in use_source and svc['name'] not in ignore:
+                if svc['name'] in use_source and svc['name'] not in no_origin:
                     config = {'source': self.source}
                     self.d.configure(svc['name'], config)
 
     def _configure_services(self, configs):
         """Configure all of the services."""
+        self.log.info('OpenStackAmuletDeployment:  configure services')
         for service, config in six.iteritems(configs):
             self.d.configure(service, config)
 
+    def _auto_wait_for_status(self, message=None, exclude_services=None,
+                              include_only=None, timeout=1800):
+        """Wait for all units to have a specific extended status, except
+        for any defined as excluded.  Unless specified via message, any
+        status containing any case of 'ready' will be considered a match.
+
+        Examples of message usage:
+
+          Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
+              message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
+
+          Wait for all units to reach this status (exact match):
+              message = re.compile('^Unit is ready and clustered$')
+
+          Wait for all units to reach any one of these (exact match):
+              message = re.compile('Unit is ready|OK|Ready')
+
+          Wait for at least one unit to reach this status (exact match):
+              message = {'ready'}
+
+        See Amulet's sentry.wait_for_messages() for message usage detail.
+        https://github.com/juju/amulet/blob/master/amulet/sentry.py
+
+        :param message: Expected status match
+        :param exclude_services: List of juju service names to ignore,
+            not to be used in conjuction with include_only.
+        :param include_only: List of juju service names to exclusively check,
+            not to be used in conjuction with exclude_services.
+        :param timeout: Maximum time in seconds to wait for status match
+        :returns: None.  Raises if timeout is hit.
+        """
+        self.log.info('Waiting for extended status on units...')
+
+        all_services = self.d.services.keys()
+
+        if exclude_services and include_only:
+            raise ValueError('exclude_services can not be used '
+                             'with include_only')
+
+        if message:
+            if isinstance(message, re._pattern_type):
+                match = message.pattern
+            else:
+                match = message
+
+            self.log.debug('Custom extended status wait match: '
+                           '{}'.format(match))
+        else:
+            self.log.debug('Default extended status wait match:  contains '
+                           'READY (case-insensitive)')
+            message = re.compile('.*ready.*', re.IGNORECASE)
+
+        if exclude_services:
+            self.log.debug('Excluding services from extended status match: '
+                           '{}'.format(exclude_services))
+        else:
+            exclude_services = []
+
+        if include_only:
+            services = include_only
+        else:
+            services = list(set(all_services) - set(exclude_services))
+
+        self.log.debug('Waiting up to {}s for extended status on services: '
+                       '{}'.format(timeout, services))
+        service_messages = {service: message for service in services}
+        self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+        self.log.info('OK')
+
     def _get_openstack_release(self):
         """Get openstack release.
 
@@ -111,7 +225,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
          self.precise_havana, self.precise_icehouse,
          self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
          self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
-         self.wily_liberty) = range(12)
+         self.wily_liberty, self.trusty_mitaka,
+         self.xenial_mitaka) = range(14)
 
         releases = {
             ('precise', None): self.precise_essex,
@@ -123,9 +238,11 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
             ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
             ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
+            ('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka,
             ('utopic', None): self.utopic_juno,
             ('vivid', None): self.vivid_kilo,
-            ('wily', None): self.wily_liberty}
+            ('wily', None): self.wily_liberty,
+            ('xenial', None): self.xenial_mitaka}
         return releases[(self.series, self.openstack)]
 
     def _get_openstack_release_string(self):
@@ -142,6 +259,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('utopic', 'juno'),
             ('vivid', 'kilo'),
             ('wily', 'liberty'),
+            ('xenial', 'mitaka'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]

charmhelpers/contrib/openstack/amulet/utils.py

@@ -18,6 +18,7 @@ import amulet
 import json
 import logging
 import os
+import re
 import six
 import time
 import urllib
@@ -27,6 +28,7 @@ import glanceclient.v1.client as glance_client
 import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
 import novaclient.v1_1.client as nova_client
+import pika
 import swiftclient
 
 from charmhelpers.contrib.amulet.utils import (
@@ -602,3 +604,382 @@ class OpenStackAmuletUtils(AmuletUtils):
             self.log.debug('Ceph {} samples (OK): '
                            '{}'.format(sample_type, samples))
             return None
+
+    # rabbitmq/amqp specific helpers:
+
+    def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
+        """Wait for rmq units extended status to show cluster readiness,
+        after an optional initial sleep period.  Initial sleep is likely
+        necessary to be effective following a config change, as status
+        message may not instantly update to non-ready."""
+
+        if init_sleep:
+            time.sleep(init_sleep)
+
+        message = re.compile('^Unit is ready and clustered$')
+        deployment._auto_wait_for_status(message=message,
+                                         timeout=timeout,
+                                         include_only=['rabbitmq-server'])
+
+    def add_rmq_test_user(self, sentry_units,
+                          username="testuser1", password="changeme"):
+        """Add a test user via the first rmq juju unit, check connection as
+        the new user against all sentry units.
+
+        :param sentry_units: list of sentry unit pointers
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Adding rmq user ({})...'.format(username))
+
+        # Check that user does not already exist
+        cmd_user_list = 'rabbitmqctl list_users'
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
+        if username in output:
+            self.log.warning('User ({}) already exists, returning '
+                             'gracefully.'.format(username))
+            return
+
+        perms = '".*" ".*" ".*"'
+        cmds = ['rabbitmqctl add_user {} {}'.format(username, password),
+                'rabbitmqctl set_permissions {} {}'.format(username, perms)]
+
+        # Add user via first unit
+        for cmd in cmds:
+            output, _ = self.run_cmd_unit(sentry_units[0], cmd)
+
+        # Check connection against the other sentry_units
+        self.log.debug('Checking user connect against units...')
+        for sentry_unit in sentry_units:
+            connection = self.connect_amqp_by_unit(sentry_unit, ssl=False,
+                                                   username=username,
+                                                   password=password)
+            connection.close()
+
+    def delete_rmq_test_user(self, sentry_units, username="testuser1"):
+        """Delete a rabbitmq user via the first rmq juju unit.
+
+        :param sentry_units: list of sentry unit pointers
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: None if successful or no such user.
+        """
+        self.log.debug('Deleting rmq user ({})...'.format(username))
+
+        # Check that the user exists
+        cmd_user_list = 'rabbitmqctl list_users'
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
+
+        if username not in output:
+            self.log.warning('User ({}) does not exist, returning '
+                             'gracefully.'.format(username))
+            return
+
+        # Delete the user
+        cmd_user_del = 'rabbitmqctl delete_user {}'.format(username)
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_del)
+
+    def get_rmq_cluster_status(self, sentry_unit):
+        """Execute rabbitmq cluster status command on a unit and return
+        the full output.
+
+        :param unit: sentry unit
+        :returns: String containing console output of cluster status command
+        """
+        cmd = 'rabbitmqctl cluster_status'
+        output, _ = self.run_cmd_unit(sentry_unit, cmd)
+        self.log.debug('{} cluster_status:\n{}'.format(
+            sentry_unit.info['unit_name'], output))
+        return str(output)
+
+    def get_rmq_cluster_running_nodes(self, sentry_unit):
+        """Parse rabbitmqctl cluster_status output string, return list of
+        running rabbitmq cluster nodes.
+
+        :param unit: sentry unit
+        :returns: List containing node names of running nodes
+        """
+        # NOTE(beisner): rabbitmqctl cluster_status output is not
+        # json-parsable, do string chop foo, then json.loads that.
+        str_stat = self.get_rmq_cluster_status(sentry_unit)
+        if 'running_nodes' in str_stat:
+            pos_start = str_stat.find("{running_nodes,") + 15
+            pos_end = str_stat.find("]},", pos_start) + 1
+            str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"')
+            run_nodes = json.loads(str_run_nodes)
+            return run_nodes
+        else:
+            return []
+
+    def validate_rmq_cluster_running_nodes(self, sentry_units):
+        """Check that all rmq unit hostnames are represented in the
+        cluster_status output of all units.
+
+        :param host_names: dict of juju unit names to host names
+        :param units: list of sentry unit pointers (all rmq units)
+        :returns: None if successful, otherwise return error message
+        """
+        host_names = self.get_unit_hostnames(sentry_units)
+        errors = []
+
+        # Query every unit for cluster_status running nodes
+        for query_unit in sentry_units:
+            query_unit_name = query_unit.info['unit_name']
+            running_nodes = self.get_rmq_cluster_running_nodes(query_unit)
+
+            # Confirm that every unit is represented in the queried unit's
+            # cluster_status running nodes output.
+            for validate_unit in sentry_units:
+                val_host_name = host_names[validate_unit.info['unit_name']]
+                val_node_name = 'rabbit@{}'.format(val_host_name)
+
+                if val_node_name not in running_nodes:
+                    errors.append('Cluster member check failed on {}: {} not '
+                                  'in {}\n'.format(query_unit_name,
+                                                   val_node_name,
+                                                   running_nodes))
+        if errors:
+            return ''.join(errors)
+
+    def rmq_ssl_is_enabled_on_unit(self, sentry_unit, port=None):
+        """Check a single juju rmq unit for ssl and port in the config file."""
+        host = sentry_unit.info['public-address']
+        unit_name = sentry_unit.info['unit_name']
+
+        conf_file = '/etc/rabbitmq/rabbitmq.config'
+        conf_contents = str(self.file_contents_safe(sentry_unit,
+                                                    conf_file, max_wait=16))
+        # Checks
+        conf_ssl = 'ssl' in conf_contents
+        conf_port = str(port) in conf_contents
+
+        # Port explicitly checked in config
+        if port and conf_port and conf_ssl:
+            self.log.debug('SSL is enabled  @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return True
+        elif port and not conf_port and conf_ssl:
+            self.log.debug('SSL is enabled @{} but not on port {} '
+                           '({})'.format(host, port, unit_name))
+            return False
+        # Port not checked (useful when checking that ssl is disabled)
+        elif not port and conf_ssl:
+            self.log.debug('SSL is enabled  @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return True
+        elif not conf_ssl:
+            self.log.debug('SSL not enabled @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return False
+        else:
+            msg = ('Unknown condition when checking SSL status @{}:{} '
+                   '({})'.format(host, port, unit_name))
+            amulet.raise_status(amulet.FAIL, msg)
+
+    def validate_rmq_ssl_enabled_units(self, sentry_units, port=None):
+        """Check that ssl is enabled on rmq juju sentry units.
+
+        :param sentry_units: list of all rmq sentry units
+        :param port: optional ssl port override to validate
+        :returns: None if successful, otherwise return error message
+        """
+        for sentry_unit in sentry_units:
+            if not self.rmq_ssl_is_enabled_on_unit(sentry_unit, port=port):
+                return ('Unexpected condition:  ssl is disabled on unit '
+                        '({})'.format(sentry_unit.info['unit_name']))
+        return None
+
+    def validate_rmq_ssl_disabled_units(self, sentry_units):
+        """Check that ssl is enabled on listed rmq juju sentry units.
+
+        :param sentry_units: list of all rmq sentry units
+        :returns: True if successful.  Raise on error.
+        """
+        for sentry_unit in sentry_units:
+            if self.rmq_ssl_is_enabled_on_unit(sentry_unit):
+                return ('Unexpected condition:  ssl is enabled on unit '
+                        '({})'.format(sentry_unit.info['unit_name']))
+        return None
+
+    def configure_rmq_ssl_on(self, sentry_units, deployment,
+                             port=None, max_wait=60):
+        """Turn ssl charm config option on, with optional non-default
+        ssl port specification.  Confirm that it is enabled on every
+        unit.
+
+        :param sentry_units: list of sentry units
+        :param deployment: amulet deployment object pointer
+        :param port: amqp port, use defaults if None
+        :param max_wait: maximum time to wait in seconds to confirm
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Setting ssl charm config option:  on')
+
+        # Enable RMQ SSL
+        config = {'ssl': 'on'}
+        if port:
+            config['ssl_port'] = port
+
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
+
+        # Confirm
+        tries = 0
+        ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
+        while ret and tries < (max_wait / 4):
+            time.sleep(4)
+            self.log.debug('Attempt {}: {}'.format(tries, ret))
+            ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
+            tries += 1
+
+        if ret:
+            amulet.raise_status(amulet.FAIL, ret)
+
+    def configure_rmq_ssl_off(self, sentry_units, deployment, max_wait=60):
+        """Turn ssl charm config option off, confirm that it is disabled
+        on every unit.
+
+        :param sentry_units: list of sentry units
+        :param deployment: amulet deployment object pointer
+        :param max_wait: maximum time to wait in seconds to confirm
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Setting ssl charm config option:  off')
+
+        # Disable RMQ SSL
+        config = {'ssl': 'off'}
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
+
+        # Confirm
+        tries = 0
+        ret = self.validate_rmq_ssl_disabled_units(sentry_units)
+        while ret and tries < (max_wait / 4):
+            time.sleep(4)
+            self.log.debug('Attempt {}: {}'.format(tries, ret))
+            ret = self.validate_rmq_ssl_disabled_units(sentry_units)
+            tries += 1
+
+        if ret:
+            amulet.raise_status(amulet.FAIL, ret)
+
+    def connect_amqp_by_unit(self, sentry_unit, ssl=False,
+                             port=None, fatal=True,
+                             username="testuser1", password="changeme"):
+        """Establish and return a pika amqp connection to the rabbitmq service
+        running on a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :param fatal: boolean, default to True (raises on connect error)
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: pika amqp connection pointer or None if failed and non-fatal
+        """
+        host = sentry_unit.info['public-address']
+        unit_name = sentry_unit.info['unit_name']
+
+        # Default port logic if port is not specified
+        if ssl and not port:
+            port = 5671
+        elif not ssl and not port:
+            port = 5672
+
+        self.log.debug('Connecting to amqp on {}:{} ({}) as '
+                       '{}...'.format(host, port, unit_name, username))
+
+        try:
+            credentials = pika.PlainCredentials(username, password)
+            parameters = pika.ConnectionParameters(host=host, port=port,
+                                                   credentials=credentials,
+                                                   ssl=ssl,
+                                                   connection_attempts=3,
+                                                   retry_delay=5,
+                                                   socket_timeout=1)
+            connection = pika.BlockingConnection(parameters)
+            assert connection.server_properties['product'] == 'RabbitMQ'
+            self.log.debug('Connect OK')
+            return connection
+        except Exception as e:
+            msg = ('amqp connection failed to {}:{} as '
+                   '{} ({})'.format(host, port, username, str(e)))
+            if fatal:
+                amulet.raise_status(amulet.FAIL, msg)
+            else:
+                self.log.warn(msg)
+                return None
+
+    def publish_amqp_message_by_unit(self, sentry_unit, message,
+                                     queue="test", ssl=False,
+                                     username="testuser1",
+                                     password="changeme",
+                                     port=None):
+        """Publish an amqp message to a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param message: amqp message string
+        :param queue: message queue, default to test
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :returns: None.  Raises exception if publish failed.
+        """
+        self.log.debug('Publishing message to {} queue:\n{}'.format(queue,
+                                                                    message))
+        connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
+                                               port=port,
+                                               username=username,
+                                               password=password)
+
+        # NOTE(beisner): extra debug here re: pika hang potential:
+        #   https://github.com/pika/pika/issues/297
+        #   https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw
+        self.log.debug('Defining channel...')
+        channel = connection.channel()
+        self.log.debug('Declaring queue...')
+        channel.queue_declare(queue=queue, auto_delete=False, durable=True)
+        self.log.debug('Publishing message...')
+        channel.basic_publish(exchange='', routing_key=queue, body=message)
+        self.log.debug('Closing channel...')
+        channel.close()
+        self.log.debug('Closing connection...')
+        connection.close()
+
+    def get_amqp_message_by_unit(self, sentry_unit, queue="test",
+                                 username="testuser1",
+                                 password="changeme",
+                                 ssl=False, port=None):
+        """Get an amqp message from a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param queue: message queue, default to test
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :returns: amqp message body as string.  Raise if get fails.
+        """
+        connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
+                                               port=port,
+                                               username=username,
+                                               password=password)
+        channel = connection.channel()
+        method_frame, _, body = channel.basic_get(queue)
+
+        if method_frame:
+            self.log.debug('Retreived message from {} queue:\n{}'.format(queue,
+                                                                         body))
+            channel.basic_ack(method_frame.delivery_tag)
+            channel.close()
+            connection.close()
+            return body
+        else:
+            msg = 'No message retrieved.'
+            amulet.raise_status(amulet.FAIL, msg)

charmhelpers/contrib/openstack/context.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import glob
 import json
 import os
 import re
@@ -56,6 +57,7 @@ from charmhelpers.core.host import (
     get_nic_hwaddr,
     mkdir,
     write_file,
+    pwgen,
 )
 from charmhelpers.contrib.hahelpers.cluster import (
     determine_apache_port,
@@ -86,6 +88,8 @@ from charmhelpers.contrib.network.ip import (
     is_bridge_member,
 )
 from charmhelpers.contrib.openstack.utils import get_host_ip
+from charmhelpers.core.unitdata import kv
+
 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
 ADDRESS_TYPES = ['admin', 'internal', 'public']
 
@@ -194,10 +198,50 @@ def config_flags_parser(config_flags):
 class OSContextGenerator(object):
     """Base class for all context generators."""
     interfaces = []
+    related = False
+    complete = False
+    missing_data = []
 
     def __call__(self):
         raise NotImplementedError
 
+    def context_complete(self, ctxt):
+        """Check for missing data for the required context data.
+        Set self.missing_data if it exists and return False.
+        Set self.complete if no missing data and return True.
+        """
+        # Fresh start
+        self.complete = False
+        self.missing_data = []
+        for k, v in six.iteritems(ctxt):
+            if v is None or v == '':
+                if k not in self.missing_data:
+                    self.missing_data.append(k)
+
+        if self.missing_data:
+            self.complete = False
+            log('Missing required data: %s' % ' '.join(self.missing_data), level=INFO)
+        else:
+            self.complete = True
+        return self.complete
+
+    def get_related(self):
+        """Check if any of the context interfaces have relation ids.
+        Set self.related and return True if one of the interfaces
+        has relation ids.
+        """
+        # Fresh start
+        self.related = False
+        try:
+            for interface in self.interfaces:
+                if relation_ids(interface):
+                    self.related = True
+            return self.related
+        except AttributeError as e:
+            log("{} {}"
+                "".format(self, e), 'INFO')
+            return self.related
+
 
 class SharedDBContext(OSContextGenerator):
     interfaces = ['shared-db']
@@ -213,6 +257,7 @@ class SharedDBContext(OSContextGenerator):
         self.database = database
         self.user = user
         self.ssl_dir = ssl_dir
+        self.rel_name = self.interfaces[0]
 
     def __call__(self):
         self.database = self.database or config('database')
@@ -246,6 +291,7 @@ class SharedDBContext(OSContextGenerator):
             password_setting = self.relation_prefix + '_password'
 
         for rid in relation_ids(self.interfaces[0]):
+            self.related = True
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 host = rdata.get('db_host')
@@ -257,7 +303,7 @@ class SharedDBContext(OSContextGenerator):
                     'database_password': rdata.get(password_setting),
                     'database_type': 'mysql'
                 }
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     db_ssl(rdata, ctxt, self.ssl_dir)
                     return ctxt
         return {}
@@ -278,6 +324,7 @@ class PostgresqlDBContext(OSContextGenerator):
 
         ctxt = {}
         for rid in relation_ids(self.interfaces[0]):
+            self.related = True
             for unit in related_units(rid):
                 rel_host = relation_get('host', rid=rid, unit=unit)
                 rel_user = relation_get('user', rid=rid, unit=unit)
@@ -287,7 +334,7 @@ class PostgresqlDBContext(OSContextGenerator):
                         'database_user': rel_user,
                         'database_password': rel_passwd,
                         'database_type': 'postgresql'}
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     return ctxt
 
         return {}
@@ -348,6 +395,7 @@ class IdentityServiceContext(OSContextGenerator):
             ctxt['signing_dir'] = cachedir
 
         for rid in relation_ids(self.rel_name):
+            self.related = True
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 serv_host = rdata.get('service_host')
@@ -366,7 +414,7 @@ class IdentityServiceContext(OSContextGenerator):
                              'service_protocol': svc_protocol,
                              'auth_protocol': auth_protocol})
 
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     # NOTE(jamespage) this is required for >= icehouse
                     # so a missing value just indicates keystone needs
                     # upgrading
@@ -405,6 +453,7 @@ class AMQPContext(OSContextGenerator):
         ctxt = {}
         for rid in relation_ids(self.rel_name):
             ha_vip_only = False
+            self.related = True
             for unit in related_units(rid):
                 if relation_get('clustered', rid=rid, unit=unit):
                     ctxt['clustered'] = True
@@ -437,7 +486,7 @@ class AMQPContext(OSContextGenerator):
                 ha_vip_only = relation_get('ha-vip-only',
                                            rid=rid, unit=unit) is not None
 
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     if 'rabbit_ssl_ca' in ctxt:
                         if not self.ssl_dir:
                             log("Charm not setup for ssl support but ssl ca "
@@ -469,7 +518,7 @@ class AMQPContext(OSContextGenerator):
             ctxt['oslo_messaging_flags'] = config_flags_parser(
                 oslo_messaging_flags)
 
-        if not context_complete(ctxt):
+        if not self.complete:
             return {}
 
         return ctxt
@@ -507,7 +556,7 @@ class CephContext(OSContextGenerator):
         if not os.path.isdir('/etc/ceph'):
             os.mkdir('/etc/ceph')
 
-        if not context_complete(ctxt):
+        if not self.context_complete(ctxt):
             return {}
 
         ensure_packages(['ceph-common'])
@@ -580,15 +629,28 @@ class HAProxyContext(OSContextGenerator):
         if config('haproxy-client-timeout'):
             ctxt['haproxy_client_timeout'] = config('haproxy-client-timeout')
 
+        if config('haproxy-queue-timeout'):
+            ctxt['haproxy_queue_timeout'] = config('haproxy-queue-timeout')
+
+        if config('haproxy-connect-timeout'):
+            ctxt['haproxy_connect_timeout'] = config('haproxy-connect-timeout')
+
         if config('prefer-ipv6'):
             ctxt['ipv6'] = True
             ctxt['local_host'] = 'ip6-localhost'
             ctxt['haproxy_host'] = '::'
-            ctxt['stat_port'] = ':::8888'
         else:
             ctxt['local_host'] = '127.0.0.1'
             ctxt['haproxy_host'] = '0.0.0.0'
-            ctxt['stat_port'] = ':8888'
+
+        ctxt['stat_port'] = '8888'
+
+        db = kv()
+        ctxt['stat_password'] = db.get('stat-password')
+        if not ctxt['stat_password']:
+            ctxt['stat_password'] = db.set('stat-password',
+                                           pwgen(32))
+            db.flush()
 
         for frontend in cluster_hosts:
             if (len(cluster_hosts[frontend]['backends']) > 1 or
@@ -906,6 +968,19 @@ class NeutronContext(OSContextGenerator):
                     'config': config}
         return ovs_ctxt
 
+    def midonet_ctxt(self):
+        driver = neutron_plugin_attribute(self.plugin, 'driver',
+                                          self.network_manager)
+        midonet_config = neutron_plugin_attribute(self.plugin, 'config',
+                                                  self.network_manager)
+        mido_ctxt = {'core_plugin': driver,
+                     'neutron_plugin': 'midonet',
+                     'neutron_security_groups': self.neutron_security_groups,
+                     'local_ip': unit_private_ip(),
+                     'config': midonet_config}
+
+        return mido_ctxt
+
     def __call__(self):
         if self.network_manager not in ['quantum', 'neutron']:
             return {}
@@ -927,6 +1002,8 @@ class NeutronContext(OSContextGenerator):
             ctxt.update(self.nuage_ctxt())
         elif self.plugin == 'plumgrid':
             ctxt.update(self.pg_ctxt())
+        elif self.plugin == 'midonet':
+            ctxt.update(self.midonet_ctxt())
 
         alchemy_flags = config('neutron-alchemy-flags')
         if alchemy_flags:
@@ -1027,6 +1104,20 @@ class OSConfigFlagContext(OSContextGenerator):
                 config_flags_parser(config_flags)}
 
 
+class LibvirtConfigFlagsContext(OSContextGenerator):
+    """
+    This context provides support for extending
+    the libvirt section through user-defined flags.
+    """
+    def __call__(self):
+        ctxt = {}
+        libvirt_flags = config('libvirt-flags')
+        if libvirt_flags:
+            ctxt['libvirt_flags'] = config_flags_parser(
+                libvirt_flags)
+        return ctxt
+
+
 class SubordinateConfigContext(OSContextGenerator):
 
     """
@@ -1059,7 +1150,7 @@ class SubordinateConfigContext(OSContextGenerator):
 
         ctxt = {
             ... other context ...
-            'subordinate_config': {
+            'subordinate_configuration': {
                 'DEFAULT': {
                     'key1': 'value1',
                 },
@@ -1100,22 +1191,23 @@ class SubordinateConfigContext(OSContextGenerator):
                     try:
                         sub_config = json.loads(sub_config)
                     except:
-                        log('Could not parse JSON from subordinate_config '
+                        log('Could not parse JSON from '
-                            'setting from %s' % rid, level=ERROR)
+                            'subordinate_configuration setting from %s'
+                            % rid, level=ERROR)
                         continue
 
                     for service in self.services:
                         if service not in sub_config:
-                            log('Found subordinate_config on %s but it contained'
+                            log('Found subordinate_configuration on %s but it '
-                                'nothing for %s service' % (rid, service),
+                                'contained nothing for %s service'
-                                level=INFO)
+                                % (rid, service), level=INFO)
                             continue
 
                         sub_config = sub_config[service]
                         if self.config_file not in sub_config:
-                            log('Found subordinate_config on %s but it contained'
+                            log('Found subordinate_configuration on %s but it '
-                                'nothing for %s' % (rid, self.config_file),
+                                'contained nothing for %s'
-                                level=INFO)
+                                % (rid, self.config_file), level=INFO)
                             continue
 
                         sub_config = sub_config[self.config_file]
@@ -1318,7 +1410,7 @@ class DataPortContext(NeutronPortContext):
             normalized.update({port: port for port in resolved
                                if port in ports})
             if resolved:
-                return {bridge: normalized[port] for port, bridge in
+                return {normalized[port]: bridge for port, bridge in
                         six.iteritems(portmap) if port in normalized.keys()}
 
         return None
@@ -1329,12 +1421,22 @@ class PhyNICMTUContext(DataPortContext):
     def __call__(self):
         ctxt = {}
         mappings = super(PhyNICMTUContext, self).__call__()
-        if mappings and mappings.values():
+        if mappings and mappings.keys():
-            ports = mappings.values()
+            ports = sorted(mappings.keys())
             napi_settings = NeutronAPIContext()()
             mtu = napi_settings.get('network_device_mtu')
+            all_ports = set()
+            # If any of ports is a vlan device, its underlying device must have
+            # mtu applied first.
+            for port in ports:
+                for lport in glob.glob("/sys/class/net/%s/lower_*" % port):
+                    lport = os.path.basename(lport)
+                    all_ports.add(lport.split('_')[1])
+
+            all_ports = list(all_ports)
+            all_ports.extend(ports)
             if mtu:
-                ctxt["devs"] = '\\n'.join(ports)
+                ctxt["devs"] = '\\n'.join(all_ports)
                 ctxt['mtu'] = mtu
 
         return ctxt
@@ -1366,6 +1468,6 @@ class NetworkServiceContext(OSContextGenerator):
                     'auth_protocol':
                     rdata.get('auth_protocol') or 'http',
                 }
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     return ctxt
         return {}

charmhelpers/contrib/openstack/files/check_haproxy.sh

@@ -9,15 +9,17 @@
 CRITICAL=0
 NOTACTIVE=''
 LOGFILE=/var/log/nagios/check_haproxy.log
-AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}')
+AUTH=$(grep -r "stats auth" /etc/haproxy | awk 'NR=1{print $4}')
 
-for appserver in $(grep '    server' /etc/haproxy/haproxy.cfg | awk '{print $2'});
+typeset -i N_INSTANCES=0
+for appserver in $(awk '/^\s+server/{print $2}' /etc/haproxy/haproxy.cfg)
 do
-    output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 --regex="class=\"(active|backup)(2|3).*${appserver}" -e ' 200 OK')
+    N_INSTANCES=N_INSTANCES+1
+    output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' --regex=",${appserver},.*,UP.*" -e ' 200 OK')
     if [ $? != 0 ]; then
         date >> $LOGFILE
         echo $output >> $LOGFILE
-        /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -v | grep $appserver >> $LOGFILE 2>&1
+        /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' -v | grep ",${appserver}," >> $LOGFILE 2>&1
         CRITICAL=1
         NOTACTIVE="${NOTACTIVE} $appserver"
     fi
@@ -28,5 +30,5 @@ if [ $CRITICAL = 1 ]; then
     exit 2
 fi
 
-echo "OK: All haproxy instances looking good"
+echo "OK: All haproxy instances ($N_INSTANCES) looking good"
 exit 0

charmhelpers/contrib/openstack/neutron.py

@@ -204,11 +204,25 @@ def neutron_plugins():
                                         database=config('database'),
                                         ssl_dir=NEUTRON_CONF_DIR)],
             'services': [],
-            'packages': [['plumgrid-lxc'],
+            'packages': ['plumgrid-lxc',
-                         ['iovisor-dkms']],
+                         'iovisor-dkms'],
             'server_packages': ['neutron-server',
                                 'neutron-plugin-plumgrid'],
             'server_services': ['neutron-server']
+        },
+        'midonet': {
+            'config': '/etc/neutron/plugins/midonet/midonet.ini',
+            'driver': 'midonet.neutron.plugin.MidonetPluginV2',
+            'contexts': [
+                context.SharedDBContext(user=config('neutron-database-user'),
+                                        database=config('neutron-database'),
+                                        relation_prefix='neutron',
+                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'services': [],
+            'packages': [[headers_package()] + determine_dkms_package()],
+            'server_packages': ['neutron-server',
+                                'python-neutron-plugin-midonet'],
+            'server_services': ['neutron-server']
         }
     }
     if release >= 'icehouse':
@@ -310,10 +324,10 @@ def parse_bridge_mappings(mappings):
 def parse_data_port_mappings(mappings, default_bridge='br-data'):
     """Parse data port mappings.
 
-    Mappings must be a space-delimited list of port:bridge mappings.
+    Mappings must be a space-delimited list of bridge:port.
 
-    Returns dict of the form {port:bridge} where port may be an mac address or
+    Returns dict of the form {port:bridge} where ports may be mac addresses or
-    interface name.
+    interface names.
     """
 
     # NOTE(dosaboy): we use rvalue for key to allow multiple values to be

charmhelpers/contrib/openstack/templates/ceph.conf

@@ -13,3 +13,9 @@ log to syslog = {{ use_syslog }}
 err to syslog = {{ use_syslog }}
 clog to syslog = {{ use_syslog }}
 
+[client]
+{% if rbd_client_cache_settings -%}
+{% for key, value in rbd_client_cache_settings.iteritems() -%}
+{{ key }} = {{ value }}
+{% endfor -%}
+{%- endif %}

charmhelpers/contrib/openstack/templates/haproxy.cfg

@@ -12,27 +12,35 @@ defaults
     option tcplog
     option dontlognull
     retries 3
-    timeout queue 1000
+{%- if haproxy_queue_timeout %}
-    timeout connect 1000
+    timeout queue {{ haproxy_queue_timeout }}
-{% if haproxy_client_timeout -%}
+{%- else %}
+    timeout queue 5000
+{%- endif %}
+{%- if haproxy_connect_timeout %}
+    timeout connect {{ haproxy_connect_timeout }}
+{%- else %}
+    timeout connect 5000
+{%- endif %}
+{%- if haproxy_client_timeout %}
     timeout client {{ haproxy_client_timeout }}
-{% else -%}
+{%- else %}
     timeout client 30000
-{% endif -%}
+{%- endif %}
-
+{%- if haproxy_server_timeout %}
-{% if haproxy_server_timeout -%}
     timeout server {{ haproxy_server_timeout }}
-{% else -%}
+{%- else %}
     timeout server 30000
-{% endif -%}
+{%- endif %}
 
-listen stats {{ stat_port }}
+listen stats
+    bind {{ local_host }}:{{ stat_port }}
     mode http
     stats enable
     stats hide-version
     stats realm Haproxy\ Statistics
     stats uri /
-    stats auth admin:password
+    stats auth admin:{{ stat_password }}
 
 {% if frontends -%}
 {% for service, ports in service_ports.items() -%}

charmhelpers/contrib/openstack/templating.py

@@ -18,7 +18,7 @@ import os
 
 import six
 
-from charmhelpers.fetch import apt_install
+from charmhelpers.fetch import apt_install, apt_update
 from charmhelpers.core.hookenv import (
     log,
     ERROR,
@@ -29,6 +29,7 @@ from charmhelpers.contrib.openstack.utils import OPENSTACK_CODENAMES
 try:
     from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 except ImportError:
+    apt_update(fatal=True)
     apt_install('python-jinja2', fatal=True)
     from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 
@@ -112,7 +113,7 @@ class OSConfigTemplate(object):
 
     def complete_contexts(self):
         '''
-        Return a list of interfaces that have atisfied contexts.
+        Return a list of interfaces that have satisfied contexts.
         '''
         if self._complete_contexts:
             return self._complete_contexts
@@ -293,3 +294,30 @@ class OSConfigRenderer(object):
         [interfaces.extend(i.complete_contexts())
          for i in six.itervalues(self.templates)]
         return interfaces
+
+    def get_incomplete_context_data(self, interfaces):
+        '''
+        Return dictionary of relation status of interfaces and any missing
+        required context data. Example:
+            {'amqp': {'missing_data': ['rabbitmq_password'], 'related': True},
+             'zeromq-configuration': {'related': False}}
+        '''
+        incomplete_context_data = {}
+
+        for i in six.itervalues(self.templates):
+            for context in i.contexts:
+                for interface in interfaces:
+                    related = False
+                    if interface in context.interfaces:
+                        related = context.get_related()
+                        missing_data = context.missing_data
+                        if missing_data:
+                            incomplete_context_data[interface] = {'missing_data': missing_data}
+                        if related:
+                            if incomplete_context_data.get(interface):
+                                incomplete_context_data[interface].update({'related': True})
+                            else:
+                                incomplete_context_data[interface] = {'related': True}
+                        else:
+                            incomplete_context_data[interface] = {'related': False}
+        return incomplete_context_data

charmhelpers/contrib/openstack/utils.py

@@ -25,6 +25,8 @@ import sys
 import re
 
 import six
+import traceback
+import uuid
 import yaml
 
 from charmhelpers.contrib.network import ip
@@ -34,12 +36,17 @@ from charmhelpers.core import (
 )
 
 from charmhelpers.core.hookenv import (
+    action_fail,
+    action_set,
     config,
     log as juju_log,
     charm_dir,
     INFO,
+    related_units,
     relation_ids,
-    relation_set
+    relation_set,
+    status_set,
+    hook_name
 )
 
 from charmhelpers.contrib.storage.linux.lvm import (
@@ -49,7 +56,8 @@ from charmhelpers.contrib.storage.linux.lvm import (
 )
 
 from charmhelpers.contrib.network.ip import (
-    get_ipv6_addr
+    get_ipv6_addr,
+    is_ipv6,
 )
 
 from charmhelpers.contrib.python.packages import (
@@ -78,6 +86,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('utopic', 'juno'),
     ('vivid', 'kilo'),
     ('wily', 'liberty'),
+    ('xenial', 'mitaka'),
 ])
 
 
@@ -91,6 +100,7 @@ OPENSTACK_CODENAMES = OrderedDict([
     ('2014.2', 'juno'),
     ('2015.1', 'kilo'),
     ('2015.2', 'liberty'),
+    ('2016.1', 'mitaka'),
 ])
 
 # The ugly duckling
@@ -114,33 +124,47 @@ SWIFT_CODENAMES = OrderedDict([
     ('2.2.1', 'kilo'),
     ('2.2.2', 'kilo'),
     ('2.3.0', 'liberty'),
+    ('2.4.0', 'liberty'),
+    ('2.5.0', 'liberty'),
 ])
 
 # >= Liberty version->codename mapping
 PACKAGE_CODENAMES = {
     'nova-common': OrderedDict([
-        ('12.0.0', 'liberty'),
+        ('12.0', 'liberty'),
+        ('13.0', 'mitaka'),
     ]),
     'neutron-common': OrderedDict([
-        ('7.0.0', 'liberty'),
+        ('7.0', 'liberty'),
+        ('8.0', 'mitaka'),
     ]),
     'cinder-common': OrderedDict([
-        ('7.0.0', 'liberty'),
+        ('7.0', 'liberty'),
+        ('8.0', 'mitaka'),
     ]),
     'keystone': OrderedDict([
-        ('8.0.0', 'liberty'),
+        ('8.0', 'liberty'),
+        ('9.0', 'mitaka'),
     ]),
     'horizon-common': OrderedDict([
-        ('8.0.0', 'liberty'),
+        ('8.0', 'liberty'),
+        ('9.0', 'mitaka'),
     ]),
     'ceilometer-common': OrderedDict([
-        ('5.0.0', 'liberty'),
+        ('5.0', 'liberty'),
+        ('6.0', 'mitaka'),
     ]),
     'heat-common': OrderedDict([
-        ('5.0.0', 'liberty'),
+        ('5.0', 'liberty'),
+        ('6.0', 'mitaka'),
     ]),
     'glance-common': OrderedDict([
-        ('11.0.0', 'liberty'),
+        ('11.0', 'liberty'),
+        ('12.0', 'mitaka'),
+    ]),
+    'openstack-dashboard': OrderedDict([
+        ('8.0', 'liberty'),
+        ('9.0', 'mitaka'),
     ]),
 }
 
@@ -227,7 +251,14 @@ def get_os_codename_package(package, fatal=True):
         error_out(e)
 
     vers = apt.upstream_version(pkg.current_ver.ver_str)
-    match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
+    if 'swift' in pkg.name:
+        # Fully x.y.z match for swift versions
+        match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
+    else:
+        # x.y match only for 20XX.X
+        # and ignore patch level for other packages
+        match = re.match('^(\d+)\.(\d+)', vers)
+
     if match:
         vers = match.group(0)
 
@@ -239,13 +270,8 @@ def get_os_codename_package(package, fatal=True):
         # < Liberty co-ordinated project versions
         try:
             if 'swift' in pkg.name:
-                swift_vers = vers[:5]
+                return SWIFT_CODENAMES[vers]
-                if swift_vers not in SWIFT_CODENAMES:
-                    # Deal with 1.10.0 upward
-                    swift_vers = vers[:6]
-                return SWIFT_CODENAMES[swift_vers]
             else:
-                vers = vers[:6]
                 return OPENSTACK_CODENAMES[vers]
         except KeyError:
             if not fatal:
@@ -364,6 +390,9 @@ def configure_installation_source(rel):
             'liberty': 'trusty-updates/liberty',
             'liberty/updates': 'trusty-updates/liberty',
             'liberty/proposed': 'trusty-proposed/liberty',
+            'mitaka': 'trusty-updates/mitaka',
+            'mitaka/updates': 'trusty-updates/mitaka',
+            'mitaka/proposed': 'trusty-proposed/mitaka',
         }
 
         try:
@@ -510,6 +539,12 @@ def sync_db_with_multi_ipv6_addresses(database, database_user,
                                       relation_prefix=None):
     hosts = get_ipv6_addr(dynamic_only=False)
 
+    if config('vip'):
+        vips = config('vip').split()
+        for vip in vips:
+            if vip and is_ipv6(vip):
+                hosts.append(vip)
+
     kwargs = {'database': database,
               'username': database_user,
               'hostname': json.dumps(hosts)}
@@ -558,7 +593,7 @@ def _git_yaml_load(projects_yaml):
     return yaml.load(projects_yaml)
 
 
-def git_clone_and_install(projects_yaml, core_project, depth=1):
+def git_clone_and_install(projects_yaml, core_project):
     """
     Clone/install all specified OpenStack repositories.
 
@@ -608,6 +643,9 @@ def git_clone_and_install(projects_yaml, core_project, depth=1):
     for p in projects['repositories']:
         repo = p['repository']
         branch = p['branch']
+        depth = '1'
+        if 'depth' in p.keys():
+            depth = p['depth']
         if p['name'] == 'requirements':
             repo_dir = _git_clone_and_install_single(repo, branch, depth,
                                                      parent_dir, http_proxy,
@@ -652,19 +690,13 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
     """
     Clone and install a single git repository.
     """
-    dest_dir = os.path.join(parent_dir, os.path.basename(repo))
-
     if not os.path.exists(parent_dir):
         juju_log('Directory already exists at {}. '
                  'No need to create directory.'.format(parent_dir))
         os.mkdir(parent_dir)
 
-    if not os.path.exists(dest_dir):
+    juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
-        juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
+    repo_dir = install_remote(repo, dest=parent_dir, branch=branch, depth=depth)
-        repo_dir = install_remote(repo, dest=parent_dir, branch=branch,
-                                  depth=depth)
-    else:
-        repo_dir = dest_dir
 
     venv = os.path.join(parent_dir, 'venv')
 
@@ -745,3 +777,235 @@ def git_yaml_value(projects_yaml, key):
         return projects[key]
 
     return None
+
+
+def os_workload_status(configs, required_interfaces, charm_func=None):
+    """
+    Decorator to set workload status based on complete contexts
+    """
+    def wrap(f):
+        @wraps(f)
+        def wrapped_f(*args, **kwargs):
+            # Run the original function first
+            f(*args, **kwargs)
+            # Set workload status now that contexts have been
+            # acted on
+            set_os_workload_status(configs, required_interfaces, charm_func)
+        return wrapped_f
+    return wrap
+
+
+def set_os_workload_status(configs, required_interfaces, charm_func=None):
+    """
+    Set workload status based on complete contexts.
+    status-set missing or incomplete contexts
+    and juju-log details of missing required data.
+    charm_func is a charm specific function to run checking
+    for charm specific requirements such as a VIP setting.
+    """
+    incomplete_rel_data = incomplete_relation_data(configs, required_interfaces)
+    state = 'active'
+    missing_relations = []
+    incomplete_relations = []
+    message = None
+    charm_state = None
+    charm_message = None
+
+    for generic_interface in incomplete_rel_data.keys():
+        related_interface = None
+        missing_data = {}
+        # Related or not?
+        for interface in incomplete_rel_data[generic_interface]:
+            if incomplete_rel_data[generic_interface][interface].get('related'):
+                related_interface = interface
+                missing_data = incomplete_rel_data[generic_interface][interface].get('missing_data')
+        # No relation ID for the generic_interface
+        if not related_interface:
+            juju_log("{} relation is missing and must be related for "
+                     "functionality. ".format(generic_interface), 'WARN')
+            state = 'blocked'
+            if generic_interface not in missing_relations:
+                missing_relations.append(generic_interface)
+        else:
+            # Relation ID exists but no related unit
+            if not missing_data:
+                # Edge case relation ID exists but departing
+                if ('departed' in hook_name() or 'broken' in hook_name()) \
+                        and related_interface in hook_name():
+                    state = 'blocked'
+                    if generic_interface not in missing_relations:
+                        missing_relations.append(generic_interface)
+                    juju_log("{} relation's interface, {}, "
+                             "relationship is departed or broken "
+                             "and is required for functionality."
+                             "".format(generic_interface, related_interface), "WARN")
+                # Normal case relation ID exists but no related unit
+                # (joining)
+                else:
+                    juju_log("{} relations's interface, {}, is related but has "
+                             "no units in the relation."
+                             "".format(generic_interface, related_interface), "INFO")
+            # Related unit exists and data missing on the relation
+            else:
+                juju_log("{} relation's interface, {}, is related awaiting "
+                         "the following data from the relationship: {}. "
+                         "".format(generic_interface, related_interface,
+                                   ", ".join(missing_data)), "INFO")
+            if state != 'blocked':
+                state = 'waiting'
+            if generic_interface not in incomplete_relations \
+                    and generic_interface not in missing_relations:
+                incomplete_relations.append(generic_interface)
+
+    if missing_relations:
+        message = "Missing relations: {}".format(", ".join(missing_relations))
+        if incomplete_relations:
+            message += "; incomplete relations: {}" \
+                       "".format(", ".join(incomplete_relations))
+        state = 'blocked'
+    elif incomplete_relations:
+        message = "Incomplete relations: {}" \
+                  "".format(", ".join(incomplete_relations))
+        state = 'waiting'
+
+    # Run charm specific checks
+    if charm_func:
+        charm_state, charm_message = charm_func(configs)
+        if charm_state != 'active' and charm_state != 'unknown':
+            state = workload_state_compare(state, charm_state)
+            if message:
+                charm_message = charm_message.replace("Incomplete relations: ",
+                                                      "")
+                message = "{}, {}".format(message, charm_message)
+            else:
+                message = charm_message
+
+    # Set to active if all requirements have been met
+    if state == 'active':
+        message = "Unit is ready"
+        juju_log(message, "INFO")
+
+    status_set(state, message)
+
+
+def workload_state_compare(current_workload_state, workload_state):
+    """ Return highest priority of two states"""
+    hierarchy = {'unknown': -1,
+                 'active': 0,
+                 'maintenance': 1,
+                 'waiting': 2,
+                 'blocked': 3,
+                 }
+
+    if hierarchy.get(workload_state) is None:
+        workload_state = 'unknown'
+    if hierarchy.get(current_workload_state) is None:
+        current_workload_state = 'unknown'
+
+    # Set workload_state based on hierarchy of statuses
+    if hierarchy.get(current_workload_state) > hierarchy.get(workload_state):
+        return current_workload_state
+    else:
+        return workload_state
+
+
+def incomplete_relation_data(configs, required_interfaces):
+    """
+    Check complete contexts against required_interfaces
+    Return dictionary of incomplete relation data.
+
+    configs is an OSConfigRenderer object with configs registered
+
+    required_interfaces is a dictionary of required general interfaces
+    with dictionary values of possible specific interfaces.
+    Example:
+    required_interfaces = {'database': ['shared-db', 'pgsql-db']}
+
+    The interface is said to be satisfied if anyone of the interfaces in the
+    list has a complete context.
+
+    Return dictionary of incomplete or missing required contexts with relation
+    status of interfaces and any missing data points. Example:
+        {'message':
+             {'amqp': {'missing_data': ['rabbitmq_password'], 'related': True},
+              'zeromq-configuration': {'related': False}},
+         'identity':
+             {'identity-service': {'related': False}},
+         'database':
+             {'pgsql-db': {'related': False},
+              'shared-db': {'related': True}}}
+    """
+    complete_ctxts = configs.complete_contexts()
+    incomplete_relations = []
+    for svc_type in required_interfaces.keys():
+        # Avoid duplicates
+        found_ctxt = False
+        for interface in required_interfaces[svc_type]:
+            if interface in complete_ctxts:
+                found_ctxt = True
+        if not found_ctxt:
+            incomplete_relations.append(svc_type)
+    incomplete_context_data = {}
+    for i in incomplete_relations:
+        incomplete_context_data[i] = configs.get_incomplete_context_data(required_interfaces[i])
+    return incomplete_context_data
+
+
+def do_action_openstack_upgrade(package, upgrade_callback, configs):
+    """Perform action-managed OpenStack upgrade.
+
+    Upgrades packages to the configured openstack-origin version and sets
+    the corresponding action status as a result.
+
+    If the charm was installed from source we cannot upgrade it.
+    For backwards compatibility a config flag (action-managed-upgrade) must
+    be set for this code to run, otherwise a full service level upgrade will
+    fire on config-changed.
+
+    @param package: package name for determining if upgrade available
+    @param upgrade_callback: function callback to charm's upgrade function
+    @param configs: templating object derived from OSConfigRenderer class
+
+    @return: True if upgrade successful; False if upgrade failed or skipped
+    """
+    ret = False
+
+    if git_install_requested():
+        action_set({'outcome': 'installed from source, skipped upgrade.'})
+    else:
+        if openstack_upgrade_available(package):
+            if config('action-managed-upgrade'):
+                juju_log('Upgrading OpenStack release')
+
+                try:
+                    upgrade_callback(configs=configs)
+                    action_set({'outcome': 'success, upgrade completed.'})
+                    ret = True
+                except:
+                    action_set({'outcome': 'upgrade failed, see traceback.'})
+                    action_set({'traceback': traceback.format_exc()})
+                    action_fail('do_openstack_upgrade resulted in an '
+                                'unexpected error')
+            else:
+                action_set({'outcome': 'action-managed-upgrade config is '
+                                       'False, skipped upgrade.'})
+        else:
+            action_set({'outcome': 'no upgrade available.'})
+
+    return ret
+
+
+def remote_restart(rel_name, remote_service=None):
+    trigger = {
+        'restart-trigger': str(uuid.uuid4()),
+    }
+    if remote_service:
+        trigger['remote-service'] = remote_service
+    for rid in relation_ids(rel_name):
+        # This subordinate can be related to two seperate services using
+        # different subordinate relations so only issue the restart if
+        # the principle is conencted down the relation we think it is
+        if related_units(relid=rid):
+            relation_set(relation_id=rid,
+                         relation_settings=trigger,
+                         )

charmhelpers/contrib/python/packages.py

@@ -42,8 +42,12 @@ def parse_options(given, available):
             yield "--{0}={1}".format(key, value)
 
 
-def pip_install_requirements(requirements, **options):
+def pip_install_requirements(requirements, constraints=None, **options):
-    """Install a requirements file """
+    """Install a requirements file.
+
+    :param constraints: Path to pip constraints file.
+    http://pip.readthedocs.org/en/stable/user_guide/#constraints-files
+    """
     command = ["install"]
 
     available_options = ('proxy', 'src', 'log', )
@@ -51,8 +55,13 @@ def pip_install_requirements(requirements, **options):
         command.append(option)
 
     command.append("-r {0}".format(requirements))
-    log("Installing from file: {} with options: {}".format(requirements,
+    if constraints:
-                                                           command))
+        command.append("-c {0}".format(constraints))
+        log("Installing from file: {} with constraints {} "
+            "and options: {}".format(requirements, constraints, command))
+    else:
+        log("Installing from file: {} with options: {}".format(requirements,
+                                                               command))
     pip_execute(command)
 
 

charmhelpers/contrib/storage/linux/ceph.py

@@ -23,6 +23,8 @@
 #  James Page <james.page@ubuntu.com>
 #  Adam Gandelman <adamg@ubuntu.com>
 #
+import bisect
+import six
 
 import os
 import shutil
@@ -59,6 +61,8 @@ from charmhelpers.fetch import (
     apt_install,
 )
 
+from charmhelpers.core.kernel import modprobe
+
 KEYRING = '/etc/ceph/ceph.client.{}.keyring'
 KEYFILE = '/etc/ceph/ceph.client.{}.key'
 
@@ -70,6 +74,394 @@ log to syslog = {use_syslog}
 err to syslog = {use_syslog}
 clog to syslog = {use_syslog}
 """
+# For 50 < osds < 240,000 OSDs (Roughly 1 Exabyte at 6T OSDs)
+powers_of_two = [8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576, 2097152, 4194304, 8388608]
+
+
+def validator(value, valid_type, valid_range=None):
+    """
+    Used to validate these: http://docs.ceph.com/docs/master/rados/operations/pools/#set-pool-values
+    Example input:
+        validator(value=1,
+                  valid_type=int,
+                  valid_range=[0, 2])
+    This says I'm testing value=1.  It must be an int inclusive in [0,2]
+
+    :param value: The value to validate
+    :param valid_type: The type that value should be.
+    :param valid_range: A range of values that value can assume.
+    :return:
+    """
+    assert isinstance(value, valid_type), "{} is not a {}".format(
+        value,
+        valid_type)
+    if valid_range is not None:
+        assert isinstance(valid_range, list), \
+            "valid_range must be a list, was given {}".format(valid_range)
+        # If we're dealing with strings
+        if valid_type is six.string_types:
+            assert value in valid_range, \
+                "{} is not in the list {}".format(value, valid_range)
+        # Integer, float should have a min and max
+        else:
+            if len(valid_range) != 2:
+                raise ValueError(
+                    "Invalid valid_range list of {} for {}.  "
+                    "List must be [min,max]".format(valid_range, value))
+            assert value >= valid_range[0], \
+                "{} is less than minimum allowed value of {}".format(
+                    value, valid_range[0])
+            assert value <= valid_range[1], \
+                "{} is greater than maximum allowed value of {}".format(
+                    value, valid_range[1])
+
+
+class PoolCreationError(Exception):
+    """
+    A custom error to inform the caller that a pool creation failed.  Provides an error message
+    """
+    def __init__(self, message):
+        super(PoolCreationError, self).__init__(message)
+
+
+class Pool(object):
+    """
+    An object oriented approach to Ceph pool creation. This base class is inherited by ReplicatedPool and ErasurePool.
+    Do not call create() on this base class as it will not do anything.  Instantiate a child class and call create().
+    """
+    def __init__(self, service, name):
+        self.service = service
+        self.name = name
+
+    # Create the pool if it doesn't exist already
+    # To be implemented by subclasses
+    def create(self):
+        pass
+
+    def add_cache_tier(self, cache_pool, mode):
+        """
+        Adds a new cache tier to an existing pool.
+        :param cache_pool: six.string_types.  The cache tier pool name to add.
+        :param mode: six.string_types. The caching mode to use for this pool.  valid range = ["readonly", "writeback"]
+        :return: None
+        """
+        # Check the input types and values
+        validator(value=cache_pool, valid_type=six.string_types)
+        validator(value=mode, valid_type=six.string_types, valid_range=["readonly", "writeback"])
+
+        check_call(['ceph', '--id', self.service, 'osd', 'tier', 'add', self.name, cache_pool])
+        check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, mode])
+        check_call(['ceph', '--id', self.service, 'osd', 'tier', 'set-overlay', self.name, cache_pool])
+        check_call(['ceph', '--id', self.service, 'osd', 'pool', 'set', cache_pool, 'hit_set_type', 'bloom'])
+
+    def remove_cache_tier(self, cache_pool):
+        """
+        Removes a cache tier from Ceph.  Flushes all dirty objects from writeback pools and waits for that to complete.
+        :param cache_pool: six.string_types.  The cache tier pool name to remove.
+        :return: None
+        """
+        # read-only is easy, writeback is much harder
+        mode = get_cache_mode(cache_pool)
+        if mode == 'readonly':
+            check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, 'none'])
+            check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool])
+
+        elif mode == 'writeback':
+            check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, 'forward'])
+            # Flush the cache and wait for it to return
+            check_call(['ceph', '--id', self.service, '-p', cache_pool, 'cache-flush-evict-all'])
+            check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove-overlay', self.name])
+            check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool])
+
+    def get_pgs(self, pool_size):
+        """
+        :param pool_size: int. pool_size is either the number of replicas for replicated pools or the K+M sum for
+            erasure coded pools
+        :return: int.  The number of pgs to use.
+        """
+        validator(value=pool_size, valid_type=int)
+        osds = get_osds(self.service)
+        if not osds:
+            # NOTE(james-page): Default to 200 for older ceph versions
+            # which don't support OSD query from cli
+            return 200
+
+        # Calculate based on Ceph best practices
+        if osds < 5:
+            return 128
+        elif 5 < osds < 10:
+            return 512
+        elif 10 < osds < 50:
+            return 4096
+        else:
+            estimate = (osds * 100) / pool_size
+            # Return the next nearest power of 2
+            index = bisect.bisect_right(powers_of_two, estimate)
+            return powers_of_two[index]
+
+
+class ReplicatedPool(Pool):
+    def __init__(self, service, name, replicas=2):
+        super(ReplicatedPool, self).__init__(service=service, name=name)
+        self.replicas = replicas
+
+    def create(self):
+        if not pool_exists(self.service, self.name):
+            # Create it
+            pgs = self.get_pgs(self.replicas)
+            cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs)]
+            try:
+                check_call(cmd)
+            except CalledProcessError:
+                raise
+
+
+# Default jerasure erasure coded pool
+class ErasurePool(Pool):
+    def __init__(self, service, name, erasure_code_profile="default"):
+        super(ErasurePool, self).__init__(service=service, name=name)
+        self.erasure_code_profile = erasure_code_profile
+
+    def create(self):
+        if not pool_exists(self.service, self.name):
+            # Try to find the erasure profile information so we can properly size the pgs
+            erasure_profile = get_erasure_profile(service=self.service, name=self.erasure_code_profile)
+
+            # Check for errors
+            if erasure_profile is None:
+                log(message='Failed to discover erasure_profile named={}'.format(self.erasure_code_profile),
+                    level=ERROR)
+                raise PoolCreationError(message='unable to find erasure profile {}'.format(self.erasure_code_profile))
+            if 'k' not in erasure_profile or 'm' not in erasure_profile:
+                # Error
+                log(message='Unable to find k (data chunks) or m (coding chunks) in {}'.format(erasure_profile),
+                    level=ERROR)
+                raise PoolCreationError(
+                    message='unable to find k (data chunks) or m (coding chunks) in {}'.format(erasure_profile))
+
+            pgs = self.get_pgs(int(erasure_profile['k']) + int(erasure_profile['m']))
+            # Create it
+            cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs),
+                   'erasure', self.erasure_code_profile]
+            try:
+                check_call(cmd)
+            except CalledProcessError:
+                raise
+
+    """Get an existing erasure code profile if it already exists.
+       Returns json formatted output"""
+
+
+def get_erasure_profile(service, name):
+    """
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param name:
+    :return:
+    """
+    try:
+        out = check_output(['ceph', '--id', service,
+                            'osd', 'erasure-code-profile', 'get',
+                            name, '--format=json'])
+        return json.loads(out)
+    except (CalledProcessError, OSError, ValueError):
+        return None
+
+
+def pool_set(service, pool_name, key, value):
+    """
+    Sets a value for a RADOS pool in ceph.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param pool_name: six.string_types
+    :param key: six.string_types
+    :param value:
+    :return: None.  Can raise CalledProcessError
+    """
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', pool_name, key, value]
+    try:
+        check_call(cmd)
+    except CalledProcessError:
+        raise
+
+
+def snapshot_pool(service, pool_name, snapshot_name):
+    """
+    Snapshots a RADOS pool in ceph.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param pool_name: six.string_types
+    :param snapshot_name: six.string_types
+    :return: None.  Can raise CalledProcessError
+    """
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'mksnap', pool_name, snapshot_name]
+    try:
+        check_call(cmd)
+    except CalledProcessError:
+        raise
+
+
+def remove_pool_snapshot(service, pool_name, snapshot_name):
+    """
+    Remove a snapshot from a RADOS pool in ceph.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param pool_name: six.string_types
+    :param snapshot_name: six.string_types
+    :return: None.  Can raise CalledProcessError
+    """
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'rmsnap', pool_name, snapshot_name]
+    try:
+        check_call(cmd)
+    except CalledProcessError:
+        raise
+
+
+# max_bytes should be an int or long
+def set_pool_quota(service, pool_name, max_bytes):
+    """
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param pool_name: six.string_types
+    :param max_bytes: int or long
+    :return: None.  Can raise CalledProcessError
+    """
+    # Set a byte quota on a RADOS pool in ceph.
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, 'max_bytes', max_bytes]
+    try:
+        check_call(cmd)
+    except CalledProcessError:
+        raise
+
+
+def remove_pool_quota(service, pool_name):
+    """
+    Set a byte quota on a RADOS pool in ceph.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param pool_name: six.string_types
+    :return: None.  Can raise CalledProcessError
+    """
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, 'max_bytes', '0']
+    try:
+        check_call(cmd)
+    except CalledProcessError:
+        raise
+
+
+def create_erasure_profile(service, profile_name, erasure_plugin_name='jerasure', failure_domain='host',
+                           data_chunks=2, coding_chunks=1,
+                           locality=None, durability_estimator=None):
+    """
+    Create a new erasure code profile if one does not already exist for it.  Updates
+    the profile if it exists. Please see http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/
+    for more details
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param profile_name: six.string_types
+    :param erasure_plugin_name: six.string_types
+    :param failure_domain: six.string_types.  One of ['chassis', 'datacenter', 'host', 'osd', 'pdu', 'pod', 'rack', 'region',
+        'room', 'root', 'row'])
+    :param data_chunks: int
+    :param coding_chunks: int
+    :param locality: int
+    :param durability_estimator: int
+    :return: None.  Can raise CalledProcessError
+    """
+    # Ensure this failure_domain is allowed by Ceph
+    validator(failure_domain, six.string_types,
+              ['chassis', 'datacenter', 'host', 'osd', 'pdu', 'pod', 'rack', 'region', 'room', 'root', 'row'])
+
+    cmd = ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'set', profile_name,
+           'plugin=' + erasure_plugin_name, 'k=' + str(data_chunks), 'm=' + str(coding_chunks),
+           'ruleset_failure_domain=' + failure_domain]
+    if locality is not None and durability_estimator is not None:
+        raise ValueError("create_erasure_profile should be called with k, m and one of l or c but not both.")
+
+    # Add plugin specific information
+    if locality is not None:
+        # For local erasure codes
+        cmd.append('l=' + str(locality))
+    if durability_estimator is not None:
+        # For Shec erasure codes
+        cmd.append('c=' + str(durability_estimator))
+
+    if erasure_profile_exists(service, profile_name):
+        cmd.append('--force')
+
+    try:
+        check_call(cmd)
+    except CalledProcessError:
+        raise
+
+
+def rename_pool(service, old_name, new_name):
+    """
+    Rename a Ceph pool from old_name to new_name
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param old_name: six.string_types
+    :param new_name: six.string_types
+    :return: None
+    """
+    validator(value=old_name, valid_type=six.string_types)
+    validator(value=new_name, valid_type=six.string_types)
+
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'rename', old_name, new_name]
+    check_call(cmd)
+
+
+def erasure_profile_exists(service, name):
+    """
+    Check to see if an Erasure code profile already exists.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param name: six.string_types
+    :return: int or None
+    """
+    validator(value=name, valid_type=six.string_types)
+    try:
+        check_call(['ceph', '--id', service,
+                    'osd', 'erasure-code-profile', 'get',
+                    name])
+        return True
+    except CalledProcessError:
+        return False
+
+
+def get_cache_mode(service, pool_name):
+    """
+    Find the current caching mode of the pool_name given.
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param pool_name: six.string_types
+    :return: int or None
+    """
+    validator(value=service, valid_type=six.string_types)
+    validator(value=pool_name, valid_type=six.string_types)
+    out = check_output(['ceph', '--id', service, 'osd', 'dump', '--format=json'])
+    try:
+        osd_json = json.loads(out)
+        for pool in osd_json['pools']:
+            if pool['pool_name'] == pool_name:
+                return pool['cache_mode']
+        return None
+    except ValueError:
+        raise
+
+
+def pool_exists(service, name):
+    """Check to see if a RADOS pool already exists."""
+    try:
+        out = check_output(['rados', '--id', service,
+                            'lspools']).decode('UTF-8')
+    except CalledProcessError:
+        return False
+
+    return name in out
+
+
+def get_osds(service):
+    """Return a list of all Ceph Object Storage Daemons currently in the
+    cluster.
+    """
+    version = ceph_version()
+    if version and version >= '0.56':
+        return json.loads(check_output(['ceph', '--id', service,
+                                        'osd', 'ls',
+                                        '--format=json']).decode('UTF-8'))
+
+    return None
 
 
 def install():
@@ -99,53 +491,37 @@ def create_rbd_image(service, pool, image, sizemb):
     check_call(cmd)
 
 
-def pool_exists(service, name):
+def update_pool(client, pool, settings):
-    """Check to see if a RADOS pool already exists."""
+    cmd = ['ceph', '--id', client, 'osd', 'pool', 'set', pool]
-    try:
+    for k, v in six.iteritems(settings):
-        out = check_output(['rados', '--id', service,
+        cmd.append(k)
-                            'lspools']).decode('UTF-8')
+        cmd.append(v)
-    except CalledProcessError:
-        return False
 
-    return name in out
+    check_call(cmd)
 
 
-def get_osds(service):
+def create_pool(service, name, replicas=3, pg_num=None):
-    """Return a list of all Ceph Object Storage Daemons currently in the
-    cluster.
-    """
-    version = ceph_version()
-    if version and version >= '0.56':
-        return json.loads(check_output(['ceph', '--id', service,
-                                        'osd', 'ls',
-                                        '--format=json']).decode('UTF-8'))
-
-    return None
-
-
-def create_pool(service, name, replicas=3):
     """Create a new RADOS pool."""
     if pool_exists(service, name):
         log("Ceph pool {} already exists, skipping creation".format(name),
             level=WARNING)
         return
 
-    # Calculate the number of placement groups based
+    if not pg_num:
-    # on upstream recommended best practices.
+        # Calculate the number of placement groups based
-    osds = get_osds(service)
+        # on upstream recommended best practices.
-    if osds:
+        osds = get_osds(service)
-        pgnum = (len(osds) * 100 // replicas)
+        if osds:
-    else:
+            pg_num = (len(osds) * 100 // replicas)
-        # NOTE(james-page): Default to 200 for older ceph versions
+        else:
-        # which don't support OSD query from cli
+            # NOTE(james-page): Default to 200 for older ceph versions
-        pgnum = 200
+            # which don't support OSD query from cli
+            pg_num = 200
 
-    cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pgnum)]
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pg_num)]
     check_call(cmd)
 
-    cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', name, 'size',
+    update_pool(service, name, settings={'size': str(replicas)})
-           str(replicas)]
-    check_call(cmd)
 
 
 def delete_pool(service, name):
@@ -200,10 +576,10 @@ def create_key_file(service, key):
     log('Created new keyfile at %s.' % keyfile, level=INFO)
 
 
-def get_ceph_nodes():
+def get_ceph_nodes(relation='ceph'):
-    """Query named relation 'ceph' to determine current nodes."""
+    """Query named relation to determine current nodes."""
     hosts = []
-    for r_id in relation_ids('ceph'):
+    for r_id in relation_ids(relation):
         for unit in related_units(r_id):
             hosts.append(relation_get('private-address', unit=unit, rid=r_id))
 
@@ -291,17 +667,6 @@ def place_data_on_block_device(blk_device, data_src_dst):
     os.chown(data_src_dst, uid, gid)
 
 
-# TODO: re-use
-def modprobe(module):
-    """Load a kernel module and configure for auto-load on reboot."""
-    log('Loading kernel module', level=INFO)
-    cmd = ['modprobe', module]
-    check_call(cmd)
-    with open('/etc/modules', 'r+') as modules:
-        if module not in modules.read():
-            modules.write(module)
-
-
 def copy_files(src, dst, symlinks=False, ignore=None):
     """Copy files from src to dst."""
     for item in os.listdir(src):
@@ -366,14 +731,14 @@ def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
             service_start(svc)
 
 
-def ensure_ceph_keyring(service, user=None, group=None):
+def ensure_ceph_keyring(service, user=None, group=None, relation='ceph'):
     """Ensures a ceph keyring is created for a named service and optionally
     ensures user and group ownership.
 
     Returns False if no ceph key is available in relation state.
     """
     key = None
-    for rid in relation_ids('ceph'):
+    for rid in relation_ids(relation):
         for unit in related_units(rid):
             key = relation_get('key', rid=rid, unit=unit)
             if key:
@@ -414,6 +779,7 @@ class CephBrokerRq(object):
 
     The API is versioned and defaults to version 1.
     """
+
     def __init__(self, api_version=1, request_id=None):
         self.api_version = api_version
         if request_id:
@@ -422,9 +788,16 @@ class CephBrokerRq(object):
             self.request_id = str(uuid.uuid1())
         self.ops = []
 
-    def add_op_create_pool(self, name, replica_count=3):
+    def add_op_create_pool(self, name, replica_count=3, pg_num=None):
+        """Adds an operation to create a pool.
+
+        @param pg_num setting:  optional setting. If not provided, this value
+        will be calculated by the broker based on how many OSDs are in the
+        cluster at the time of creation. Note that, if provided, this value
+        will be capped at the current available maximum.
+        """
         self.ops.append({'op': 'create-pool', 'name': name,
-                         'replicas': replica_count})
+                         'replicas': replica_count, 'pg_num': pg_num})
 
     def set_ops(self, ops):
         """Set request ops to provided value.
@@ -442,8 +815,8 @@ class CephBrokerRq(object):
     def _ops_equal(self, other):
         if len(self.ops) == len(other.ops):
             for req_no in range(0, len(self.ops)):
-                for key in ['replicas', 'name', 'op']:
+                for key in ['replicas', 'name', 'op', 'pg_num']:
-                    if self.ops[req_no][key] != other.ops[req_no][key]:
+                    if self.ops[req_no].get(key) != other.ops[req_no].get(key):
                         return False
         else:
             return False
@@ -549,7 +922,7 @@ def get_previous_request(rid):
     return request
 
 
-def get_request_states(request):
+def get_request_states(request, relation='ceph'):
     """Return a dict of requests per relation id with their corresponding
        completion state.
 
@@ -561,7 +934,7 @@ def get_request_states(request):
     """
     complete = []
     requests = {}
-    for rid in relation_ids('ceph'):
+    for rid in relation_ids(relation):
         complete = False
         previous_request = get_previous_request(rid)
         if request == previous_request:
@@ -579,14 +952,14 @@ def get_request_states(request):
     return requests
 
 
-def is_request_sent(request):
+def is_request_sent(request, relation='ceph'):
     """Check to see if a functionally equivalent request has already been sent
 
     Returns True if a similair request has been sent
 
     @param request: A CephBrokerRq object
     """
-    states = get_request_states(request)
+    states = get_request_states(request, relation=relation)
     for rid in states.keys():
         if not states[rid]['sent']:
             return False
@@ -594,7 +967,7 @@ def is_request_sent(request):
     return True
 
 
-def is_request_complete(request):
+def is_request_complete(request, relation='ceph'):
     """Check to see if a functionally equivalent request has already been
     completed
 
@@ -602,7 +975,7 @@ def is_request_complete(request):
 
     @param request: A CephBrokerRq object
     """
-    states = get_request_states(request)
+    states = get_request_states(request, relation=relation)
     for rid in states.keys():
         if not states[rid]['complete']:
             return False
@@ -652,15 +1025,15 @@ def get_broker_rsp_key():
     return 'broker-rsp-' + local_unit().replace('/', '-')
 
 
-def send_request_if_needed(request):
+def send_request_if_needed(request, relation='ceph'):
     """Send broker request if an equivalent request has not already been sent
 
     @param request: A CephBrokerRq object
     """
-    if is_request_sent(request):
+    if is_request_sent(request, relation=relation):
         log('Request already sent but not complete, not sending new request',
             level=DEBUG)
     else:
-        for rid in relation_ids('ceph'):
+        for rid in relation_ids(relation):
             log('Sending request {}'.format(request.request_id), level=DEBUG)
             relation_set(relation_id=rid, broker_req=request.request)

charmhelpers/core/hookenv.py

@@ -490,6 +490,19 @@ def relation_types():
     return rel_types
 
 
+@cached
+def peer_relation_id():
+    '''Get the peers relation id if a peers relation has been joined, else None.'''
+    md = metadata()
+    section = md.get('peers')
+    if section:
+        for key in section:
+            relids = relation_ids(key)
+            if relids:
+                return relids[0]
+    return None
+
+
 @cached
 def relation_to_interface(relation_name):
     """
@@ -504,12 +517,12 @@ def relation_to_interface(relation_name):
 def relation_to_role_and_interface(relation_name):
     """
     Given the name of a relation, return the role and the name of the interface
-    that relation uses (where role is one of ``provides``, ``requires``, or ``peer``).
+    that relation uses (where role is one of ``provides``, ``requires``, or ``peers``).
 
     :returns: A tuple containing ``(role, interface)``, or ``(None, None)``.
     """
     _metadata = metadata()
-    for role in ('provides', 'requires', 'peer'):
+    for role in ('provides', 'requires', 'peers'):
         interface = _metadata.get(role, {}).get(relation_name, {}).get('interface')
         if interface:
             return role, interface
@@ -521,7 +534,7 @@ def role_and_interface_to_relations(role, interface_name):
     """
     Given a role and interface name, return a list of relation names for the
     current charm that use that interface under that role (where role is one
-    of ``provides``, ``requires``, or ``peer``).
+    of ``provides``, ``requires``, or ``peers``).
 
     :returns: A list of relation names.
     """
@@ -542,7 +555,7 @@ def interface_to_relations(interface_name):
     :returns: A list of relation names.
     """
     results = []
-    for role in ('provides', 'requires', 'peer'):
+    for role in ('provides', 'requires', 'peers'):
         results.extend(role_and_interface_to_relations(role, interface_name))
     return results
 
@@ -623,6 +636,38 @@ def unit_private_ip():
     return unit_get('private-address')
 
 
+@cached
+def storage_get(attribute=None, storage_id=None):
+    """Get storage attributes"""
+    _args = ['storage-get', '--format=json']
+    if storage_id:
+        _args.extend(('-s', storage_id))
+    if attribute:
+        _args.append(attribute)
+    try:
+        return json.loads(subprocess.check_output(_args).decode('UTF-8'))
+    except ValueError:
+        return None
+
+
+@cached
+def storage_list(storage_name=None):
+    """List the storage IDs for the unit"""
+    _args = ['storage-list', '--format=json']
+    if storage_name:
+        _args.append(storage_name)
+    try:
+        return json.loads(subprocess.check_output(_args).decode('UTF-8'))
+    except ValueError:
+        return None
+    except OSError as e:
+        import errno
+        if e.errno == errno.ENOENT:
+            # storage-list does not exist
+            return []
+        raise
+
+
 class UnregisteredHookError(Exception):
     """Raised when an undefined hook is called"""
     pass
@@ -788,6 +833,7 @@ def status_get():
 
 def translate_exc(from_exc, to_exc):
     def inner_translate_exc1(f):
+        @wraps(f)
         def inner_translate_exc2(*args, **kwargs):
             try:
                 return f(*args, **kwargs)
@@ -832,6 +878,40 @@ def leader_set(settings=None, **kwargs):
     subprocess.check_call(cmd)
 
 
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def payload_register(ptype, klass, pid):
+    """ is used while a hook is running to let Juju know that a
+        payload has been started."""
+    cmd = ['payload-register']
+    for x in [ptype, klass, pid]:
+        cmd.append(x)
+    subprocess.check_call(cmd)
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def payload_unregister(klass, pid):
+    """ is used while a hook is running to let Juju know
+    that a payload has been manually stopped. The <class> and <id> provided
+    must match a payload that has been previously registered with juju using
+    payload-register."""
+    cmd = ['payload-unregister']
+    for x in [klass, pid]:
+        cmd.append(x)
+    subprocess.check_call(cmd)
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def payload_status_set(klass, pid, status):
+    """is used to update the current status of a registered payload.
+    The <class> and <id> provided must match a payload that has been previously
+    registered with juju using payload-register. The <status> must be one of the
+    follow: starting, started, stopping, stopped"""
+    cmd = ['payload-status-set']
+    for x in [klass, pid, status]:
+        cmd.append(x)
+    subprocess.check_call(cmd)
+
+
 @cached
 def juju_version():
     """Full version string (eg. '1.23.3.1-trusty-amd64')"""

charmhelpers/core/host.py

@@ -63,55 +63,85 @@ def service_reload(service_name, restart_on_failure=False):
     return service_result
 
 
-def service_pause(service_name, init_dir=None):
+def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"):
     """Pause a system service.
 
     Stop it, and prevent it from starting again at boot."""
-    if init_dir is None:
+    stopped = True
-        init_dir = "/etc/init"
+    if service_running(service_name):
-    stopped = service_stop(service_name)
+        stopped = service_stop(service_name)
-    # XXX: Support systemd too
+    upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
-    override_path = os.path.join(
+    sysv_file = os.path.join(initd_dir, service_name)
-        init_dir, '{}.override'.format(service_name))
+    if init_is_systemd():
-    with open(override_path, 'w') as fh:
+        service('disable', service_name)
-        fh.write("manual\n")
+    elif os.path.exists(upstart_file):
+        override_path = os.path.join(
+            init_dir, '{}.override'.format(service_name))
+        with open(override_path, 'w') as fh:
+            fh.write("manual\n")
+    elif os.path.exists(sysv_file):
+        subprocess.check_call(["update-rc.d", service_name, "disable"])
+    else:
+        raise ValueError(
+            "Unable to detect {0} as SystemD, Upstart {1} or"
+            " SysV {2}".format(
+                service_name, upstart_file, sysv_file))
     return stopped
 
 
-def service_resume(service_name, init_dir=None):
+def service_resume(service_name, init_dir="/etc/init",
+                   initd_dir="/etc/init.d"):
     """Resume a system service.
 
     Reenable starting again at boot. Start the service"""
-    # XXX: Support systemd too
+    upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
-    if init_dir is None:
+    sysv_file = os.path.join(initd_dir, service_name)
-        init_dir = "/etc/init"
+    if init_is_systemd():
-    override_path = os.path.join(
+        service('enable', service_name)
-        init_dir, '{}.override'.format(service_name))
+    elif os.path.exists(upstart_file):
-    if os.path.exists(override_path):
+        override_path = os.path.join(
-        os.unlink(override_path)
+            init_dir, '{}.override'.format(service_name))
-    started = service_start(service_name)
+        if os.path.exists(override_path):
+            os.unlink(override_path)
+    elif os.path.exists(sysv_file):
+        subprocess.check_call(["update-rc.d", service_name, "enable"])
+    else:
+        raise ValueError(
+            "Unable to detect {0} as SystemD, Upstart {1} or"
+            " SysV {2}".format(
+                service_name, upstart_file, sysv_file))
+
+    started = service_running(service_name)
+    if not started:
+        started = service_start(service_name)
     return started
 
 
 def service(action, service_name):
     """Control a system service"""
-    cmd = ['service', service_name, action]
+    if init_is_systemd():
+        cmd = ['systemctl', action, service_name]
+    else:
+        cmd = ['service', service_name, action]
     return subprocess.call(cmd) == 0
 
 
-def service_running(service):
+def service_running(service_name):
     """Determine whether a system service is running"""
-    try:
+    if init_is_systemd():
-        output = subprocess.check_output(
+        return service('is-active', service_name)
-            ['service', service, 'status'],
-            stderr=subprocess.STDOUT).decode('UTF-8')
-    except subprocess.CalledProcessError:
-        return False
     else:
-        if ("start/running" in output or "is running" in output):
+        try:
-            return True
+            output = subprocess.check_output(
-        else:
+                ['service', service_name, 'status'],
+                stderr=subprocess.STDOUT).decode('UTF-8')
+        except subprocess.CalledProcessError:
             return False
+        else:
+            if ("start/running" in output or "is running" in output):
+                return True
+            else:
+                return False
 
 
 def service_available(service_name):
@@ -126,8 +156,29 @@ def service_available(service_name):
         return True
 
 
-def adduser(username, password=None, shell='/bin/bash', system_user=False):
+SYSTEMD_SYSTEM = '/run/systemd/system'
-    """Add a user to the system"""
+
+
+def init_is_systemd():
+    return os.path.isdir(SYSTEMD_SYSTEM)
+
+
+def adduser(username, password=None, shell='/bin/bash', system_user=False,
+            primary_group=None, secondary_groups=None):
+    """
+    Add a user to the system.
+
+    Will log but otherwise succeed if the user already exists.
+
+    :param str username: Username to create
+    :param str password: Password for user; if ``None``, create a system user
+    :param str shell: The default shell for the user
+    :param bool system_user: Whether to create a login or system user
+    :param str primary_group: Primary group for user; defaults to their username
+    :param list secondary_groups: Optional list of additional groups
+
+    :returns: The password database entry struct, as returned by `pwd.getpwnam`
+    """
     try:
         user_info = pwd.getpwnam(username)
         log('user {0} already exists!'.format(username))
@@ -142,6 +193,16 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False):
                 '--shell', shell,
                 '--password', password,
             ])
+        if not primary_group:
+            try:
+                grp.getgrnam(username)
+                primary_group = username  # avoid "group exists" error
+            except KeyError:
+                pass
+        if primary_group:
+            cmd.extend(['-g', primary_group])
+        if secondary_groups:
+            cmd.extend(['-G', ','.join(secondary_groups)])
         cmd.append(username)
         subprocess.check_call(cmd)
         user_info = pwd.getpwnam(username)
@@ -550,7 +611,14 @@ def chdir(d):
         os.chdir(cur)
 
 
-def chownr(path, owner, group, follow_links=True):
+def chownr(path, owner, group, follow_links=True, chowntopdir=False):
+    """
+    Recursively change user and group ownership of files and directories
+    in given path. Doesn't chown path itself by default, only its children.
+
+    :param bool follow_links: Also Chown links if True
+    :param bool chowntopdir: Also chown path itself if True
+    """
     uid = pwd.getpwnam(owner).pw_uid
     gid = grp.getgrnam(group).gr_gid
     if follow_links:
@@ -558,6 +626,10 @@ def chownr(path, owner, group, follow_links=True):
     else:
         chown = os.lchown
 
+    if chowntopdir:
+        broken_symlink = os.path.lexists(path) and not os.path.exists(path)
+        if not broken_symlink:
+            chown(path, uid, gid)
     for root, dirs, files in os.walk(path):
         for name in dirs + files:
             full = os.path.join(root, name)
@@ -568,3 +640,19 @@ def chownr(path, owner, group, follow_links=True):
 
 def lchownr(path, owner, group):
     chownr(path, owner, group, follow_links=False)
+
+
+def get_total_ram():
+    '''The total amount of system RAM in bytes.
+
+    This is what is reported by the OS, and may be overcommitted when
+    there are multiple containers hosted on the same machine.
+    '''
+    with open('/proc/meminfo', 'r') as f:
+        for line in f.readlines():
+            if line:
+                key, value, unit = line.split()
+                if key == 'MemTotal:':
+                    assert unit == 'kB', 'Unknown unit'
+                    return int(value) * 1024  # Classic, not KiB.
+        raise NotImplementedError()

charmhelpers/core/hugepage.py

@@ -25,11 +25,13 @@ from charmhelpers.core.host import (
     fstab_mount,
     mkdir,
 )
+from charmhelpers.core.strutils import bytes_from_string
+from subprocess import check_output
 
 
 def hugepage_support(user, group='hugetlb', nr_hugepages=256,
                      max_map_count=65536, mnt_point='/run/hugepages/kvm',
-                     pagesize='2MB', mount=True):
+                     pagesize='2MB', mount=True, set_shmmax=False):
     """Enable hugepages on system.
 
     Args:
@@ -44,11 +46,18 @@ def hugepage_support(user, group='hugetlb', nr_hugepages=256,
     group_info = add_group(group)
     gid = group_info.gr_gid
     add_user_to_group(user, group)
+    if max_map_count < 2 * nr_hugepages:
+        max_map_count = 2 * nr_hugepages
     sysctl_settings = {
         'vm.nr_hugepages': nr_hugepages,
         'vm.max_map_count': max_map_count,
         'vm.hugetlb_shm_group': gid,
     }
+    if set_shmmax:
+        shmmax_current = int(check_output(['sysctl', '-n', 'kernel.shmmax']))
+        shmmax_minsize = bytes_from_string(pagesize) * nr_hugepages
+        if shmmax_minsize > shmmax_current:
+            sysctl_settings['kernel.shmmax'] = shmmax_minsize
     sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf')
     mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False)
     lfstab = fstab.Fstab()

charmhelpers/core/kernel.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
+
+from charmhelpers.core.hookenv import (
+    log,
+    INFO
+)
+
+from subprocess import check_call, check_output
+import re
+
+
+def modprobe(module, persist=True):
+    """Load a kernel module and configure for auto-load on reboot."""
+    cmd = ['modprobe', module]
+
+    log('Loading kernel module %s' % module, level=INFO)
+
+    check_call(cmd)
+    if persist:
+        with open('/etc/modules', 'r+') as modules:
+            if module not in modules.read():
+                modules.write(module)
+
+
+def rmmod(module, force=False):
+    """Remove a module from the linux kernel"""
+    cmd = ['rmmod']
+    if force:
+        cmd.append('-f')
+    cmd.append(module)
+    log('Removing kernel module %s' % module, level=INFO)
+    return check_call(cmd)
+
+
+def lsmod():
+    """Shows what kernel modules are currently loaded"""
+    return check_output(['lsmod'],
+                        universal_newlines=True)
+
+
+def is_module_loaded(module):
+    """Checks if a kernel module is already loaded"""
+    matches = re.findall('^%s[ ]+' % module, lsmod(), re.M)
+    return len(matches) > 0
+
+
+def update_initramfs(version='all'):
+    """Updates an initramfs image"""
+    return check_call(["update-initramfs", "-k", version, "-u"])

charmhelpers/core/services/helpers.py

@@ -243,33 +243,40 @@ class TemplateCallback(ManagerCallback):
     :param str source: The template source file, relative to
         `$CHARM_DIR/templates`
 
-    :param str target: The target to write the rendered template to
+    :param str target: The target to write the rendered template to (or None)
     :param str owner: The owner of the rendered file
     :param str group: The group of the rendered file
     :param int perms: The permissions of the rendered file
     :param partial on_change_action: functools partial to be executed when
                                      rendered file changes
+    :param jinja2 loader template_loader: A jinja2 template loader
+
+    :return str: The rendered template
     """
     def __init__(self, source, target,
                  owner='root', group='root', perms=0o444,
-                 on_change_action=None):
+                 on_change_action=None, template_loader=None):
         self.source = source
         self.target = target
         self.owner = owner
         self.group = group
         self.perms = perms
         self.on_change_action = on_change_action
+        self.template_loader = template_loader
 
     def __call__(self, manager, service_name, event_name):
         pre_checksum = ''
         if self.on_change_action and os.path.isfile(self.target):
             pre_checksum = host.file_hash(self.target)
         service = manager.get_service(service_name)
-        context = {}
+        context = {'ctx': {}}
         for ctx in service.get('required_data', []):
             context.update(ctx)
-        templating.render(self.source, self.target, context,
+            context['ctx'].update(ctx)
-                          self.owner, self.group, self.perms)
+
+        result = templating.render(self.source, self.target, context,
+                                   self.owner, self.group, self.perms,
+                                   template_loader=self.template_loader)
         if self.on_change_action:
             if pre_checksum == host.file_hash(self.target):
                 hookenv.log(
@@ -278,6 +285,8 @@ class TemplateCallback(ManagerCallback):
             else:
                 self.on_change_action()
 
+        return result
+
 
 # Convenience aliases for templates
 render_template = template = TemplateCallback

charmhelpers/core/strutils.py

@@ -18,6 +18,7 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import six
+import re
 
 
 def bool_from_string(value):
@@ -40,3 +41,32 @@ def bool_from_string(value):
 
     msg = "Unable to interpret string value '%s' as boolean" % (value)
     raise ValueError(msg)
+
+
+def bytes_from_string(value):
+    """Interpret human readable string value as bytes.
+
+    Returns int
+    """
+    BYTE_POWER = {
+        'K': 1,
+        'KB': 1,
+        'M': 2,
+        'MB': 2,
+        'G': 3,
+        'GB': 3,
+        'T': 4,
+        'TB': 4,
+        'P': 5,
+        'PB': 5,
+    }
+    if isinstance(value, six.string_types):
+        value = six.text_type(value)
+    else:
+        msg = "Unable to interpret non-string value '%s' as boolean" % (value)
+        raise ValueError(msg)
+    matches = re.match("([0-9]+)([a-zA-Z]+)", value)
+    if not matches:
+        msg = "Unable to interpret string value '%s' as bytes" % (value)
+        raise ValueError(msg)
+    return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])

charmhelpers/core/templating.py

@@ -21,13 +21,14 @@ from charmhelpers.core import hookenv
 
 
 def render(source, target, context, owner='root', group='root',
-           perms=0o444, templates_dir=None, encoding='UTF-8'):
+           perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None):
     """
     Render a template.
 
     The `source` path, if not absolute, is relative to the `templates_dir`.
 
-    The `target` path should be absolute.
+    The `target` path should be absolute.  It can also be `None`, in which
+    case no file will be written.
 
     The context should be a dict containing the values to be replaced in the
     template.
@@ -36,6 +37,9 @@ def render(source, target, context, owner='root', group='root',
 
     If omitted, `templates_dir` defaults to the `templates` folder in the charm.
 
+    The rendered template will be written to the file as well as being returned
+    as a string.
+
     Note: Using this requires python-jinja2; if it is not installed, calling
     this will attempt to use charmhelpers.fetch.apt_install to install it.
     """
@@ -52,17 +56,26 @@ def render(source, target, context, owner='root', group='root',
         apt_install('python-jinja2', fatal=True)
         from jinja2 import FileSystemLoader, Environment, exceptions
 
-    if templates_dir is None:
+    if template_loader:
-        templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
+        template_env = Environment(loader=template_loader)
-    loader = Environment(loader=FileSystemLoader(templates_dir))
+    else:
+        if templates_dir is None:
+            templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
+        template_env = Environment(loader=FileSystemLoader(templates_dir))
     try:
         source = source
-        template = loader.get_template(source)
+        template = template_env.get_template(source)
     except exceptions.TemplateNotFound as e:
         hookenv.log('Could not load template %s from %s.' %
                     (source, templates_dir),
                     level=hookenv.ERROR)
         raise e
     content = template.render(context)
-    host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
+    if target is not None:
-    host.write_file(target, content.encode(encoding), owner, group, perms)
+        target_dir = os.path.dirname(target)
+        if not os.path.exists(target_dir):
+            # This is a terrible default directory permission, as the file
+            # or its siblings will often contain secrets.
+            host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
+        host.write_file(target, content.encode(encoding), owner, group, perms)
+    return content

charmhelpers/fetch/__init__.py

@@ -98,6 +98,14 @@ CLOUD_ARCHIVE_POCKETS = {
     'liberty/proposed': 'trusty-proposed/liberty',
     'trusty-liberty/proposed': 'trusty-proposed/liberty',
     'trusty-proposed/liberty': 'trusty-proposed/liberty',
+    # Mitaka
+    'mitaka': 'trusty-updates/mitaka',
+    'trusty-mitaka': 'trusty-updates/mitaka',
+    'trusty-mitaka/updates': 'trusty-updates/mitaka',
+    'trusty-updates/mitaka': 'trusty-updates/mitaka',
+    'mitaka/proposed': 'trusty-proposed/mitaka',
+    'trusty-mitaka/proposed': 'trusty-proposed/mitaka',
+    'trusty-proposed/mitaka': 'trusty-proposed/mitaka',
 }
 
 # The order of this list is very important. Handlers should be listed in from
@@ -225,12 +233,12 @@ def apt_purge(packages, fatal=False):
 
 def apt_mark(packages, mark, fatal=False):
     """Flag one or more packages using apt-mark"""
+    log("Marking {} as {}".format(packages, mark))
     cmd = ['apt-mark', mark]
     if isinstance(packages, six.string_types):
         cmd.append(packages)
     else:
         cmd.extend(packages)
-    log("Holding {}".format(packages))
 
     if fatal:
         subprocess.check_call(cmd, universal_newlines=True)
@@ -411,7 +419,7 @@ def plugins(fetch_handlers=None):
                 importlib.import_module(package),
                 classname)
             plugin_list.append(handler_class())
-        except (ImportError, AttributeError):
+        except NotImplementedError:
             # Skip missing plugins so that they can be ommitted from
             # installation if desired
             log("FetchHandler {} not found, skipping plugin".format(

charmhelpers/fetch/archiveurl.py

@@ -108,7 +108,7 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
                 install_opener(opener)
         response = urlopen(source)
         try:
-            with open(dest, 'w') as dest_file:
+            with open(dest, 'wb') as dest_file:
                 dest_file.write(response.read())
         except Exception as e:
             if os.path.isfile(dest):

charmhelpers/fetch/bzrurl.py

@@ -15,60 +15,50 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+from subprocess import check_call
 from charmhelpers.fetch import (
     BaseFetchHandler,
-    UnhandledSource
+    UnhandledSource,
+    filter_installed_packages,
+    apt_install,
 )
 from charmhelpers.core.host import mkdir
 
-import six
-if six.PY3:
-    raise ImportError('bzrlib does not support Python3')
 
-try:
+if filter_installed_packages(['bzr']) != []:
-    from bzrlib.branch import Branch
+    apt_install(['bzr'])
-    from bzrlib import bzrdir, workingtree, errors
+    if filter_installed_packages(['bzr']) != []:
-except ImportError:
+        raise NotImplementedError('Unable to install bzr')
-    from charmhelpers.fetch import apt_install
-    apt_install("python-bzrlib")
-    from bzrlib.branch import Branch
-    from bzrlib import bzrdir, workingtree, errors
 
 
 class BzrUrlFetchHandler(BaseFetchHandler):
     """Handler for bazaar branches via generic and lp URLs"""
     def can_handle(self, source):
         url_parts = self.parse_url(source)
-        if url_parts.scheme not in ('bzr+ssh', 'lp'):
+        if url_parts.scheme not in ('bzr+ssh', 'lp', ''):
             return False
+        elif not url_parts.scheme:
+            return os.path.exists(os.path.join(source, '.bzr'))
         else:
             return True
 
     def branch(self, source, dest):
-        url_parts = self.parse_url(source)
-        # If we use lp:branchname scheme we need to load plugins
         if not self.can_handle(source):
             raise UnhandledSource("Cannot handle {}".format(source))
-        if url_parts.scheme == "lp":
+        if os.path.exists(dest):
-            from bzrlib.plugin import load_plugins
+            check_call(['bzr', 'pull', '--overwrite', '-d', dest, source])
-            load_plugins()
+        else:
-        try:
+            check_call(['bzr', 'branch', source, dest])
-            local_branch = bzrdir.BzrDir.create_branch_convenience(dest)
-        except errors.AlreadyControlDirError:
-            local_branch = Branch.open(dest)
-        try:
-            remote_branch = Branch.open(source)
-            remote_branch.push(local_branch)
-            tree = workingtree.WorkingTree.open(dest)
-            tree.update()
-        except Exception as e:
-            raise e
 
-    def install(self, source):
+    def install(self, source, dest=None):
         url_parts = self.parse_url(source)
         branch_name = url_parts.path.strip("/").split("/")[-1]
-        dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
+        if dest:
-                                branch_name)
+            dest_dir = os.path.join(dest, branch_name)
+        else:
+            dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
+                                    branch_name)
+
         if not os.path.exists(dest_dir):
             mkdir(dest_dir, perms=0o755)
         try:

charmhelpers/fetch/giturl.py

@@ -15,24 +15,18 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+from subprocess import check_call
 from charmhelpers.fetch import (
     BaseFetchHandler,
-    UnhandledSource
+    UnhandledSource,
+    filter_installed_packages,
+    apt_install,
 )
-from charmhelpers.core.host import mkdir
 
-import six
+if filter_installed_packages(['git']) != []:
-if six.PY3:
+    apt_install(['git'])
-    raise ImportError('GitPython does not support Python 3')
+    if filter_installed_packages(['git']) != []:
-
+        raise NotImplementedError('Unable to install git')
-try:
-    from git import Repo
-except ImportError:
-    from charmhelpers.fetch import apt_install
-    apt_install("python-git")
-    from git import Repo
-
-from git.exc import GitCommandError  # noqa E402
 
 
 class GitUrlFetchHandler(BaseFetchHandler):
@@ -40,19 +34,24 @@ class GitUrlFetchHandler(BaseFetchHandler):
     def can_handle(self, source):
         url_parts = self.parse_url(source)
         # TODO (mattyw) no support for ssh git@ yet
-        if url_parts.scheme not in ('http', 'https', 'git'):
+        if url_parts.scheme not in ('http', 'https', 'git', ''):
             return False
+        elif not url_parts.scheme:
+            return os.path.exists(os.path.join(source, '.git'))
         else:
             return True
 
-    def clone(self, source, dest, branch, depth=None):
+    def clone(self, source, dest, branch="master", depth=None):
         if not self.can_handle(source):
             raise UnhandledSource("Cannot handle {}".format(source))
 
-        if depth:
+        if os.path.exists(dest):
-            Repo.clone_from(source, dest, branch=branch, depth=depth)
+            cmd = ['git', '-C', dest, 'pull', source, branch]
         else:
-            Repo.clone_from(source, dest, branch=branch)
+            cmd = ['git', 'clone', source, dest, '--branch', branch]
+        if depth:
+            cmd.extend(['--depth', depth])
+        check_call(cmd)
 
     def install(self, source, branch="master", dest=None, depth=None):
         url_parts = self.parse_url(source)
@@ -62,12 +61,8 @@ class GitUrlFetchHandler(BaseFetchHandler):
         else:
             dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
                                     branch_name)
-        if not os.path.exists(dest_dir):
-            mkdir(dest_dir, perms=0o755)
         try:
             self.clone(source, dest_dir, branch, depth)
-        except GitCommandError as e:
-            raise UnhandledSource(e)
         except OSError as e:
             raise UnhandledSource(e.strerror)
         return dest_dir

charmhelpers/payload/archive.py

@@ -0,0 +1,73 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import tarfile
+import zipfile
+from charmhelpers.core import (
+    host,
+    hookenv,
+)
+
+
+class ArchiveError(Exception):
+    pass
+
+
+def get_archive_handler(archive_name):
+    if os.path.isfile(archive_name):
+        if tarfile.is_tarfile(archive_name):
+            return extract_tarfile
+        elif zipfile.is_zipfile(archive_name):
+            return extract_zipfile
+    else:
+        # look at the file name
+        for ext in ('.tar', '.tar.gz', '.tgz', 'tar.bz2', '.tbz2', '.tbz'):
+            if archive_name.endswith(ext):
+                return extract_tarfile
+        for ext in ('.zip', '.jar'):
+            if archive_name.endswith(ext):
+                return extract_zipfile
+
+
+def archive_dest_default(archive_name):
+    archive_file = os.path.basename(archive_name)
+    return os.path.join(hookenv.charm_dir(), "archives", archive_file)
+
+
+def extract(archive_name, destpath=None):
+    handler = get_archive_handler(archive_name)
+    if handler:
+        if not destpath:
+            destpath = archive_dest_default(archive_name)
+        if not os.path.isdir(destpath):
+            host.mkdir(destpath)
+        handler(archive_name, destpath)
+        return destpath
+    else:
+        raise ArchiveError("No handler for archive")
+
+
+def extract_tarfile(archive_name, destpath):
+    "Unpack a tar archive, optionally compressed"
+    archive = tarfile.open(archive_name)
+    archive.extractall(destpath)
+
+
+def extract_zipfile(archive_name, destpath):
+    "Unpack a zip file"
+    archive = zipfile.ZipFile(archive_name)
+    archive.extractall(destpath)

config.yaml

@@ -134,6 +134,18 @@ options:
     description: |
        Client timeout configuration in ms for haproxy, used in HA
        configurations. If not provided, default value of 30000ms is used.
+  haproxy-queue-timeout:
+    type: int
+    default:
+    description: |
+       Queue timeout configuration in ms for haproxy, used in HA
+       configurations. If not provided, default value of 5000ms is used.
+  haproxy-connect-timeout:
+    type: int
+    default:
+    description: |
+       Connect timeout configuration in ms for haproxy, used in HA
+       configurations. If not provided, default value of 5000ms is used.
   ssl_cert:
     type: string
     default:
@@ -243,3 +255,10 @@ options:
       wait for you to execute the openstack-upgrade action for this charm on
       each unit. If False it will revert to existing behavior of upgrading
       all units on config change.
+  expose-image-locations:
+    type: boolean
+    default: True
+    description: |
+      Expose underlying image locations via the API when using Ceph for image
+      storage.  Only disable this option if you do not wish to use copy-on-write
+      clones of RAW format images with Ceph in Cinder and Nova.

hooks/glance_contexts.py

@@ -32,6 +32,7 @@ class CephGlanceContext(OSContextGenerator):
             # pool created based on service name.
             'rbd_pool': service,
             'rbd_user': service,
+            'expose_image_locations': config('expose-image-locations')
         }
 
 

hooks/glance_relations.py

@@ -23,7 +23,10 @@ from glance_utils import (
     GLANCE_API_PASTE_INI,
     HAPROXY_CONF,
     ceph_config_file,
-    setup_ipv6
+    setup_ipv6,
+    REQUIRED_INTERFACES,
+    check_optional_relations,
+    swift_temp_url_key
 )
 from charmhelpers.core.hookenv import (
     config,
@@ -38,7 +41,8 @@ from charmhelpers.core.hookenv import (
     relation_ids,
     service_name,
     unit_get,
-    UnregisteredHookError
+    UnregisteredHookError,
+    status_set,
 )
 from charmhelpers.core.host import (
     restart_on_change,
@@ -63,6 +67,7 @@ from charmhelpers.contrib.openstack.utils import (
     openstack_upgrade_available,
     os_release,
     sync_db_with_multi_ipv6_addresses,
+    set_os_workload_status,
 )
 from charmhelpers.contrib.storage.linux.ceph import (
     send_request_if_needed,
@@ -95,9 +100,9 @@ hooks = Hooks()
 CONFIGS = register_configs()
 
 
-@hooks.hook('install')
+@hooks.hook('install.real')
 def install_hook():
-    juju_log('Installing glance packages')
+    status_set('maintenance', 'Executing pre-install')
     execd_preinstall()
     src = config('openstack-origin')
     if (lsb_release()['DISTRIB_CODENAME'] == 'precise' and
@@ -106,9 +111,11 @@ def install_hook():
 
     configure_installation_source(src)
 
+    status_set('maintenance', 'Installing apt packages')
     apt_update(fatal=True)
     apt_install(determine_packages(), fatal=True)
 
+    status_set('maintenance', 'Git install')
     git_install(config('openstack-origin-git'))
 
     for service in SERVICES:
@@ -216,6 +223,13 @@ def image_service_joined(relation_id=None):
     juju_log("%s: image-service_joined: To peer glance-api-server=%s" %
              (CHARM, relation_data['glance-api-server']))
 
+    if ('object-store' in CONFIGS.complete_contexts() and
+       'identity-service' in CONFIGS.complete_contexts()):
+        relation_data.update({
+            'swift-temp-url-key': swift_temp_url_key(),
+            'swift-container': 'glance'
+        })
+
     relation_set(relation_id=relation_id, **relation_data)
 
 
@@ -232,6 +246,8 @@ def object_store_joined():
         juju_log('swift relation incomplete')
         return
 
+    [image_service_joined(rid) for rid in relation_ids('image-service')]
+
     CONFIGS.write(GLANCE_API_CONF)
 
 
@@ -321,15 +337,17 @@ def keystone_changed():
 def config_changed():
     if config('prefer-ipv6'):
         setup_ipv6()
+        status_set('maintenance', 'Sync DB')
         sync_db_with_multi_ipv6_addresses(config('database'),
                                           config('database-user'))
 
     if git_install_requested():
         if config_value_changed('openstack-origin-git'):
+            status_set('maintenance', 'Running Git install')
             git_install(config('openstack-origin-git'))
     elif not config('action-managed-upgrade'):
         if openstack_upgrade_available('glance-common'):
-            juju_log('Upgrading OpenStack release')
+            status_set('maintenance', 'Upgrading OpenStack release')
             do_openstack_upgrade(CONFIGS)
 
     open_port(9292)
@@ -518,3 +536,5 @@ if __name__ == '__main__':
         hooks.execute(sys.argv)
     except UnregisteredHookError as e:
         juju_log('Unknown hook {} - skipping.'.format(e))
+    set_os_workload_status(CONFIGS, REQUIRED_INTERFACES,
+                           charm_func=check_optional_relations)

hooks/glance_utils.py

@@ -24,7 +24,10 @@ from charmhelpers.core.hookenv import (
     config,
     log,
     relation_ids,
-    service_name)
+    service_name,
+    status_get,
+)
+
 
 from charmhelpers.core.host import (
     adduser,
@@ -36,6 +39,7 @@ from charmhelpers.core.host import (
     service_restart,
     lsb_release,
     write_file,
+    pwgen
 )
 
 from charmhelpers.contrib.openstack import (
@@ -44,6 +48,7 @@ from charmhelpers.contrib.openstack import (
 
 from charmhelpers.contrib.hahelpers.cluster import (
     is_elected_leader,
+    get_hacluster_config,
 )
 
 from charmhelpers.contrib.openstack.alternatives import install_alternative
@@ -56,6 +61,7 @@ from charmhelpers.contrib.openstack.utils import (
     git_pip_venv_dir,
     configure_installation_source,
     os_release,
+    set_os_workload_status,
 )
 
 from charmhelpers.core.templating import render
@@ -64,6 +70,7 @@ from charmhelpers.core.decorators import (
     retry_on_exception,
 )
 
+
 CLUSTER_RES = "grp_glance_vips"
 
 PACKAGES = [
@@ -115,6 +122,13 @@ CONF_DIR = "/etc/glance"
 
 TEMPLATES = 'templates/'
 
+# The interface is said to be satisfied if anyone of the interfaces in the
+# list has a complete context.
+REQUIRED_INTERFACES = {
+    'database': ['shared-db', 'pgsql-db'],
+    'identity': ['identity-service'],
+}
+
 
 def ceph_config_file():
     return CHARM_CEPH_CONF.format(service_name())
@@ -305,12 +319,11 @@ def setup_ipv6():
         raise Exception("IPv6 is not supported in the charms for Ubuntu "
                         "versions less than Trusty 14.04")
 
-    # NOTE(xianghui): Need to install haproxy(1.5.3) from trusty-backports
+    # Need haproxy >= 1.5.3 for ipv6 so for Trusty if we are <= Kilo we need to
-    # to support ipv6 address, so check is required to make sure not
+    # use trusty-backports otherwise we can use the UCA.
-    # breaking other versions, IPv6 only support for >= Trusty
+    if ubuntu_rel == 'trusty' and os_release('glance') < 'liberty':
-    if ubuntu_rel == 'trusty':
+        add_source('deb http://archive.ubuntu.com/ubuntu trusty-backports '
-        add_source('deb http://archive.ubuntu.com/ubuntu trusty-backports'
+                   'main')
-                   ' main')
         apt_update()
         apt_install('haproxy/trusty-backports', fatal=True)
 
@@ -424,3 +437,57 @@ def git_post_install(projects_yaml):
 
     service_restart('glance-api')
     service_restart('glance-registry')
+
+
+def check_optional_relations(configs):
+    required_interfaces = {}
+    if relation_ids('ha'):
+        required_interfaces['ha'] = ['cluster']
+        try:
+            get_hacluster_config()
+        except:
+            return ('blocked',
+                    'hacluster missing configuration: '
+                    'vip, vip_iface, vip_cidr')
+
+    if relation_ids('ceph') or relation_ids('object-store'):
+        required_interfaces['storage-backend'] = ['ceph', 'object-store']
+
+    if relation_ids('amqp'):
+        required_interfaces['messaging'] = ['amqp']
+
+    if required_interfaces:
+        set_os_workload_status(configs, required_interfaces)
+        return status_get()
+    else:
+        return 'unknown', 'No optional relations'
+
+
+def swift_temp_url_key():
+    """Generate a temp URL key, post it to Swift and return its value.
+       If it is already posted, the current value of the key will be returned.
+    """
+    keystone_ctxt = context.IdentityServiceContext(service='glance',
+                                                   service_user='glance')()
+    if not keystone_ctxt:
+        log('Missing identity-service relation. Skipping generation of '
+            'swift temporary url key.')
+        return
+
+    auth_url = '%s://%s:%s/v2.0/' % (keystone_ctxt['service_protocol'],
+                                     keystone_ctxt['service_host'],
+                                     keystone_ctxt['service_port'])
+    from swiftclient import client
+    swift_connection = client.Connection(
+        authurl=auth_url, user='glance', key=keystone_ctxt['admin_password'],
+        tenant_name=keystone_ctxt['admin_tenant_name'], auth_version='2.0')
+
+    account_stats = swift_connection.head_account()
+    if 'x-account-meta-temp-url-key' in account_stats:
+        log("Temp URL key was already posted.")
+        return account_stats['x-account-meta-temp-url-key']
+
+    temp_url_key = pwgen(length=64)
+    swift_connection.post_account(headers={'x-account-meta-temp-url-key':
+                                           temp_url_key})
+    return temp_url_key

hooks/install

@@ -1 +0,0 @@
-glance_relations.py

hooks/install

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Wrapper to deal with newer Ubuntu versions that don't have py2 installed
+# by default.
+
+declare -a DEPS=('apt' 'netaddr' 'netifaces' 'pip' 'yaml')
+
+check_and_install() {
+    pkg="${1}-${2}"
+    if ! dpkg -s ${pkg} 2>&1 > /dev/null; then
+        apt-get -y install ${pkg}
+    fi
+}
+
+PYTHON="python"
+
+for dep in ${DEPS[@]}; do
+    check_and_install ${PYTHON} ${dep}
+done
+
+exec ./hooks/install.real

hooks/install.real

@@ -0,0 +1 @@
+glance_relations.py

hooks/update-status

@@ -0,0 +1 @@
+glance_relations.py

metadata.yaml

@@ -1,11 +1,11 @@
 name: glance
-maintainer: Adam Gandelman <adamg@canonical.com>
+maintainer: OpenStack Charmers <openstack-charmers@lists.ubuntu.com>
-summary: "OpenStack Image Registry and Delivery Service"
+summary: OpenStack Image Registry and Delivery Service
 description: |
  The Glance project provides an image registration and discovery service
- (Parallax) and an image delivery service (Teller). These services are used
+ and an image delivery service. These services are used in conjunction
- in conjunction by Nova to deliver images from object stores, such as
+ by Nova to deliver images from object stores, such as OpenStack's Swift
- OpenStack's Swift service, to Nova's compute nodes.
+ service, to Nova's compute nodes.
 tags:
   - openstack
   - storage

requirements/requirements-precise.txt

@@ -0,0 +1,6 @@
+PyYAML==3.10
+simplejson==2.3.2
+netifaces==0.8
+netaddr==0.7.10
+Jinja2==2.6
+six==1.1.0

requirements/requirements-trusty.txt

@@ -0,0 +1,6 @@
+PyYAML>=3.10
+simplejson==3.3.1
+netifaces==0.8
+netaddr==0.7.10
+Jinja2==2.7.2
+six==1.5.2

requirements/test-requirements.txt

@@ -0,0 +1,7 @@
+testtools==0.9.35
+coverage==3.7.1
+mock==1.0.1
+flake8==2.1.0
+# No version required
+charm-tools
+os-testr

templates/kilo/glance-api.conf

@@ -20,6 +20,10 @@ registry_host = {{ registry_host }}
 registry_port = 9191
 registry_client_protocol = http
 
+{% if expose_image_locations -%}
+show_multiple_locations = {{ expose_image_locations }}
+{% endif -%}
+
 {% if api_config_flags -%}
 {% for key, value in api_config_flags.iteritems() -%}
 {{ key }} = {{ value }}

tests/019-basic-trusty-mitaka

@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic glance deployment on trusty-liberty."""
+
+from basic_deployment import GlanceBasicDeployment
+
+if __name__ == '__main__':
+    deployment = GlanceBasicDeployment(series='trusty',
+                                         openstack='cloud:trusty-liberty',
+                                         source='cloud:trusty-updates/liberty')
+    deployment.run_tests()

tests/021-basic-xenial-mitaka

@@ -1,9 +1,9 @@
 #!/usr/bin/python
 
-"""Amulet tests on a basic Glance deployment on vivid-kilo."""
+"""Amulet tests on a basic Glance deployment on xenial-mitaka."""
 
 from basic_deployment import GlanceBasicDeployment
 
 if __name__ == '__main__':
-    deployment = GlanceBasicDeployment(series='vivid')
+    deployment = GlanceBasicDeployment(series='xenial')
     deployment.run_tests()

tests/README

@@ -1,53 +1,103 @@
-This directory provides Amulet tests that focus on verification of Glance
+This directory provides Amulet tests to verify basic deployment functionality
-deployments.
+from the perspective of this charm, its requirements and its features, as
+exercised in a subset of the full OpenStack deployment test bundle topology.
 
-test_* methods are called in lexical sort order.
+Reference:  lp:openstack-charm-testing for full test bundles.
 
-Test name convention to ensure desired test order:
+A single topology and configuration is defined and deployed, once for each of
+the defined Ubuntu:OpenStack release combos.  The ongoing goal is for this
+charm to always possess tests and combo definitions for all currently-supported
+release combinations of U:OS.
+
+test_* methods are called in lexical sort order, as with most runners.  However,
+each individual test method should be idempotent and expected to pass regardless
+of run order or Ubuntu:OpenStack combo.  When writing or modifying tests,
+ensure that every individual test is not dependent on another test_ method.
+
+Test naming convention, purely for code organization purposes:
     1xx service and endpoint checks
     2xx relation checks
     3xx config checks
     4xx functional checks
-    9xx restarts and other final checks
+    9xx restarts, config changes, actions and other final checks
 
-In order to run tests, you'll need charm-tools installed (in addition to
+In order to run tests, charm-tools and juju must be installed:
-juju, of course):
     sudo add-apt-repository ppa:juju/stable
     sudo apt-get update
-    sudo apt-get install charm-tools
+    sudo apt-get install charm-tools juju juju-deployer amulet
 
-If you use a web proxy server to access the web, you'll need to set the
+Alternatively, tests may be exercised with proposed or development versions
-AMULET_HTTP_PROXY environment variable to the http URL of the proxy server.
+of juju and related tools:
+
+    # juju proposed version
+    sudo add-apt-repository ppa:juju/proposed
+    sudo apt-get update
+    sudo apt-get install charm-tools juju juju-deployer
+
+    # juju development version
+    sudo add-apt-repository ppa:juju/devel
+    sudo apt-get update
+    sudo apt-get install charm-tools juju juju-deployer
+
+Some tests may need to download files. If a web proxy server is required in
+the environment, the AMULET_HTTP_PROXY environment variable must be set and
+passed into the juju test command.  This is unrelated to juju's http proxy
+settings or behavior.
 
 The following examples demonstrate different ways that tests can be executed.
 All examples are run from the charm's root directory.
 
-  * To run all tests (starting with 00-setup):
+  * To run all +x tests in the tests directory:
 
-      make test
+      bzr branch lp:charms/trusty/foo
+      cd foo
+      make functional_test
 
-  * To run a specific test module (or modules):
+  * To run the tests against a specific release combo as defined in tests/:
 
-      juju test -v -p AMULET_HTTP_PROXY 15-basic-trusty-icehouse
+      bzr branch lp:charms/trusty/foo
+      cd foo
+      juju test -v -p AMULET_HTTP_PROXY 015-basic-trusty-icehouse
 
-  * To run a specific test module (or modules), and keep the environment
+  * To run tests and keep the juju environment deployed after a failure:
-    deployed after a failure:
 
-      juju test --set-e -v -p AMULET_HTTP_PROXY 15-basic-trusty-icehouse
+      bzr branch lp:charms/trusty/foo
+      cd foo
+      juju test --set-e -v -p AMULET_HTTP_PROXY 015-basic-trusty-icehouse
 
   * To re-run a test module against an already deployed environment (one
     that was deployed by a previous call to 'juju test --set-e'):
 
-      ./tests/15-basic-trusty-icehouse
+      ./tests/015-basic-trusty-icehouse
+
+  * Even with --set-e, `juju test` will tear down the deployment when all
+    tests pass. The following work flow may be more effective when
+    iterating on test writing.
+
+      bzr branch lp:charms/trusty/foo
+      cd foo
+      ./tests/setup/00-setup
+      juju bootstrap
+      ./tests/015-basic-trusty-icehouse
+      # make some changes, run tests again
+      ./tests/015-basic-trusty-icehouse
+      # make some changes, run tests again
+      ./tests/015-basic-trusty-icehouse
+
+  * There may be test definitions in the tests/ dir which are not set +x
+    executable.  This is generally true for deprecated releases, or for
+    upcoming releases which are not yet validated and enabled.  To enable
+    and run these tests:
+      bzr branch lp:charms/trusty/foo
+      cd foo
+      ls tests
+      chmod +x tests/017-basic-trusty-kilo
+      ./tests/setup/00-setup
+      juju bootstrap
+      ./tests/017-basic-trusty-kilo
 
 
-For debugging and test development purposes, all code should be idempotent.
+Additional notes:
-In other words, the code should have the ability to be re-run without changing
-the results beyond the initial run.  This enables editing and re-running of a
-test module against an already deployed environment, as described above.
-
-
-Notes for additional test writing:
 
   * Use DEBUG to turn on debug logging, use ERROR otherwise.
       u = OpenStackAmuletUtils(ERROR)
@@ -61,13 +111,3 @@ Notes for additional test writing:
       export OS_AUTH_URL=${OS_AUTH_PROTOCOL:-http}://`juju-deployer -e trusty -f keystone`:5000/v2.0
       keystone user-list
       glance image-list
-
-  * Preserving the deployed environment:
-    Even with juju --set-e, amulet will tear down the juju environment
-    when all tests pass.  This force_fail 'test' can be used in basic_deployment.py
-    to simulate a failed test and keep the environment.
-
-    def test_zzzz_fake_fail(self):
-        '''Force a fake fail to keep juju environment after a successful test run'''
-        # Useful in test writing, when used with:   juju test --set-e
-        amulet.raise_status(amulet.FAIL, msg='using fake fail to keep juju environment')

tests/basic_deployment.py

@@ -47,16 +47,6 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment):
             {self.glance_sentry: self.SERVICES},
             expect_success=should_run)
 
-    def get_service_overrides(self, unit):
-        """
-        Return a dict mapping service names to a boolean indicating whether
-        an override file exists for that service.
-        """
-        init_contents = unit.directory_contents("/etc/init/")
-        return {
-            service: "{}.override".format(service) in init_contents["files"]
-            for service in self.SERVICES}
-
     def _add_services(self):
         """Add services
 
@@ -361,7 +351,7 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment):
             expected['keystone_authtoken'].update({
                 'auth_host': rel_ks_gl['auth_host'],
                 'auth_port': rel_ks_gl['auth_port'],
-                'auth_protocol':  rel_ks_gl['auth_protocol']
+                'auth_protocol': rel_ks_gl['auth_protocol']
             })
 
         return expected
@@ -571,11 +561,7 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment):
         assert u.wait_on_action(action_id), "Pause action failed."
 
         self._assert_services(should_run=False)
-        assert all(self.get_service_overrides(unit).itervalues()), \
-            "Not all override files were created."
 
         action_id = u.run_action(unit, "resume")
         assert u.wait_on_action(action_id), "Resume action failed"
-        assert not any(self.get_service_overrides(unit).itervalues()), \
-            "Not all override files were removed."
         self._assert_services(should_run=True)

tests/charmhelpers/contrib/amulet/deployment.py

@@ -51,7 +51,8 @@ class AmuletDeployment(object):
         if 'units' not in this_service:
             this_service['units'] = 1
 
-        self.d.add(this_service['name'], units=this_service['units'])
+        self.d.add(this_service['name'], units=this_service['units'],
+                   constraints=this_service.get('constraints'))
 
         for svc in other_services:
             if 'location' in svc:
@@ -64,7 +65,8 @@ class AmuletDeployment(object):
             if 'units' not in svc:
                 svc['units'] = 1
 
-            self.d.add(svc['name'], charm=branch_location, units=svc['units'])
+            self.d.add(svc['name'], charm=branch_location, units=svc['units'],
+                       constraints=svc.get('constraints'))
 
     def _add_relations(self, relations):
         """Add all of the relations for the services."""

tests/charmhelpers/contrib/amulet/utils.py

@@ -326,7 +326,7 @@ class AmuletUtils(object):
 
     def service_restarted_since(self, sentry_unit, mtime, service,
                                 pgrep_full=None, sleep_time=20,
-                                retry_count=2, retry_sleep_time=30):
+                                retry_count=30, retry_sleep_time=10):
         """Check if service was been started after a given time.
 
         Args:
@@ -334,8 +334,9 @@ class AmuletUtils(object):
           mtime (float): The epoch time to check against
           service (string): service name to look for in process table
           pgrep_full: [Deprecated] Use full command line search mode with pgrep
-          sleep_time (int): Seconds to sleep before looking for process
+          sleep_time (int): Initial sleep time (s) before looking for file
-          retry_count (int): If service is not found, how many times to retry
+          retry_sleep_time (int): Time (s) to sleep between retries
+          retry_count (int): If file is not found, how many times to retry
 
         Returns:
           bool: True if service found and its start time it newer than mtime,
@@ -359,11 +360,12 @@ class AmuletUtils(object):
                                                             pgrep_full)
                 self.log.debug('Attempt {} to get {} proc start time on {} '
                                'OK'.format(tries, service, unit_name))
-            except IOError:
+            except IOError as e:
                 # NOTE(beisner) - race avoidance, proc may not exist yet.
                 # https://bugs.launchpad.net/charm-helpers/+bug/1474030
                 self.log.debug('Attempt {} to get {} proc start time on {} '
-                               'failed'.format(tries, service, unit_name))
+                               'failed\n{}'.format(tries, service,
+                                                   unit_name, e))
                 time.sleep(retry_sleep_time)
                 tries += 1
 
@@ -383,35 +385,62 @@ class AmuletUtils(object):
             return False
 
     def config_updated_since(self, sentry_unit, filename, mtime,
-                             sleep_time=20):
+                             sleep_time=20, retry_count=30,
+                             retry_sleep_time=10):
         """Check if file was modified after a given time.
 
         Args:
           sentry_unit (sentry): The sentry unit to check the file mtime on
           filename (string): The file to check mtime of
           mtime (float): The epoch time to check against
-          sleep_time (int): Seconds to sleep before looking for process
+          sleep_time (int): Initial sleep time (s) before looking for file
+          retry_sleep_time (int): Time (s) to sleep between retries
+          retry_count (int): If file is not found, how many times to retry
 
         Returns:
           bool: True if file was modified more recently than mtime, False if
-                file was modified before mtime,
+                file was modified before mtime, or if file not found.
         """
-        self.log.debug('Checking %s updated since %s' % (filename, mtime))
+        unit_name = sentry_unit.info['unit_name']
+        self.log.debug('Checking that %s updated since %s on '
+                       '%s' % (filename, mtime, unit_name))
         time.sleep(sleep_time)
-        file_mtime = self._get_file_mtime(sentry_unit, filename)
+        file_mtime = None
+        tries = 0
+        while tries <= retry_count and not file_mtime:
+            try:
+                file_mtime = self._get_file_mtime(sentry_unit, filename)
+                self.log.debug('Attempt {} to get {} file mtime on {} '
+                               'OK'.format(tries, filename, unit_name))
+            except IOError as e:
+                # NOTE(beisner) - race avoidance, file may not exist yet.
+                # https://bugs.launchpad.net/charm-helpers/+bug/1474030
+                self.log.debug('Attempt {} to get {} file mtime on {} '
+                               'failed\n{}'.format(tries, filename,
+                                                   unit_name, e))
+                time.sleep(retry_sleep_time)
+                tries += 1
+
+        if not file_mtime:
+            self.log.warn('Could not determine file mtime, assuming '
+                          'file does not exist')
+            return False
+
         if file_mtime >= mtime:
             self.log.debug('File mtime is newer than provided mtime '
-                           '(%s >= %s)' % (file_mtime, mtime))
+                           '(%s >= %s) on %s (OK)' % (file_mtime,
+                                                      mtime, unit_name))
             return True
         else:
-            self.log.warn('File mtime %s is older than provided mtime %s'
+            self.log.warn('File mtime is older than provided mtime'
-                          % (file_mtime, mtime))
+                          '(%s < on %s) on %s' % (file_mtime,
+                                                  mtime, unit_name))
             return False
 
     def validate_service_config_changed(self, sentry_unit, mtime, service,
                                         filename, pgrep_full=None,
-                                        sleep_time=20, retry_count=2,
+                                        sleep_time=20, retry_count=30,
-                                        retry_sleep_time=30):
+                                        retry_sleep_time=10):
         """Check service and file were updated after mtime
 
         Args:
@@ -456,7 +485,9 @@ class AmuletUtils(object):
             sentry_unit,
             filename,
             mtime,
-            sleep_time=0)
+            sleep_time=sleep_time,
+            retry_count=retry_count,
+            retry_sleep_time=retry_sleep_time)
 
         return service_restart and config_update
 
@@ -776,3 +807,12 @@ class AmuletUtils(object):
         output = _check_output(command, universal_newlines=True)
         data = json.loads(output)
         return data.get(u"status") == "completed"
+
+    def status_get(self, unit):
+        """Return the current service status of this unit."""
+        raw_status, return_code = unit.run(
+            "status-get --format=json --include-data")
+        if return_code != 0:
+            return ("unknown", "")
+        status = json.loads(raw_status)
+        return (status["status"], status["message"])

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -14,12 +14,18 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
+import re
+import sys
 import six
 from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
 
+DEBUG = logging.DEBUG
+ERROR = logging.ERROR
+
 
 class OpenStackAmuletDeployment(AmuletDeployment):
     """OpenStack amulet deployment.
@@ -28,9 +34,12 @@ class OpenStackAmuletDeployment(AmuletDeployment):
        that is specifically for use by OpenStack charms.
        """
 
-    def __init__(self, series=None, openstack=None, source=None, stable=True):
+    def __init__(self, series=None, openstack=None, source=None,
+                 stable=True, log_level=DEBUG):
         """Initialize the deployment environment."""
         super(OpenStackAmuletDeployment, self).__init__(series)
+        self.log = self.get_logger(level=log_level)
+        self.log.info('OpenStackAmuletDeployment:  init')
         self.openstack = openstack
         self.source = source
         self.stable = stable
@@ -38,6 +47,22 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         # out.
         self.current_next = "trusty"
 
+    def get_logger(self, name="deployment-logger", level=logging.DEBUG):
+        """Get a logger object that will log to stdout."""
+        log = logging
+        logger = log.getLogger(name)
+        fmt = log.Formatter("%(asctime)s %(funcName)s "
+                            "%(levelname)s: %(message)s")
+
+        handler = log.StreamHandler(stream=sys.stdout)
+        handler.setLevel(level)
+        handler.setFormatter(fmt)
+
+        logger.addHandler(handler)
+        logger.setLevel(level)
+
+        return logger
+
     def _determine_branch_locations(self, other_services):
         """Determine the branch locations for the other services.
 
@@ -45,6 +70,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
 
+        self.log.info('OpenStackAmuletDeployment:  determine branch locations')
+
         # Charms outside the lp:~openstack-charmers namespace
         base_charms = ['mysql', 'mongodb', 'nrpe']
 
@@ -58,19 +85,17 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         else:
             base_series = self.current_next
 
-        if self.stable:
+        for svc in other_services:
-            for svc in other_services:
+            if svc['name'] in force_series_current:
-                if svc['name'] in force_series_current:
+                base_series = self.current_next
-                    base_series = self.current_next
+            # If a location has been explicitly set, use it
-
+            if svc.get('location'):
+                continue
+            if self.stable:
                 temp = 'lp:charms/{}/{}'
                 svc['location'] = temp.format(base_series,
                                               svc['name'])
-        else:
+            else:
-            for svc in other_services:
-                if svc['name'] in force_series_current:
-                    base_series = self.current_next
-
                 if svc['name'] in base_charms:
                     temp = 'lp:charms/{}/{}'
                     svc['location'] = temp.format(base_series,
@@ -79,10 +104,13 @@ class OpenStackAmuletDeployment(AmuletDeployment):
                     temp = 'lp:~openstack-charmers/charms/{}/{}/next'
                     svc['location'] = temp.format(self.current_next,
                                                   svc['name'])
+
         return other_services
 
     def _add_services(self, this_service, other_services):
         """Add services to the deployment and set openstack-origin/source."""
+        self.log.info('OpenStackAmuletDeployment:  adding services')
+
         other_services = self._determine_branch_locations(other_services)
 
         super(OpenStackAmuletDeployment, self)._add_services(this_service,
@@ -96,7 +124,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
                       'ceph-osd', 'ceph-radosgw']
 
         # Charms which can not use openstack-origin, ie. many subordinates
-        no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
+        no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe',
+                     'openvswitch-odl', 'neutron-api-odl', 'odl-controller']
 
         if self.openstack:
             for svc in services:
@@ -112,9 +141,79 @@ class OpenStackAmuletDeployment(AmuletDeployment):
 
     def _configure_services(self, configs):
         """Configure all of the services."""
+        self.log.info('OpenStackAmuletDeployment:  configure services')
         for service, config in six.iteritems(configs):
             self.d.configure(service, config)
 
+    def _auto_wait_for_status(self, message=None, exclude_services=None,
+                              include_only=None, timeout=1800):
+        """Wait for all units to have a specific extended status, except
+        for any defined as excluded.  Unless specified via message, any
+        status containing any case of 'ready' will be considered a match.
+
+        Examples of message usage:
+
+          Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
+              message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
+
+          Wait for all units to reach this status (exact match):
+              message = re.compile('^Unit is ready and clustered$')
+
+          Wait for all units to reach any one of these (exact match):
+              message = re.compile('Unit is ready|OK|Ready')
+
+          Wait for at least one unit to reach this status (exact match):
+              message = {'ready'}
+
+        See Amulet's sentry.wait_for_messages() for message usage detail.
+        https://github.com/juju/amulet/blob/master/amulet/sentry.py
+
+        :param message: Expected status match
+        :param exclude_services: List of juju service names to ignore,
+            not to be used in conjuction with include_only.
+        :param include_only: List of juju service names to exclusively check,
+            not to be used in conjuction with exclude_services.
+        :param timeout: Maximum time in seconds to wait for status match
+        :returns: None.  Raises if timeout is hit.
+        """
+        self.log.info('Waiting for extended status on units...')
+
+        all_services = self.d.services.keys()
+
+        if exclude_services and include_only:
+            raise ValueError('exclude_services can not be used '
+                             'with include_only')
+
+        if message:
+            if isinstance(message, re._pattern_type):
+                match = message.pattern
+            else:
+                match = message
+
+            self.log.debug('Custom extended status wait match: '
+                           '{}'.format(match))
+        else:
+            self.log.debug('Default extended status wait match:  contains '
+                           'READY (case-insensitive)')
+            message = re.compile('.*ready.*', re.IGNORECASE)
+
+        if exclude_services:
+            self.log.debug('Excluding services from extended status match: '
+                           '{}'.format(exclude_services))
+        else:
+            exclude_services = []
+
+        if include_only:
+            services = include_only
+        else:
+            services = list(set(all_services) - set(exclude_services))
+
+        self.log.debug('Waiting up to {}s for extended status on services: '
+                       '{}'.format(timeout, services))
+        service_messages = {service: message for service in services}
+        self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+        self.log.info('OK')
+
     def _get_openstack_release(self):
         """Get openstack release.
 
@@ -126,7 +225,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
          self.precise_havana, self.precise_icehouse,
          self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
          self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
-         self.wily_liberty) = range(12)
+         self.wily_liberty, self.trusty_mitaka,
+         self.xenial_mitaka) = range(14)
 
         releases = {
             ('precise', None): self.precise_essex,
@@ -138,9 +238,11 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
             ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
             ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
+            ('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka,
             ('utopic', None): self.utopic_juno,
             ('vivid', None): self.vivid_kilo,
-            ('wily', None): self.wily_liberty}
+            ('wily', None): self.wily_liberty,
+            ('xenial', None): self.xenial_mitaka}
         return releases[(self.series, self.openstack)]
 
     def _get_openstack_release_string(self):
@@ -157,6 +259,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('utopic', 'juno'),
             ('vivid', 'kilo'),
             ('wily', 'liberty'),
+            ('xenial', 'mitaka'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -18,6 +18,7 @@ import amulet
 import json
 import logging
 import os
+import re
 import six
 import time
 import urllib
@@ -604,7 +605,22 @@ class OpenStackAmuletUtils(AmuletUtils):
                            '{}'.format(sample_type, samples))
             return None
 
-# rabbitmq/amqp specific helpers:
+    # rabbitmq/amqp specific helpers:
+
+    def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
+        """Wait for rmq units extended status to show cluster readiness,
+        after an optional initial sleep period.  Initial sleep is likely
+        necessary to be effective following a config change, as status
+        message may not instantly update to non-ready."""
+
+        if init_sleep:
+            time.sleep(init_sleep)
+
+        message = re.compile('^Unit is ready and clustered$')
+        deployment._auto_wait_for_status(message=message,
+                                         timeout=timeout,
+                                         include_only=['rabbitmq-server'])
+
     def add_rmq_test_user(self, sentry_units,
                           username="testuser1", password="changeme"):
         """Add a test user via the first rmq juju unit, check connection as
@@ -752,7 +768,7 @@ class OpenStackAmuletUtils(AmuletUtils):
             self.log.debug('SSL is enabled  @{}:{} '
                            '({})'.format(host, port, unit_name))
             return True
-        elif not port and not conf_ssl:
+        elif not conf_ssl:
             self.log.debug('SSL not enabled @{}:{} '
                            '({})'.format(host, port, unit_name))
             return False
@@ -805,7 +821,10 @@ class OpenStackAmuletUtils(AmuletUtils):
         if port:
             config['ssl_port'] = port
 
-        deployment.configure('rabbitmq-server', config)
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
 
         # Confirm
         tries = 0
@@ -832,7 +851,10 @@ class OpenStackAmuletUtils(AmuletUtils):
 
         # Disable RMQ SSL
         config = {'ssl': 'off'}
-        deployment.configure('rabbitmq-server', config)
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
 
         # Confirm
         tries = 0

tests/setup/00-setup

@@ -4,11 +4,14 @@ set -ex
 
 sudo add-apt-repository --yes ppa:juju/stable
 sudo apt-get update --yes
-sudo apt-get install --yes python-amulet \
+sudo apt-get install --yes amulet \
+                           distro-info-data \
                            python-cinderclient \
                            python-distro-info \
                            python-glanceclient \
                            python-heatclient \
                            python-keystoneclient \
+                           python-neutronclient \
                            python-novaclient \
+                           python-pika \
                            python-swiftclient

tests/tests.yaml

@@ -1,5 +1,5 @@
 bootstrap: true
-reset: true
+reset: false
 virtualenv: true
 makefile:
     - lint
@@ -8,11 +8,14 @@ sources:
     - ppa:juju/stable
 packages:
     - amulet
-    - python-amulet
+    - distro-info-data
+    - python-ceilometerclient
     - python-cinderclient
     - python-distro-info
     - python-glanceclient
     - python-heatclient
     - python-keystoneclient
+    - python-neutronclient
     - python-novaclient
+    - python-pika
     - python-swiftclient

tox.ini

@@ -0,0 +1,35 @@
+[tox]
+# Default to current LTS
+envlist = lint,py27-trusty
+skipsdist = True
+
+[testenv]
+setenv = VIRTUAL_ENV={envdir}
+         PYTHONHASHSEED=0
+install_command =
+  pip install --allow-unverified python-apt {opts} {packages}
+commands = ostestr {posargs}
+
+[testenv:py27-precise]
+basepython = python2.7
+deps = -r{toxinidir}/requirements/requirements-precise.txt
+       -r{toxinidir}/requirements/test-requirements.txt
+
+[testenv:py27-trusty]
+basepython = python2.7
+deps = -r{toxinidir}/requirements/requirements-trusty.txt
+       -r{toxinidir}/requirements/test-requirements.txt
+
+[testenv:lint]
+basepython = python2.7
+deps = -r{toxinidir}/requirements/requirements-trusty.txt
+       -r{toxinidir}/requirements/test-requirements.txt
+commands = flake8 {posargs} actions hooks unit_tests tests
+           charm proof
+
+[testenv:venv]
+commands = {posargs}
+
+[flake8]
+ignore = E402,E226
+exclude = hooks/charmhelpers

unit_tests/test_actions_git_reinstall.py

@@ -13,6 +13,7 @@ from test_utils import (
 
 TO_PATCH = [
     'config',
+    'git_install_requested',
 ]
 
 
@@ -31,6 +32,7 @@ class TestGlanceActions(CharmTestCase):
     def setUp(self):
         super(TestGlanceActions, self).setUp(git_reinstall, TO_PATCH)
         self.config.side_effect = self.test_config.get
+        self.git_install_requested.return_value = True
 
     @patch.object(git_reinstall, 'action_set')
     @patch.object(git_reinstall, 'action_fail')
@@ -56,6 +58,7 @@ class TestGlanceActions(CharmTestCase):
     def test_git_reinstall_not_configured(self, config_changed, git_install,
                                           action_fail, action_set):
         config.return_value = None
+        self.git_install_requested.return_value = False
 
         git_reinstall.git_reinstall()
 

unit_tests/test_actions_openstack_upgrade.py

@@ -5,14 +5,15 @@ os.environ['JUJU_UNIT_NAME'] = 'glance'
 
 with patch('actions.hooks.glance_utils.register_configs'):
     with patch('hooks.glance_utils.register_configs'):
-        from actions import openstack_upgrade
+            from actions import openstack_upgrade
 
 from test_utils import (
     CharmTestCase
 )
 
 TO_PATCH = [
-    'config'
+    'config_changed',
+    'do_openstack_upgrade'
 ]
 
 
@@ -21,97 +22,39 @@ class TestGlanceUpgradeActions(CharmTestCase):
     def setUp(self):
         super(TestGlanceUpgradeActions, self).setUp(openstack_upgrade,
                                                     TO_PATCH)
-        self.config.side_effect = self.test_config.get
 
-    @patch.object(openstack_upgrade, 'action_set')
+    @patch('actions.charmhelpers.contrib.openstack.utils.config')
-    @patch.object(openstack_upgrade, 'action_fail')
+    @patch('actions.charmhelpers.contrib.openstack.utils.action_set')
-    @patch.object(openstack_upgrade, 'do_openstack_upgrade')
+    @patch('actions.charmhelpers.contrib.openstack.utils.git_install_requested')  # noqa
-    @patch.object(openstack_upgrade, 'openstack_upgrade_available')
+    @patch('actions.charmhelpers.contrib.openstack.utils.openstack_upgrade_available')  # noqa
-    @patch.object(openstack_upgrade, 'config_changed')
+    @patch('actions.charmhelpers.contrib.openstack.utils.juju_log')
     @patch('subprocess.check_output')
-    def test_openstack_upgrade(self, _check_output, config_changed,
+    def test_openstack_upgrade_true(self, _check_output, log, upgrade_avail,
-                               openstack_upgrade_available,
+                                    git_requested, action_set, config):
-                               do_openstack_upgrade, action_fail,
-                               action_set):
         _check_output.return_value = 'null'
-        openstack_upgrade_available.return_value = True
+        git_requested.return_value = False
-
+        upgrade_avail.return_value = True
-        self.test_config.set('action-managed-upgrade', True)
+        config.return_value = True
 
         openstack_upgrade.openstack_upgrade()
 
-        self.assertTrue(do_openstack_upgrade.called)
+        self.assertTrue(self.do_openstack_upgrade.called)
-        self.assertTrue(config_changed.called)
+        self.assertTrue(self.config_changed.called)
-        self.assertFalse(action_fail.called)
 
-    @patch.object(openstack_upgrade, 'action_set')
+    @patch('actions.charmhelpers.contrib.openstack.utils.config')
-    @patch.object(openstack_upgrade, 'do_openstack_upgrade')
+    @patch('actions.charmhelpers.contrib.openstack.utils.action_set')
-    @patch.object(openstack_upgrade, 'openstack_upgrade_available')
+    @patch('actions.charmhelpers.contrib.openstack.utils.git_install_requested')  # noqa
-    @patch.object(openstack_upgrade, 'config_changed')
+    @patch('actions.charmhelpers.contrib.openstack.utils.openstack_upgrade_available')  # noqa
+    @patch('actions.charmhelpers.contrib.openstack.utils.juju_log')
     @patch('subprocess.check_output')
-    def test_openstack_upgrade_not_configured(self, _check_output,
+    def test_openstack_upgrade_false(self, _check_output, log, upgrade_avail,
-                                              config_changed,
+                                     git_requested, action_set, config):
-                                              openstack_upgrade_available,
-                                              do_openstack_upgrade,
-                                              action_set):
         _check_output.return_value = 'null'
-        openstack_upgrade_available.return_value = True
+        git_requested.return_value = False
+        upgrade_avail.return_value = True
+        config.return_value = False
 
         openstack_upgrade.openstack_upgrade()
 
-        msg = ('action-managed-upgrade config is False, skipped upgrade.')
+        self.assertFalse(self.do_openstack_upgrade.called)
-
+        self.assertFalse(self.config_changed.called)
-        action_set.assert_called_with({'outcome': msg})
-        self.assertFalse(do_openstack_upgrade.called)
-
-    @patch.object(openstack_upgrade, 'action_set')
-    @patch.object(openstack_upgrade, 'do_openstack_upgrade')
-    @patch.object(openstack_upgrade, 'openstack_upgrade_available')
-    @patch.object(openstack_upgrade, 'config_changed')
-    @patch.object(openstack_upgrade, 'git_install_requested')
-    def test_openstack_upgrade_git_install(self, git_install_requested,
-                                           config_changed,
-                                           openstack_upgrade_available,
-                                           do_openstack_upgrade, action_set):
-        git_install_requested.return_value = True
-
-        openstack_upgrade.openstack_upgrade()
-
-        msg = ('installed from source, skipped upgrade.')
-        action_set.assert_called_with({'outcome': msg})
-        self.assertFalse(do_openstack_upgrade.called)
-
-    @patch.object(openstack_upgrade, 'action_set')
-    @patch.object(openstack_upgrade, 'action_fail')
-    @patch.object(openstack_upgrade, 'do_openstack_upgrade')
-    @patch.object(openstack_upgrade, 'openstack_upgrade_available')
-    @patch.object(openstack_upgrade, 'config_changed')
-    @patch('traceback.format_exc')
-    @patch('charmhelpers.core.hookenv.config')
-    def test_openstack_upgrade_exception(self, _config, format_exc,
-                                         config_changed,
-                                         openstack_upgrade_available,
-                                         do_openstack_upgrade,
-                                         action_fail, action_set):
-        _config.return_value = None
-        self.test_config.set('action-managed-upgrade', True)
-        openstack_upgrade_available.return_value = True
-
-        e = OSError('something bad happened')
-        do_openstack_upgrade.side_effect = e
-        traceback = (
-            "Traceback (most recent call last):\n"
-            "  File \"actions/openstack_upgrade.py\", line 37, in openstack_upgrade\n"  # noqa
-            "    openstack_upgrade(config(\'openstack-origin-git\'))\n"
-            "  File \"/usr/lib/python2.7/dist-packages/mock.py\", line 964, in __call__\n"  # noqa
-            "    return _mock_self._mock_call(*args, **kwargs)\n"
-            "  File \"/usr/lib/python2.7/dist-packages/mock.py\", line 1019, in _mock_call\n"  # noqa
-            "    raise effect\n"
-            "OSError: something bad happened\n")
-        format_exc.return_value = traceback
-
-        openstack_upgrade.openstack_upgrade()
-
-        msg = 'do_openstack_upgrade resulted in an unexpected error'
-        action_fail.assert_called_with(msg)
-        action_set.assert_called_with({'traceback': traceback})

unit_tests/test_glance_contexts.py

@@ -42,10 +42,13 @@ class TestGlanceContexts(CharmTestCase):
         self.is_relation_made.return_value = True
         service = 'glance'
         self.service_name.return_value = service
+        self.config.return_value = True
         self.assertEquals(
             contexts.CephGlanceContext()(),
             {'rbd_pool': service,
-             'rbd_user': service})
+             'rbd_user': service,
+             'expose_image_locations': True})
+        self.config.assert_called_with('expose-image-locations')
 
     def test_multistore(self):
         self.relation_ids.return_value = ['random_rid']
@@ -81,17 +84,23 @@ class TestGlanceContexts(CharmTestCase):
                                    'ext_ports': [9282],
                                    'namespace': 'glance'})
 
+    @patch('charmhelpers.contrib.openstack.context.config')
     @patch("subprocess.check_output")
-    def test_glance_ipv6_context_service_enabled(self, mock_subprocess):
+    def test_glance_ipv6_context_service_enabled(self, mock_subprocess,
+                                                 mock_config):
         self.config.return_value = True
+        mock_config.return_value = True
         mock_subprocess.return_value = 'true'
         ctxt = contexts.GlanceIPv6Context()
         self.assertEquals(ctxt(), {'bind_host': '::',
                                    'registry_host': '[::]'})
 
+    @patch('charmhelpers.contrib.openstack.context.config')
     @patch("subprocess.check_output")
-    def test_glance_ipv6_context_service_disabled(self, mock_subprocess):
+    def test_glance_ipv6_context_service_disabled(self, mock_subprocess,
+                                                  mock_config):
         self.config.return_value = False
+        mock_config.return_value = False
         mock_subprocess.return_value = 'false'
         ctxt = contexts.GlanceIPv6Context()
         self.assertEquals(ctxt(), {'bind_host': '0.0.0.0',

unit_tests/test_glance_relations.py

@@ -143,13 +143,8 @@ class GlanceRelationTests(CharmTestCase):
                                              hostname='glance.foohost.com')
         self.unit_get.assert_called_with('private-address')
 
-    @patch.object(relations, 'sync_db_with_multi_ipv6_addresses')
+    def test_db_joined_with_ipv6(self):
-    @patch.object(relations, 'get_ipv6_addr')
-    def test_db_joined_with_ipv6(self, mock_get_ipv6_addr,
-                                 mock_sync_db):
         self.test_config.set('prefer-ipv6', True)
-        mock_get_ipv6_addr.return_value = ['2001:db8:1::1']
-        mock_sync_db.return_value = MagicMock()
         self.is_relation_made.return_value = False
         relations.db_joined()
         relation_data = {
@@ -158,9 +153,8 @@ class GlanceRelationTests(CharmTestCase):
         }
         relation_data['hostname'] = '2001:db8:1::1'
 
-        self.sync_db_with_multi_ipv6_addresses.assert_called_with_once(
+        self.sync_db_with_multi_ipv6_addresses.assert_called_with(
             'glance', 'glance')
-        self.get_ipv6_addr.assert_called_once()
 
     def test_postgresql_db_joined(self):
         self.unit_get.return_value = 'glance.foohost.com'
@@ -540,14 +534,17 @@ class GlanceRelationTests(CharmTestCase):
         self.open_port.assert_called_with(9292)
         self.assertTrue(configure_https.called)
 
+    @patch.object(relations, 'status_set')
     @patch.object(relations, 'configure_https')
     @patch.object(relations, 'git_install_requested')
     def test_config_changed_with_openstack_upgrade(self, git_requested,
-                                                   configure_https):
+                                                   configure_https,
+                                                   status):
         git_requested.return_value = False
         self.openstack_upgrade_available.return_value = True
         relations.config_changed()
-        self.juju_log.assert_called_with(
+        status.assert_called_with(
+            'maintenance',
             'Upgrading OpenStack release'
         )
         self.assertTrue(self.do_openstack_upgrade.called)
@@ -724,9 +721,10 @@ class GlanceRelationTests(CharmTestCase):
         configs.write = MagicMock()
         self.relation_ids.return_value = ['identity-service:0']
         relations.configure_https()
-        calls = [call('a2dissite', 'openstack_https_frontend'),
+        self.check_call.assert_called_with(['a2ensite',
-                 call('service', 'apache2', 'reload')]
+                                            'openstack_https_frontend'])
-        self.check_call.assert_called_has_calls(calls)
+        self.service_reload.assert_called_with('apache2',
+                                               restart_on_failure=True)
         keystone_joined.assert_called_with(relation_id='identity-service:0')
 
     @patch.object(relations, 'canonical_url')
@@ -739,9 +737,10 @@ class GlanceRelationTests(CharmTestCase):
         configs.write = MagicMock()
         self.relation_ids.return_value = ['identity-service:0']
         relations.configure_https()
-        calls = [call('a2dissite', 'openstack_https_frontend'),
+        self.check_call.assert_called_with(['a2dissite',
-                 call('service', 'apache2', 'reload')]
+                                            'openstack_https_frontend'])
-        self.check_call.assert_called_has_calls(calls)
+        self.service_reload.assert_called_with('apache2',
+                                               restart_on_failure=True)
         keystone_joined.assert_called_with(relation_id='identity-service:0')
 
     @patch.object(relations, 'canonical_url')
@@ -754,9 +753,10 @@ class GlanceRelationTests(CharmTestCase):
         configs.write = MagicMock()
         self.relation_ids.return_value = ['image-service:0']
         relations.configure_https()
-        calls = [call('a2dissite', 'openstack_https_frontend'),
+        self.check_call.assert_called_with(['a2ensite',
-                 call('service', 'apache2', 'reload')]
+                                            'openstack_https_frontend'])
-        self.check_call.assert_called_has_calls(calls)
+        self.service_reload.assert_called_with('apache2',
+                                               restart_on_failure=True)
         image_service_joined.assert_called_with(relation_id='image-service:0')
 
     @patch.object(relations, 'canonical_url')
@@ -769,9 +769,10 @@ class GlanceRelationTests(CharmTestCase):
         configs.write = MagicMock()
         self.relation_ids.return_value = ['image-service:0']
         relations.configure_https()
-        calls = [call('a2dissite', 'openstack_https_frontend'),
+        self.check_call.assert_called_with(['a2dissite',
-                 call('service', 'apache2', 'reload')]
+                                            'openstack_https_frontend'])
-        self.check_call.assert_called_has_calls(calls)
+        self.service_reload.assert_called_with('apache2',
+                                               restart_on_failure=True)
         image_service_joined.assert_called_with(relation_id='image-service:0')
 
     def test_amqp_joined(self):
@@ -785,7 +786,7 @@ class GlanceRelationTests(CharmTestCase):
         configs.complete_contexts = MagicMock()
         configs.complete_contexts.return_value = []
         relations.amqp_changed()
-        self.juju_log.assert_called()
+        self.assertTrue(self.juju_log.called)
 
     @patch.object(relations, 'CONFIGS')
     def test_amqp_changed_relation_data(self, configs):

unit_tests/test_glance_utils.py

@@ -5,9 +5,7 @@ import os
 
 os.environ['JUJU_UNIT_NAME'] = 'glance'
 
-with patch('charmhelpers.core.hookenv.config') as config:
+import hooks.glance_utils as utils
-    import hooks.glance_utils as utils
-
 from test_utils import (
     CharmTestCase,
 )

unit_tests/test_utils.py

@@ -6,6 +6,10 @@ import yaml
 from contextlib import contextmanager
 from mock import patch, MagicMock
 
+patch('hooks.charmhelpers.contrib.openstack.utils.'
+      'set_os_workload_status').start()
+patch('hooks.charmhelpers.core.hookenv.status_set').start()
+
 
 def load_config():
     '''Walk backwords from __file__ looking for config.yaml,