Swift storage ACLs

Ensure that only the swift-proxy units and swift-storage peers have
access to direct communication with swift storage daemons.

Charm-helpers sync to include ufw module and the ingress_address and
iter_units_for_relation_name functions.

Please review and merge first:
https://github.com/juju/charm-helpers/pull/35

Closes-Bug: #1727463

Change-Id: Id5677edbc40b0b891cbe66867d39d076a94c5436

charm-helpers-hooks.yaml

@@ -12,6 +12,7 @@ include:
         - cluster
     - payload.execd
     - contrib.network.ip
+    - contrib.network.ufw
     - contrib.python.packages
     - contrib.charmsupport
     - contrib.hardening|inc=*

charmhelpers/contrib/charmsupport/nrpe.py

@@ -285,7 +285,7 @@ class NRPE(object):
         try:
             nagios_uid = pwd.getpwnam('nagios').pw_uid
             nagios_gid = grp.getgrnam('nagios').gr_gid
-        except:
+        except Exception:
             log("Nagios user not set up, nrpe checks not updated")
             return
 

charmhelpers/contrib/hahelpers/cluster.py

@@ -27,6 +27,7 @@ clustering-related helpers.
 
 import subprocess
 import os
+import time
 
 from socket import gethostname as get_unit_hostname
 
@@ -45,6 +46,9 @@ from charmhelpers.core.hookenv import (
     is_leader as juju_is_leader,
     status_set,
 )
+from charmhelpers.core.host import (
+    modulo_distribution,
+)
 from charmhelpers.core.decorators import (
     retry_on_exception,
 )
@@ -361,3 +365,29 @@ def canonical_url(configs, vip_setting='vip'):
     else:
         addr = unit_get('private-address')
     return '%s://%s' % (scheme, addr)
+
+
+def distributed_wait(modulo=None, wait=None, operation_name='operation'):
+    ''' Distribute operations by waiting based on modulo_distribution
+
+    If modulo and or wait are not set, check config_get for those values.
+
+    :param modulo: int The modulo number creates the group distribution
+    :param wait: int The constant time wait value
+    :param operation_name: string Operation name for status message
+                           i.e.  'restart'
+    :side effect: Calls config_get()
+    :side effect: Calls log()
+    :side effect: Calls status_set()
+    :side effect: Calls time.sleep()
+    '''
+    if modulo is None:
+        modulo = config_get('modulo-nodes')
+    if wait is None:
+        wait = config_get('known-wait')
+    calculated_wait = modulo_distribution(modulo=modulo, wait=wait)
+    msg = "Waiting {} seconds for {} ...".format(calculated_wait,
+                                                 operation_name)
+    log(msg, DEBUG)
+    status_set('maintenance', msg)
+    time.sleep(calculated_wait)

charmhelpers/contrib/hardening/audits/apache.py

@@ -70,12 +70,12 @@ class DisabledModuleAudit(BaseAudit):
         """Returns the modules which are enabled in Apache."""
         output = subprocess.check_output(['apache2ctl', '-M'])
         modules = []
-        for line in output.strip().split():
+        for line in output.splitlines():
             # Each line of the enabled module output looks like:
             #  module_name (static|shared)
             # Plus a header line at the top of the output which is stripped
             # out by the regex.
-            matcher = re.search(r'^ (\S*)', line)
+            matcher = re.search(r'^ (\S*)_module (\S*)', line)
             if matcher:
                 modules.append(matcher.group(1))
         return modules

charmhelpers/contrib/network/ip.py

@@ -490,7 +490,7 @@ def get_host_ip(hostname, fallback=None):
     if not ip_addr:
         try:
             ip_addr = socket.gethostbyname(hostname)
-        except:
+        except Exception:
             log("Failed to resolve hostname '%s'" % (hostname),
                 level=WARNING)
             return fallback
@@ -518,7 +518,7 @@ def get_hostname(address, fqdn=True):
         if not result:
             try:
                 result = socket.gethostbyaddr(address)[0]
-            except:
+            except Exception:
                 return None
     else:
         result = address

charmhelpers/contrib/network/ufw.py

@@ -0,0 +1,316 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains helpers to add and remove ufw rules.
+
+Examples:
+
+- open SSH port for subnet 10.0.3.0/24:
+
+  >>> from charmhelpers.contrib.network import ufw
+  >>> ufw.enable()
+  >>> ufw.grant_access(src='10.0.3.0/24', dst='any', port='22', proto='tcp')
+
+- open service by name as defined in /etc/services:
+
+  >>> from charmhelpers.contrib.network import ufw
+  >>> ufw.enable()
+  >>> ufw.service('ssh', 'open')
+
+- close service by port number:
+
+  >>> from charmhelpers.contrib.network import ufw
+  >>> ufw.enable()
+  >>> ufw.service('4949', 'close')  # munin
+"""
+import re
+import os
+import subprocess
+
+from charmhelpers.core import hookenv
+from charmhelpers.core.kernel import modprobe, is_module_loaded
+
+__author__ = "Felipe Reyes <felipe.reyes@canonical.com>"
+
+
+class UFWError(Exception):
+    pass
+
+
+class UFWIPv6Error(UFWError):
+    pass
+
+
+def is_enabled():
+    """
+    Check if `ufw` is enabled
+
+    :returns: True if ufw is enabled
+    """
+    output = subprocess.check_output(['ufw', 'status'],
+                                     universal_newlines=True,
+                                     env={'LANG': 'en_US',
+                                          'PATH': os.environ['PATH']})
+
+    m = re.findall(r'^Status: active\n', output, re.M)
+
+    return len(m) >= 1
+
+
+def is_ipv6_ok(soft_fail=False):
+    """
+    Check if IPv6 support is present and ip6tables functional
+
+    :param soft_fail: If set to True and IPv6 support is broken, then reports
+                      that the host doesn't have IPv6 support, otherwise a
+                      UFWIPv6Error exception is raised.
+    :returns: True if IPv6 is working, False otherwise
+    """
+
+    # do we have IPv6 in the machine?
+    if os.path.isdir('/proc/sys/net/ipv6'):
+        # is ip6tables kernel module loaded?
+        if not is_module_loaded('ip6_tables'):
+            # ip6tables support isn't complete, let's try to load it
+            try:
+                modprobe('ip6_tables')
+                # great, we can load the module
+                return True
+            except subprocess.CalledProcessError as ex:
+                hookenv.log("Couldn't load ip6_tables module: %s" % ex.output,
+                            level="WARN")
+                # we are in a world where ip6tables isn't working
+                if soft_fail:
+                    # so we inform that the machine doesn't have IPv6
+                    return False
+                else:
+                    raise UFWIPv6Error("IPv6 firewall support broken")
+        else:
+            # the module is present :)
+            return True
+
+    else:
+        # the system doesn't have IPv6
+        return False
+
+
+def disable_ipv6():
+    """
+    Disable ufw IPv6 support in /etc/default/ufw
+    """
+    exit_code = subprocess.call(['sed', '-i', 's/IPV6=.*/IPV6=no/g',
+                                 '/etc/default/ufw'])
+    if exit_code == 0:
+        hookenv.log('IPv6 support in ufw disabled', level='INFO')
+    else:
+        hookenv.log("Couldn't disable IPv6 support in ufw", level="ERROR")
+        raise UFWError("Couldn't disable IPv6 support in ufw")
+
+
+def enable(soft_fail=False):
+    """
+    Enable ufw
+
+    :param soft_fail: If set to True silently disables IPv6 support in ufw,
+                      otherwise a UFWIPv6Error exception is raised when IP6
+                      support is broken.
+    :returns: True if ufw is successfully enabled
+    """
+    if is_enabled():
+        return True
+
+    if not is_ipv6_ok(soft_fail):
+        disable_ipv6()
+
+    output = subprocess.check_output(['ufw', 'enable'],
+                                     universal_newlines=True,
+                                     env={'LANG': 'en_US',
+                                          'PATH': os.environ['PATH']})
+
+    m = re.findall('^Firewall is active and enabled on system startup\n',
+                   output, re.M)
+    hookenv.log(output, level='DEBUG')
+
+    if len(m) == 0:
+        hookenv.log("ufw couldn't be enabled", level='WARN')
+        return False
+    else:
+        hookenv.log("ufw enabled", level='INFO')
+        return True
+
+
+def disable():
+    """
+    Disable ufw
+
+    :returns: True if ufw is successfully disabled
+    """
+    if not is_enabled():
+        return True
+
+    output = subprocess.check_output(['ufw', 'disable'],
+                                     universal_newlines=True,
+                                     env={'LANG': 'en_US',
+                                          'PATH': os.environ['PATH']})
+
+    m = re.findall(r'^Firewall stopped and disabled on system startup\n',
+                   output, re.M)
+    hookenv.log(output, level='DEBUG')
+
+    if len(m) == 0:
+        hookenv.log("ufw couldn't be disabled", level='WARN')
+        return False
+    else:
+        hookenv.log("ufw disabled", level='INFO')
+        return True
+
+
+def default_policy(policy='deny', direction='incoming'):
+    """
+    Changes the default policy for traffic `direction`
+
+    :param policy: allow, deny or reject
+    :param direction: traffic direction, possible values: incoming, outgoing,
+                      routed
+    """
+    if policy not in ['allow', 'deny', 'reject']:
+        raise UFWError(('Unknown policy %s, valid values: '
+                        'allow, deny, reject') % policy)
+
+    if direction not in ['incoming', 'outgoing', 'routed']:
+        raise UFWError(('Unknown direction %s, valid values: '
+                        'incoming, outgoing, routed') % direction)
+
+    output = subprocess.check_output(['ufw', 'default', policy, direction],
+                                     universal_newlines=True,
+                                     env={'LANG': 'en_US',
+                                          'PATH': os.environ['PATH']})
+    hookenv.log(output, level='DEBUG')
+
+    m = re.findall("^Default %s policy changed to '%s'\n" % (direction,
+                                                             policy),
+                   output, re.M)
+    if len(m) == 0:
+        hookenv.log("ufw couldn't change the default policy to %s for %s"
+                    % (policy, direction), level='WARN')
+        return False
+    else:
+        hookenv.log("ufw default policy for %s changed to %s"
+                    % (direction, policy), level='INFO')
+        return True
+
+
+def modify_access(src, dst='any', port=None, proto=None, action='allow',
+                  index=None):
+    """
+    Grant access to an address or subnet
+
+    :param src: address (e.g. 192.168.1.234) or subnet
+                (e.g. 192.168.1.0/24).
+    :param dst: destiny of the connection, if the machine has multiple IPs and
+                connections to only one of those have to accepted this is the
+                field has to be set.
+    :param port: destiny port
+    :param proto: protocol (tcp or udp)
+    :param action: `allow` or `delete`
+    :param index: if different from None the rule is inserted at the given
+                  `index`.
+    """
+    if not is_enabled():
+        hookenv.log('ufw is disabled, skipping modify_access()', level='WARN')
+        return
+
+    if action == 'delete':
+        cmd = ['ufw', 'delete', 'allow']
+    elif index is not None:
+        cmd = ['ufw', 'insert', str(index), action]
+    else:
+        cmd = ['ufw', action]
+
+    if src is not None:
+        cmd += ['from', src]
+
+    if dst is not None:
+        cmd += ['to', dst]
+
+    if port is not None:
+        cmd += ['port', str(port)]
+
+    if proto is not None:
+        cmd += ['proto', proto]
+
+    hookenv.log('ufw {}: {}'.format(action, ' '.join(cmd)), level='DEBUG')
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+    (stdout, stderr) = p.communicate()
+
+    hookenv.log(stdout, level='INFO')
+
+    if p.returncode != 0:
+        hookenv.log(stderr, level='ERROR')
+        hookenv.log('Error running: {}, exit code: {}'.format(' '.join(cmd),
+                                                              p.returncode),
+                    level='ERROR')
+
+
+def grant_access(src, dst='any', port=None, proto=None, index=None):
+    """
+    Grant access to an address or subnet
+
+    :param src: address (e.g. 192.168.1.234) or subnet
+                (e.g. 192.168.1.0/24).
+    :param dst: destiny of the connection, if the machine has multiple IPs and
+                connections to only one of those have to accepted this is the
+                field has to be set.
+    :param port: destiny port
+    :param proto: protocol (tcp or udp)
+    :param index: if different from None the rule is inserted at the given
+                  `index`.
+    """
+    return modify_access(src, dst=dst, port=port, proto=proto, action='allow',
+                         index=index)
+
+
+def revoke_access(src, dst='any', port=None, proto=None):
+    """
+    Revoke access to an address or subnet
+
+    :param src: address (e.g. 192.168.1.234) or subnet
+                (e.g. 192.168.1.0/24).
+    :param dst: destiny of the connection, if the machine has multiple IPs and
+                connections to only one of those have to accepted this is the
+                field has to be set.
+    :param port: destiny port
+    :param proto: protocol (tcp or udp)
+    """
+    return modify_access(src, dst=dst, port=port, proto=proto, action='delete')
+
+
+def service(name, action):
+    """
+    Open/close access to a service
+
+    :param name: could be a service name defined in `/etc/services` or a port
+                 number.
+    :param action: `open` or `close`
+    """
+    if action == 'open':
+        subprocess.check_output(['ufw', 'allow', str(name)],
+                                universal_newlines=True)
+    elif action == 'close':
+        subprocess.check_output(['ufw', 'delete', 'allow', str(name)],
+                                universal_newlines=True)
+    else:
+        raise UFWError(("'{}' not supported, use 'allow' "
+                        "or 'delete'").format(action))

charmhelpers/contrib/openstack/alternatives.py

@@ -29,3 +29,16 @@ def install_alternative(name, target, source, priority=50):
         target, name, source, str(priority)
     ]
     subprocess.check_call(cmd)
+
+
+def remove_alternative(name, source):
+    """Remove an installed alternative configuration file
+
+    :param name: string name of the alternative to remove
+    :param source: string full path to alternative to remove
+    """
+    cmd = [
+        'update-alternatives', '--remove',
+        name, source
+    ]
+    subprocess.check_call(cmd)

charmhelpers/contrib/openstack/amulet/deployment.py

@@ -307,7 +307,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             # Kilo or later
             pools = [
                 'rbd',
-                'cinder',
+                'cinder-ceph',
                 'glance'
             ]
         else:
@@ -316,7 +316,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
                 'data',
                 'metadata',
                 'rbd',
-                'cinder',
+                'cinder-ceph',
                 'glance'
             ]
 

charmhelpers/contrib/openstack/amulet/utils.py

@@ -617,7 +617,7 @@ class OpenStackAmuletUtils(AmuletUtils):
             self.log.debug('Keypair ({}) already exists, '
                            'using it.'.format(keypair_name))
             return _keypair
-        except:
+        except Exception:
             self.log.debug('Keypair ({}) does not exist, '
                            'creating it.'.format(keypair_name))
 

charmhelpers/contrib/openstack/context.py

@@ -628,6 +628,8 @@ class HAProxyContext(OSContextGenerator):
             ctxt['local_host'] = '127.0.0.1'
             ctxt['haproxy_host'] = '0.0.0.0'
 
+        ctxt['ipv6_enabled'] = not is_ipv6_disabled()
+
         ctxt['stat_port'] = '8888'
 
         db = kv()
@@ -802,8 +804,9 @@ class ApacheSSLContext(OSContextGenerator):
         else:
             # Expect cert/key provided in config (currently assumed that ca
             # uses ip for cn)
-            cn = resolve_address(endpoint_type=INTERNAL)
-            self.configure_cert(cn)
+            for net_type in (INTERNAL, ADMIN, PUBLIC):
+                cn = resolve_address(endpoint_type=net_type)
+                self.configure_cert(cn)
 
         addresses = self.get_network_addresses()
         for address, endpoint in addresses:
@@ -1176,7 +1179,7 @@ class SubordinateConfigContext(OSContextGenerator):
                 if sub_config and sub_config != '':
                     try:
                         sub_config = json.loads(sub_config)
-                    except:
+                    except Exception:
                         log('Could not parse JSON from '
                             'subordinate_configuration setting from %s'
                             % rid, level=ERROR)

charmhelpers/contrib/openstack/files/check_haproxy.sh

@@ -9,7 +9,7 @@
 CRITICAL=0
 NOTACTIVE=''
 LOGFILE=/var/log/nagios/check_haproxy.log
-AUTH=$(grep -r "stats auth" /etc/haproxy | awk 'NR=1{print $4}')
+AUTH=$(grep -r "stats auth" /etc/haproxy/haproxy.cfg | awk 'NR=1{print $4}')
 
 typeset -i N_INSTANCES=0
 for appserver in $(awk '/^\s+server/{print $2}' /etc/haproxy/haproxy.cfg)

charmhelpers/contrib/openstack/ha/utils.py

@@ -82,15 +82,18 @@ def update_dns_ha_resource_params(resources, resource_params,
             continue
         m = re.search('os-(.+?)-hostname', setting)
         if m:
-            networkspace = m.group(1)
+            endpoint_type = m.group(1)
+            # resolve_address's ADDRESS_MAP uses 'int' not 'internal'
+            if endpoint_type == 'internal':
+                endpoint_type = 'int'
         else:
             msg = ('Unexpected DNS hostname setting: {}. '
-                   'Cannot determine network space name'
+                   'Cannot determine endpoint_type name'
                    ''.format(setting))
             status_set('blocked', msg)
             raise DNSHAException(msg)
 
-        hostname_key = 'res_{}_{}_hostname'.format(charm_name(), networkspace)
+        hostname_key = 'res_{}_{}_hostname'.format(charm_name(), endpoint_type)
         if hostname_key in hostname_group:
             log('DNS HA: Resource {}: {} already exists in '
                 'hostname group - skipping'.format(hostname_key, hostname),
@@ -101,7 +104,7 @@ def update_dns_ha_resource_params(resources, resource_params,
         resources[hostname_key] = crm_ocf
         resource_params[hostname_key] = (
             'params fqdn="{}" ip_address="{}" '
-            ''.format(hostname, resolve_address(endpoint_type=networkspace,
+            ''.format(hostname, resolve_address(endpoint_type=endpoint_type,
                                                 override=False)))
 
     if len(hostname_group) >= 1:

charmhelpers/contrib/openstack/neutron.py

@@ -59,18 +59,13 @@ def determine_dkms_package():
 
 
 def quantum_plugins():
-    from charmhelpers.contrib.openstack import context
     return {
         'ovs': {
             'config': '/etc/quantum/plugins/openvswitch/'
                       'ovs_quantum_plugin.ini',
             'driver': 'quantum.plugins.openvswitch.ovs_quantum_plugin.'
                       'OVSQuantumPluginV2',
-            'contexts': [
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=QUANTUM_CONF_DIR)],
+            'contexts': [],
             'services': ['quantum-plugin-openvswitch-agent'],
             'packages': [determine_dkms_package(),
                          ['quantum-plugin-openvswitch-agent']],
@@ -82,11 +77,7 @@ def quantum_plugins():
             'config': '/etc/quantum/plugins/nicira/nvp.ini',
             'driver': 'quantum.plugins.nicira.nicira_nvp_plugin.'
                       'QuantumPlugin.NvpPluginV2',
-            'contexts': [
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=QUANTUM_CONF_DIR)],
+            'contexts': [],
             'services': [],
             'packages': [],
             'server_packages': ['quantum-server',
@@ -100,7 +91,6 @@ NEUTRON_CONF_DIR = '/etc/neutron'
 
 
 def neutron_plugins():
-    from charmhelpers.contrib.openstack import context
     release = os_release('nova-common')
     plugins = {
         'ovs': {
@@ -108,11 +98,7 @@ def neutron_plugins():
                       'ovs_neutron_plugin.ini',
             'driver': 'neutron.plugins.openvswitch.ovs_neutron_plugin.'
                       'OVSNeutronPluginV2',
-            'contexts': [
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'contexts': [],
             'services': ['neutron-plugin-openvswitch-agent'],
             'packages': [determine_dkms_package(),
                          ['neutron-plugin-openvswitch-agent']],
@@ -124,11 +110,7 @@ def neutron_plugins():
             'config': '/etc/neutron/plugins/nicira/nvp.ini',
             'driver': 'neutron.plugins.nicira.nicira_nvp_plugin.'
                       'NeutronPlugin.NvpPluginV2',
-            'contexts': [
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'contexts': [],
             'services': [],
             'packages': [],
             'server_packages': ['neutron-server',
@@ -138,11 +120,7 @@ def neutron_plugins():
         'nsx': {
             'config': '/etc/neutron/plugins/vmware/nsx.ini',
             'driver': 'vmware',
-            'contexts': [
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'contexts': [],
             'services': [],
             'packages': [],
             'server_packages': ['neutron-server',
@@ -152,11 +130,7 @@ def neutron_plugins():
         'n1kv': {
             'config': '/etc/neutron/plugins/cisco/cisco_plugins.ini',
             'driver': 'neutron.plugins.cisco.network_plugin.PluginV2',
-            'contexts': [
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'contexts': [],
             'services': [],
             'packages': [determine_dkms_package(),
                          ['neutron-plugin-cisco']],
@@ -167,11 +141,7 @@ def neutron_plugins():
         'Calico': {
             'config': '/etc/neutron/plugins/ml2/ml2_conf.ini',
             'driver': 'neutron.plugins.ml2.plugin.Ml2Plugin',
-            'contexts': [
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'contexts': [],
             'services': ['calico-felix',
                          'bird',
                          'neutron-dhcp-agent',
@@ -189,11 +159,7 @@ def neutron_plugins():
         'vsp': {
             'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini',
             'driver': 'neutron.plugins.nuage.plugin.NuagePlugin',
-            'contexts': [
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'contexts': [],
             'services': [],
             'packages': [],
             'server_packages': ['neutron-server', 'neutron-plugin-nuage'],
@@ -203,10 +169,7 @@ def neutron_plugins():
             'config': '/etc/neutron/plugins/plumgrid/plumgrid.ini',
             'driver': ('neutron.plugins.plumgrid.plumgrid_plugin'
                        '.plumgrid_plugin.NeutronPluginPLUMgridV2'),
-            'contexts': [
-                context.SharedDBContext(user=config('database-user'),
-                                        database=config('database'),
-                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'contexts': [],
             'services': [],
             'packages': ['plumgrid-lxc',
                          'iovisor-dkms'],
@@ -217,11 +180,7 @@ def neutron_plugins():
         'midonet': {
             'config': '/etc/neutron/plugins/midonet/midonet.ini',
             'driver': 'midonet.neutron.plugin.MidonetPluginV2',
-            'contexts': [
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'contexts': [],
             'services': [],
             'packages': [determine_dkms_package()],
             'server_packages': ['neutron-server',

charmhelpers/contrib/openstack/templates/haproxy.cfg

@@ -48,7 +48,9 @@ listen stats
 {% for service, ports in service_ports.items() -%}
 frontend tcp-in_{{ service }}
     bind *:{{ ports[0] }}
+    {% if ipv6_enabled -%}
     bind :::{{ ports[0] }}
+    {% endif -%}
     {% for frontend in frontends -%}
     acl net_{{ frontend }} dst {{ frontends[frontend]['network'] }}
     use_backend {{ service }}_{{ frontend }} if net_{{ frontend }}

charmhelpers/contrib/openstack/templates/section-oslo-cache

@@ -0,0 +1,6 @@
+[cache]
+{% if memcache_url %}
+enabled = true
+backend = oslo_cache.memcache_pool
+memcache_servers = {{ memcache_url }}
+{% endif %}

charmhelpers/contrib/openstack/utils.py

@@ -95,7 +95,7 @@ from charmhelpers.fetch import (
 from charmhelpers.fetch.snap import (
     snap_install,
     snap_refresh,
-    SNAP_CHANNELS,
+    valid_snap_channel,
 )
 
 from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
@@ -426,7 +426,7 @@ def get_os_codename_package(package, fatal=True):
 
     try:
         pkg = cache[package]
-    except:
+    except Exception:
         if not fatal:
             return None
         # the package is unknown to the current apt cache.
@@ -579,6 +579,9 @@ def configure_installation_source(source_plus_key):
     Note that the behaviour on error is to log the error to the juju log and
     then call sys.exit(1).
     """
+    if source_plus_key.startswith('snap'):
+        # Do nothing for snap installs
+        return
     # extract the key if there is one, denoted by a '|' in the rel
     source, key = get_source_and_pgp_key(source_plus_key)
 
@@ -1615,7 +1618,7 @@ def do_action_openstack_upgrade(package, upgrade_callback, configs):
                     upgrade_callback(configs=configs)
                     action_set({'outcome': 'success, upgrade completed.'})
                     ret = True
-                except:
+                except Exception:
                     action_set({'outcome': 'upgrade failed, see traceback.'})
                     action_set({'traceback': traceback.format_exc()})
                     action_fail('do_openstack_upgrade resulted in an '
@@ -1720,7 +1723,7 @@ def is_unit_paused_set():
             kv = t[0]
             # transform something truth-y into a Boolean.
             return not(not(kv.get('unit-paused')))
-    except:
+    except Exception:
         return False
 
 
@@ -2048,7 +2051,7 @@ def update_json_file(filename, items):
 def snap_install_requested():
     """ Determine if installing from snaps
 
-    If openstack-origin is of the form snap:channel-series-release
+    If openstack-origin is of the form snap:track/channel[/branch]
     and channel is in SNAPS_CHANNELS return True.
     """
     origin = config('openstack-origin') or ""
@@ -2056,10 +2059,12 @@ def snap_install_requested():
         return False
 
     _src = origin[5:]
-    channel, series, release = _src.split('-')
-    if channel.lower() in SNAP_CHANNELS:
-        return True
-    return False
+    if '/' in _src:
+        channel = _src.split('/')[1]
+    else:
+        # Handle snap:track with no channel
+        channel = 'stable'
+    return valid_snap_channel(channel)
 
 
 def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
@@ -2067,7 +2072,7 @@ def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
 
     @param snaps: List of snaps
     @param src: String of openstack-origin or source of the form
-        snap:channel-series-track
+        snap:track/channel
     @param mode: String classic, devmode or jailmode
     @returns: Dictionary of snaps with channels and modes
     """
@@ -2077,8 +2082,7 @@ def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
         return {}
 
     _src = src[5:]
-    _channel, _series, _release = _src.split('-')
-    channel = '--channel={}/{}'.format(_release, _channel)
+    channel = '--channel={}'.format(_src)
 
     return {snap: {'channel': channel, 'mode': mode}
             for snap in snaps}
@@ -2090,8 +2094,8 @@ def install_os_snaps(snaps, refresh=False):
     @param snaps: Dictionary of snaps with channels and modes of the form:
         {'snap_name': {'channel': 'snap_channel',
                        'mode': 'snap_mode'}}
-        Where channel a snapstore channel and mode is --classic, --devmode or
-        --jailmode.
+        Where channel is a snapstore channel and mode is --classic, --devmode
+        or --jailmode.
     @param post_snap_install: Callback function to run after snaps have been
     installed
     """

charmhelpers/contrib/storage/linux/ceph.py

@@ -370,9 +370,10 @@ def get_mon_map(service):
       Also raises CalledProcessError if our ceph command fails
     """
     try:
-        mon_status = check_output(
-            ['ceph', '--id', service,
-             'mon_status', '--format=json'])
+        mon_status = check_output(['ceph', '--id', service,
+                                   'mon_status', '--format=json'])
+        if six.PY3:
+            mon_status = mon_status.decode('UTF-8')
         try:
             return json.loads(mon_status)
         except ValueError as v:
@@ -457,7 +458,7 @@ def monitor_key_get(service, key):
     try:
         output = check_output(
             ['ceph', '--id', service,
-             'config-key', 'get', str(key)])
+             'config-key', 'get', str(key)]).decode('UTF-8')
         return output
     except CalledProcessError as e:
         log("Monitor config-key get failed with message: {}".format(
@@ -500,6 +501,8 @@ def get_erasure_profile(service, name):
         out = check_output(['ceph', '--id', service,
                             'osd', 'erasure-code-profile', 'get',
                             name, '--format=json'])
+        if six.PY3:
+            out = out.decode('UTF-8')
         return json.loads(out)
     except (CalledProcessError, OSError, ValueError):
         return None
@@ -686,7 +689,10 @@ def get_cache_mode(service, pool_name):
     """
     validator(value=service, valid_type=six.string_types)
     validator(value=pool_name, valid_type=six.string_types)
-    out = check_output(['ceph', '--id', service, 'osd', 'dump', '--format=json'])
+    out = check_output(['ceph', '--id', service,
+                        'osd', 'dump', '--format=json'])
+    if six.PY3:
+        out = out.decode('UTF-8')
     try:
         osd_json = json.loads(out)
         for pool in osd_json['pools']:
@@ -700,8 +706,9 @@ def get_cache_mode(service, pool_name):
 def pool_exists(service, name):
     """Check to see if a RADOS pool already exists."""
     try:
-        out = check_output(['rados', '--id', service,
-                            'lspools']).decode('UTF-8')
+        out = check_output(['rados', '--id', service, 'lspools'])
+        if six.PY3:
+            out = out.decode('UTF-8')
     except CalledProcessError:
         return False
 
@@ -714,9 +721,12 @@ def get_osds(service):
     """
     version = ceph_version()
     if version and version >= '0.56':
-        return json.loads(check_output(['ceph', '--id', service,
-                                        'osd', 'ls',
-                                        '--format=json']).decode('UTF-8'))
+        out = check_output(['ceph', '--id', service,
+                            'osd', 'ls',
+                            '--format=json'])
+        if six.PY3:
+            out = out.decode('UTF-8')
+        return json.loads(out)
 
     return None
 
@@ -734,7 +744,9 @@ def rbd_exists(service, pool, rbd_img):
     """Check to see if a RADOS block device exists."""
     try:
         out = check_output(['rbd', 'list', '--id',
-                            service, '--pool', pool]).decode('UTF-8')
+                            service, '--pool', pool])
+        if six.PY3:
+            out = out.decode('UTF-8')
     except CalledProcessError:
         return False
 
@@ -859,7 +871,9 @@ def configure(service, key, auth, use_syslog):
 def image_mapped(name):
     """Determine whether a RADOS block device is mapped locally."""
     try:
-        out = check_output(['rbd', 'showmapped']).decode('UTF-8')
+        out = check_output(['rbd', 'showmapped'])
+        if six.PY3:
+            out = out.decode('UTF-8')
     except CalledProcessError:
         return False
 
@@ -1018,7 +1032,9 @@ def ceph_version():
     """Retrieve the local version of ceph."""
     if os.path.exists('/usr/bin/ceph'):
         cmd = ['ceph', '-v']
-        output = check_output(cmd).decode('US-ASCII')
+        output = check_output(cmd)
+        if six.PY3:
+            output = output.decode('UTF-8')
         output = output.split()
         if len(output) > 3:
             return output[2]

charmhelpers/contrib/storage/linux/lvm.py

@@ -74,10 +74,10 @@ def list_lvm_volume_group(block_device):
     '''
     vg = None
     pvd = check_output(['pvdisplay', block_device]).splitlines()
-    for l in pvd:
-        l = l.decode('UTF-8')
-        if l.strip().startswith('VG Name'):
-            vg = ' '.join(l.strip().split()[2:])
+    for lvm in pvd:
+        lvm = lvm.decode('UTF-8')
+        if lvm.strip().startswith('VG Name'):
+            vg = ' '.join(lvm.strip().split()[2:])
     return vg
 
 

charmhelpers/contrib/storage/linux/utils.py

@@ -64,6 +64,6 @@ def is_device_mounted(device):
     '''
     try:
         out = check_output(['lsblk', '-P', device]).decode('UTF-8')
-    except:
+    except Exception:
         return False
     return bool(re.search(r'MOUNTPOINT=".+"', out))

charmhelpers/core/hookenv.py

@@ -22,6 +22,7 @@ from __future__ import print_function
 import copy
 from distutils.version import LooseVersion
 from functools import wraps
+from collections import namedtuple
 import glob
 import os
 import json
@@ -218,6 +219,8 @@ def principal_unit():
         for rid in relation_ids(reltype):
             for unit in related_units(rid):
                 md = _metadata_unit(unit)
+                if not md:
+                    continue
                 subordinate = md.pop('subordinate', None)
                 if not subordinate:
                     return unit
@@ -511,7 +514,10 @@ def _metadata_unit(unit):
     """
     basedir = os.sep.join(charm_dir().split(os.sep)[:-2])
     unitdir = 'unit-{}'.format(unit.replace(os.sep, '-'))
-    with open(os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')) as md:
+    joineddir = os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')
+    if not os.path.exists(joineddir):
+        return None
+    with open(joineddir) as md:
         return yaml.safe_load(md)
 
 
@@ -639,18 +645,31 @@ def is_relation_made(relation, keys='private-address'):
     return False
 
 
+def _port_op(op_name, port, protocol="TCP"):
+    """Open or close a service network port"""
+    _args = [op_name]
+    icmp = protocol.upper() == "ICMP"
+    if icmp:
+        _args.append(protocol)
+    else:
+        _args.append('{}/{}'.format(port, protocol))
+    try:
+        subprocess.check_call(_args)
+    except subprocess.CalledProcessError:
+        # Older Juju pre 2.3 doesn't support ICMP
+        # so treat it as a no-op if it fails.
+        if not icmp:
+            raise
+
+
 def open_port(port, protocol="TCP"):
     """Open a service network port"""
-    _args = ['open-port']
-    _args.append('{}/{}'.format(port, protocol))
-    subprocess.check_call(_args)
+    _port_op('open-port', port, protocol)
 
 
 def close_port(port, protocol="TCP"):
     """Close a service network port"""
-    _args = ['close-port']
-    _args.append('{}/{}'.format(port, protocol))
-    subprocess.check_call(_args)
+    _port_op('close-port', port, protocol)
 
 
 def open_ports(start, end, protocol="TCP"):
@@ -667,6 +686,17 @@ def close_ports(start, end, protocol="TCP"):
     subprocess.check_call(_args)
 
 
+def opened_ports():
+    """Get the opened ports
+
+    *Note that this will only show ports opened in a previous hook*
+
+    :returns: Opened ports as a list of strings: ``['8080/tcp', '8081-8083/tcp']``
+    """
+    _args = ['opened-ports', '--format=json']
+    return json.loads(subprocess.check_output(_args).decode('UTF-8'))
+
+
 @cached
 def unit_get(attribute):
     """Get the unit ID for the remote unit"""
@@ -1077,6 +1107,35 @@ def network_get_primary_address(binding):
     return subprocess.check_output(cmd).decode('UTF-8').strip()
 
 
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def network_get(endpoint, relation_id=None):
+    """
+    Retrieve the network details for a relation endpoint
+
+    :param endpoint: string. The name of a relation endpoint
+    :param relation_id: int. The ID of the relation for the current context.
+    :return: dict. The loaded YAML output of the network-get query.
+    :raise: NotImplementedError if run on Juju < 2.1
+    """
+    cmd = ['network-get', endpoint, '--format', 'yaml']
+    if relation_id:
+        cmd.append('-r')
+        cmd.append(relation_id)
+    try:
+        response = subprocess.check_output(
+            cmd,
+            stderr=subprocess.STDOUT).decode('UTF-8').strip()
+    except CalledProcessError as e:
+        # Early versions of Juju 2.0.x required the --primary-address argument.
+        # We catch that condition here and raise NotImplementedError since
+        # the requested semantics are not available - the caller can then
+        # use the network_get_primary_address() method instead.
+        if '--primary-address is currently required' in e.output.decode('UTF-8'):
+            raise NotImplementedError
+        raise
+    return yaml.safe_load(response)
+
+
 def add_metric(*args, **kwargs):
     """Add metric values. Values may be expressed with keyword arguments. For
     metric names containing dashes, these may be expressed as one or more
@@ -1106,3 +1165,42 @@ def meter_info():
     """Get the meter status information, if running in the meter-status-changed
     hook."""
     return os.environ.get('JUJU_METER_INFO')
+
+
+def iter_units_for_relation_name(relation_name):
+    """Iterate through all units in a relation
+
+    Generator that iterates through all the units in a relation and yields
+    a named tuple with rid and unit field names.
+
+    Usage:
+    data = [(u.rid, u.unit)
+            for u in iter_units_for_relation_name(relation_name)]
+
+    :param relation_name: string relation name
+    :yield: Named Tuple with rid and unit field names
+    """
+    RelatedUnit = namedtuple('RelatedUnit', 'rid, unit')
+    for rid in relation_ids(relation_name):
+        for unit in related_units(rid):
+            yield RelatedUnit(rid, unit)
+
+
+def ingress_address(rid=None, unit=None):
+    """
+    Retrieve the ingress-address from a relation when available. Otherwise,
+    return the private-address. This function is to be used on the consuming
+    side of the relation.
+
+    Usage:
+    addresses = [ingress_address(rid=u.rid, unit=u.unit)
+                 for u in iter_units_for_relation_name(relation_name)]
+
+    :param rid: string relation id
+    :param unit: string unit name
+    :side effect: calls relation_get
+    :return: string IP address
+    """
+    settings = relation_get(rid=rid, unit=unit)
+    return (settings.get('ingress-address') or
+            settings.get('private-address'))

charmhelpers/core/host.py

@@ -34,7 +34,7 @@ import six
 
 from contextlib import contextmanager
 from collections import OrderedDict
-from .hookenv import log, DEBUG
+from .hookenv import log, DEBUG, local_unit
 from .fstab import Fstab
 from charmhelpers.osplatform import get_platform
 
@@ -441,6 +441,49 @@ def add_user_to_group(username, group):
     subprocess.check_call(cmd)
 
 
+def chage(username, lastday=None, expiredate=None, inactive=None,
+           mindays=None, maxdays=None, root=None, warndays=None):
+    """Change user password expiry information
+
+    :param str username: User to update
+    :param str lastday: Set when password was changed in YYYY-MM-DD format
+    :param str expiredate: Set when user's account will no longer be
+                           accessible in YYYY-MM-DD format.
+                           -1 will remove an account expiration date.
+    :param str inactive: Set the number of days of inactivity after a password
+                         has expired before the account is locked.
+                         -1 will remove an account's inactivity.
+    :param str mindays: Set the minimum number of days between password
+                        changes to MIN_DAYS.
+                        0 indicates the password can be changed anytime.
+    :param str maxdays: Set the maximum number of days during which a
+                        password is valid.
+                        -1 as MAX_DAYS will remove checking maxdays
+    :param str root: Apply changes in the CHROOT_DIR directory
+    :param str warndays: Set the number of days of warning before a password
+                         change is required
+    :raises subprocess.CalledProcessError: if call to chage fails
+    """
+    cmd = ['chage']
+    if root:
+        cmd.extend(['--root', root])
+    if lastday:
+        cmd.extend(['--lastday', lastday])
+    if expiredate:
+        cmd.extend(['--expiredate', expiredate])
+    if inactive:
+        cmd.extend(['--inactive', inactive])
+    if mindays:
+        cmd.extend(['--mindays', mindays])
+    if maxdays:
+        cmd.extend(['--maxdays', maxdays])
+    if warndays:
+        cmd.extend(['--warndays', warndays])
+    cmd.append(username)
+    subprocess.check_call(cmd)
+
+remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1')
+
 def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
     """Replicate the contents of a path"""
     options = options or ['--delete', '--executability']
@@ -946,3 +989,31 @@ def updatedb(updatedb_text, new_path):
                 lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths))
     output = "\n".join(lines)
     return output
+
+
+def modulo_distribution(modulo=3, wait=30):
+    """ Modulo distribution
+
+    This helper uses the unit number, a modulo value and a constant wait time
+    to produce a calculated wait time distribution. This is useful in large
+    scale deployments to distribute load during an expensive operation such as
+    service restarts.
+
+    If you have 1000 nodes that need to restart 100 at a time 1 minute at a
+    time:
+
+      time.wait(modulo_distribution(modulo=100, wait=60))
+      restart()
+
+    If you need restarts to happen serially set modulo to the exact number of
+    nodes and set a high constant wait time:
+
+      time.wait(modulo_distribution(modulo=10, wait=120))
+      restart()
+
+    @param modulo: int The modulo number creates the group distribution
+    @param wait: int The constant time wait value
+    @return: int Calculated time to wait for unit operation
+    """
+    unit_number = int(local_unit().split('/')[1])
+    return (unit_number % modulo) * wait

charmhelpers/core/unitdata.py

@@ -358,7 +358,7 @@ class Storage(object):
         try:
             yield self.revision
             self.revision = None
-        except:
+        except Exception:
             self.flush(False)
             self.revision = None
             raise

charmhelpers/fetch/snap.py

@@ -41,6 +41,10 @@ class CouldNotAcquireLockException(Exception):
     pass
 
 
+class InvalidSnapChannel(Exception):
+    pass
+
+
 def _snap_exec(commands):
     """
     Execute snap commands.
@@ -132,3 +136,15 @@ def snap_refresh(packages, *flags):
 
     log(message, level='INFO')
     return _snap_exec(['refresh'] + flags + packages)
+
+
+def valid_snap_channel(channel):
+    """ Validate snap channel exists
+
+    :raises InvalidSnapChannel: When channel does not exist
+    :return: Boolean
+    """
+    if channel.lower() in SNAP_CHANNELS:
+        return True
+    else:
+        raise InvalidSnapChannel("Invalid Snap Channel: {}".format(channel))

charmhelpers/fetch/ubuntu.py

@@ -572,7 +572,7 @@ def get_upstream_version(package):
     cache = apt_cache()
     try:
         pkg = cache[package]
-    except:
+    except Exception:
         # the package is unknown to the current apt cache.
         return None
 

config.yaml

@@ -171,3 +171,14 @@ options:
       Sample rate determines what percentage of the metric points a
       client should send to the server.
       Only takes effect if statsd-host is set.
+  allow-ufw-ip6-softfail:
+    description: |
+      When this option is set to True the charm will disable the IPv6
+      support in ufw in case ip6tables couldn't be activated, situations
+      where this could happen is in a LXC container running on top of a
+      host that doesn't have loaded the ip6_tables.
+
+      If this option is False (the default) and ip6_tables module couldn't
+      be loaded, the charm will fail to install.
+    type: boolean
+    default: False

hooks/swift-storage-relation-departed

@@ -0,0 +1 @@
+swift_storage_hooks.py

hooks/swift_storage_hooks.py

@@ -35,6 +35,8 @@ from lib.swift_storage_utils import (
     assess_status,
     ensure_devs_tracked,
     VERSION_PACKAGE,
+    setup_ufw,
+    revoke_access,
 )
 
 from lib.misc_utils import pause_aware_restart_on_change
@@ -48,6 +50,7 @@ from charmhelpers.core.hookenv import (
     relation_set,
     relations_of_type,
     status_set,
+    ingress_address,
 )
 
 from charmhelpers.fetch import (
@@ -73,6 +76,7 @@ from charmhelpers.contrib.openstack.utils import (
 from charmhelpers.contrib.network.ip import (
     get_relation_ip,
 )
+from charmhelpers.contrib.network import ufw
 from charmhelpers.contrib.charmsupport import nrpe
 from charmhelpers.contrib.hardening.harden import harden
 
@@ -85,6 +89,24 @@ SUDOERS_D = '/etc/sudoers.d'
 STORAGE_MOUNT_PATH = '/srv/node'
 
 
+def initialize_ufw():
+    """Initialize the UFW firewall
+
+    Ensure critical ports have explicit allows
+
+    :return: None
+    """
+    # this charm will monitor exclusively the ports used, using 'allow' as
+    # default policy enables sharing the machine with other services
+    ufw.default_policy('allow', 'incoming')
+    # Rsync manages its own ACLs
+    ufw.service('rsync', 'open')
+    # Guarantee SSH access
+    ufw.service('ssh', 'open')
+    # Enable
+    ufw.enable(soft_fail=config('allow-ufw-ip6-softfail'))
+
+
 @hooks.hook('install.real')
 @harden()
 def install():
@@ -94,6 +116,7 @@ def install():
     status_set('maintenance', 'Installing apt packages')
     apt_update()
     apt_install(PACKAGES, fatal=True)
+    initialize_ufw()
     status_set('maintenance', 'Setting up storage')
     setup_storage()
     ensure_swift_directories()
@@ -103,6 +126,7 @@ def install():
 @pause_aware_restart_on_change(RESTART_MAP)
 @harden()
 def config_changed():
+    initialize_ufw()
     if config('prefer-ipv6'):
         status_set('maintenance', 'Configuring ipv6')
         assert_charm_supports_ipv6()
@@ -136,6 +160,7 @@ def config_changed():
 @hooks.hook('upgrade-charm')
 @harden()
 def upgrade_charm():
+    initialize_ufw()
     apt_install(filter_installed_packages(PACKAGES), fatal=True)
     update_nrpe_config()
     ensure_devs_tracked()
@@ -164,6 +189,7 @@ def swift_storage_relation_joined(rid=None):
 @hooks.hook('swift-storage-relation-changed')
 @pause_aware_restart_on_change(RESTART_MAP)
 def swift_storage_relation_changed():
+    setup_ufw()
     rings_url = relation_get('rings_url')
     swift_hash = relation_get('swift_hash')
     if '' in [rings_url, swift_hash] or None in [rings_url, swift_hash]:
@@ -176,6 +202,17 @@ def swift_storage_relation_changed():
     fetch_swift_rings(rings_url)
 
 
+@hooks.hook('swift-storage-relation-departed')
+def swift_storage_relation_departed():
+    ports = [config('object-server-port'),
+             config('container-server-port'),
+             config('account-server-port')]
+    removed_client = ingress_address()
+    if removed_client:
+        for port in ports:
+            revoke_access(removed_client, port)
+
+
 @hooks.hook('nrpe-external-master-relation-joined')
 @hooks.hook('nrpe-external-master-relation-changed')
 def update_nrpe_config():

lib/swift_storage_utils.py

@@ -50,8 +50,12 @@ from charmhelpers.core.hookenv import (
     local_unit,
     relation_get,
     relation_ids,
+    iter_units_for_relation_name,
+    ingress_address,
 )
 
+from charmhelpers.contrib.network import ufw
+
 from charmhelpers.contrib.storage.linux.utils import (
     is_block_device,
     is_device_mounted,
@@ -119,6 +123,8 @@ RESTART_MAP = {
 SWIFT_CONF_DIR = '/etc/swift'
 SWIFT_RING_EXT = 'ring.gz'
 
+FIRST = 1
+
 # NOTE(hopem): we intentionally place this database outside of unit context so
 #              that if the unit, service or even entire environment is
 #              destroyed, there will still be a record of what devices were in
@@ -576,3 +582,53 @@ def assess_status(configs):
                 "Paused. Use 'resume' action to resume normal service.")
     else:
         return ("active", "Unit is ready")
+
+
+def grant_access(address, port):
+    """Grant TCP access to address and port via UFW
+
+    :side effect: calls ufw.grant_access
+    :return: None
+    """
+    log('granting access: {}:{}'.format(address, port), level='DEBUG')
+    ufw.grant_access(address, port=str(port), proto='tcp',
+                     index=FIRST)
+
+
+def revoke_access(address, port):
+    """Revoke TCP access to address and port via UFW
+
+    :side effect: calls ufw.revoke_access
+    :return: None
+    """
+    log('revoking access: {}'.format(address), level='DEBUG')
+    ufw.revoke_access(address, port=port, proto='tcp')
+
+
+def setup_ufw():
+    """Setup UFW firewall to ensure only swift-storage clients and storage
+    peers have access to the swift daemons.
+
+    :side effect: calls several external functions
+    :return: None
+    """
+    ports = [config('object-server-port'),
+             config('container-server-port'),
+             config('account-server-port')]
+
+    # Storage peers
+    allowed_hosts = RsyncContext()().get('allowed_hosts', '').split(' ')
+
+    # Storage clients (swift-proxy)
+    allowed_hosts += [ingress_address(rid=u.rid, unit=u.unit)
+                      for u in iter_units_for_relation_name('swift-storage')]
+
+    # Grant access for peers and clients
+    for host in allowed_hosts:
+        for port in ports:
+            grant_access(host, port)
+
+    # Default deny for storage ports
+    for port in ports:
+        ufw.modify_access(src=None, dst='any', port=port,
+                          proto='tcp', action='reject')

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -307,7 +307,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             # Kilo or later
             pools = [
                 'rbd',
-                'cinder',
+                'cinder-ceph',
                 'glance'
             ]
         else:
@@ -316,7 +316,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
                 'data',
                 'metadata',
                 'rbd',
-                'cinder',
+                'cinder-ceph',
                 'glance'
             ]
 

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -617,7 +617,7 @@ class OpenStackAmuletUtils(AmuletUtils):
             self.log.debug('Keypair ({}) already exists, '
                            'using it.'.format(keypair_name))
             return _keypair
-        except:
+        except Exception:
             self.log.debug('Keypair ({}) does not exist, '
                            'creating it.'.format(keypair_name))
 

tests/charmhelpers/core/hookenv.py

@@ -218,6 +218,8 @@ def principal_unit():
         for rid in relation_ids(reltype):
             for unit in related_units(rid):
                 md = _metadata_unit(unit)
+                if not md:
+                    continue
                 subordinate = md.pop('subordinate', None)
                 if not subordinate:
                     return unit
@@ -511,7 +513,10 @@ def _metadata_unit(unit):
     """
     basedir = os.sep.join(charm_dir().split(os.sep)[:-2])
     unitdir = 'unit-{}'.format(unit.replace(os.sep, '-'))
-    with open(os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')) as md:
+    joineddir = os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')
+    if not os.path.exists(joineddir):
+        return None
+    with open(joineddir) as md:
         return yaml.safe_load(md)
 
 
@@ -667,6 +672,17 @@ def close_ports(start, end, protocol="TCP"):
     subprocess.check_call(_args)
 
 
+def opened_ports():
+    """Get the opened ports
+
+    *Note that this will only show ports opened in a previous hook*
+
+    :returns: Opened ports as a list of strings: ``['8080/tcp', '8081-8083/tcp']``
+    """
+    _args = ['opened-ports', '--format=json']
+    return json.loads(subprocess.check_output(_args).decode('UTF-8'))
+
+
 @cached
 def unit_get(attribute):
     """Get the unit ID for the remote unit"""
@@ -1077,6 +1093,24 @@ def network_get_primary_address(binding):
     return subprocess.check_output(cmd).decode('UTF-8').strip()
 
 
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def network_get(endpoint, relation_id=None):
+    """
+    Retrieve the network details for a relation endpoint
+
+    :param endpoint: string. The name of a relation endpoint
+    :param relation_id: int. The ID of the relation for the current context.
+    :return: dict. The loaded YAML output of the network-get query.
+    :raise: NotImplementedError if run on Juju < 2.0
+    """
+    cmd = ['network-get', endpoint, '--format', 'yaml']
+    if relation_id:
+        cmd.append('-r')
+        cmd.append(relation_id)
+    response = subprocess.check_output(cmd).decode('UTF-8').strip()
+    return yaml.safe_load(response)
+
+
 def add_metric(*args, **kwargs):
     """Add metric values. Values may be expressed with keyword arguments. For
     metric names containing dashes, these may be expressed as one or more

tests/charmhelpers/core/host.py

@@ -34,7 +34,7 @@ import six
 
 from contextlib import contextmanager
 from collections import OrderedDict
-from .hookenv import log, DEBUG
+from .hookenv import log, DEBUG, local_unit
 from .fstab import Fstab
 from charmhelpers.osplatform import get_platform
 
@@ -441,6 +441,49 @@ def add_user_to_group(username, group):
     subprocess.check_call(cmd)
 
 
+def chage(username, lastday=None, expiredate=None, inactive=None,
+           mindays=None, maxdays=None, root=None, warndays=None):
+    """Change user password expiry information
+
+    :param str username: User to update
+    :param str lastday: Set when password was changed in YYYY-MM-DD format
+    :param str expiredate: Set when user's account will no longer be
+                           accessible in YYYY-MM-DD format.
+                           -1 will remove an account expiration date.
+    :param str inactive: Set the number of days of inactivity after a password
+                         has expired before the account is locked.
+                         -1 will remove an account's inactivity.
+    :param str mindays: Set the minimum number of days between password
+                        changes to MIN_DAYS.
+                        0 indicates the password can be changed anytime.
+    :param str maxdays: Set the maximum number of days during which a
+                        password is valid.
+                        -1 as MAX_DAYS will remove checking maxdays
+    :param str root: Apply changes in the CHROOT_DIR directory
+    :param str warndays: Set the number of days of warning before a password
+                         change is required
+    :raises subprocess.CalledProcessError: if call to chage fails
+    """
+    cmd = ['chage']
+    if root:
+        cmd.extend(['--root', root])
+    if lastday:
+        cmd.extend(['--lastday', lastday])
+    if expiredate:
+        cmd.extend(['--expiredate', expiredate])
+    if inactive:
+        cmd.extend(['--inactive', inactive])
+    if mindays:
+        cmd.extend(['--mindays', mindays])
+    if maxdays:
+        cmd.extend(['--maxdays', maxdays])
+    if warndays:
+        cmd.extend(['--warndays', warndays])
+    cmd.append(username)
+    subprocess.check_call(cmd)
+
+remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1')
+
 def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
     """Replicate the contents of a path"""
     options = options or ['--delete', '--executability']
@@ -946,3 +989,31 @@ def updatedb(updatedb_text, new_path):
                 lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths))
     output = "\n".join(lines)
     return output
+
+
+def modulo_distribution(modulo=3, wait=30):
+    """ Modulo distribution
+
+    This helper uses the unit number, a modulo value and a constant wait time
+    to produce a calculated wait time distribution. This is useful in large
+    scale deployments to distribute load during an expensive operation such as
+    service restarts.
+
+    If you have 1000 nodes that need to restart 100 at a time 1 minute at a
+    time:
+
+      time.wait(modulo_distribution(modulo=100, wait=60))
+      restart()
+
+    If you need restarts to happen serially set modulo to the exact number of
+    nodes and set a high constant wait time:
+
+      time.wait(modulo_distribution(modulo=10, wait=120))
+      restart()
+
+    @param modulo: int The modulo number creates the group distribution
+    @param wait: int The constant time wait value
+    @return: int Calculated time to wait for unit operation
+    """
+    unit_number = int(local_unit().split('/')[1])
+    return (unit_number % modulo) * wait

tests/charmhelpers/core/unitdata.py

@@ -358,7 +358,7 @@ class Storage(object):
         try:
             yield self.revision
             self.revision = None
-        except:
+        except Exception:
             self.flush(False)
             self.revision = None
             raise

unit_tests/test_swift_storage_relations.py

@@ -61,6 +61,9 @@ TO_PATCH = [
     'set_os_workload_status',
     'os_application_version_set',
     'add_to_updatedb_prunepath',
+    'ufw',
+    'setup_ufw',
+    'revoke_access',
 ]
 
 

unit_tests/test_swift_storage_utils.py

@@ -14,6 +14,7 @@
 
 import shutil
 import tempfile
+from collections import namedtuple
 
 from mock import call, patch, MagicMock
 from test_utils import CharmTestCase, patch_open
@@ -45,6 +46,10 @@ TO_PATCH = [
     'fstab_add',
     'mount',
     'is_mapped_loopback_device',
+    'ufw',
+    'iter_units_for_relation_name',
+    'ingress_address',
+    'relation_ids',
 ]
 
 
@@ -453,3 +458,45 @@ class SwiftStorageUtilsTests(CharmTestCase):
 
         mock_check_output.side_effect = fake_check_output
         self.assertIsNone(swift_utils.get_device_blkid(dev))
+
+    def test_grant_access(self):
+        addr = '10.1.1.1'
+        port = '80'
+        self.ufw.grant_access = MagicMock()
+        swift_utils.grant_access(addr, port)
+        self.ufw.grant_access.assert_called_with(addr, port=port, index=1, proto='tcp')
+
+
+    def test_revoke_access(self):
+        addr = '10.1.1.1'
+        port = '80'
+        self.ufw.revoke_access = MagicMock()
+        swift_utils.revoke_access(addr, port)
+        self.ufw.revoke_access.assert_called_with(addr, port=port, proto='tcp')
+
+    @patch.object(swift_utils, 'RsyncContext')
+    @patch.object(swift_utils, 'grant_access')
+    def test_setup_ufw(self, mock_grant_access, mock_rsync):
+        peer_addr_1 = '10.1.1.1'
+        peer_addr_2 = '10.1.1.2'
+        client_addrs = ['10.3.3.1', '10.3.3.2','10.3.3.3']
+        ports = [6660, 6661, 6662]
+        self.test_config.set('object-server-port', ports[0])
+        self.test_config.set('container-server-port', ports[1])
+        self.test_config.set('account-server-port', ports[2])
+        RelatedUnits = namedtuple('RelatedUnits', 'rid, unit')
+        self.iter_units_for_relation_name.return_value = [
+                RelatedUnits(rid='rid:1', unit='unit/1'),
+                RelatedUnits(rid='rid:1', unit='unit/2'),
+                RelatedUnits(rid='rid:1', unit='unit/3')]
+        self.ingress_address.side_effect = client_addrs
+        context_call = MagicMock()
+        context_call.return_value = {'allowed_hosts': '{} {}'
+                                     ''.format(peer_addr_1, peer_addr_2)}
+        mock_rsync.return_value = context_call
+        swift_utils.setup_ufw()
+        calls = []
+        for addr in [peer_addr_1, peer_addr_2] + client_addrs:
+            for port in ports:
+                calls.append(call(addr, port))
+        mock_grant_access.assert_has_calls(calls)