Merge "Dual Stack VIPs"

hooks/charmhelpers/contrib/openstack/context.py

@@ -41,9 +41,9 @@ from charmhelpers.core.hookenv import (
     charm_name,
     DEBUG,
     INFO,
-    WARNING,
     ERROR,
     status_set,
+    network_get_primary_address
 )
 
 from charmhelpers.core.sysctl import create as sysctl_create
@@ -80,6 +80,9 @@ from charmhelpers.contrib.openstack.neutron import (
 from charmhelpers.contrib.openstack.ip import (
     resolve_address,
     INTERNAL,
+    ADMIN,
+    PUBLIC,
+    ADDRESS_MAP,
 )
 from charmhelpers.contrib.network.ip import (
     get_address_in_network,
@@ -87,7 +90,6 @@ from charmhelpers.contrib.network.ip import (
     get_ipv6_addr,
     get_netmask_for_address,
     format_ipv6_addr,
-    is_address_in_network,
     is_bridge_member,
     is_ipv6_disabled,
 )
@@ -620,7 +622,6 @@ class HAProxyContext(OSContextGenerator):
             ctxt['haproxy_connect_timeout'] = config('haproxy-connect-timeout')
 
         if config('prefer-ipv6'):
-            ctxt['ipv6'] = True
             ctxt['local_host'] = 'ip6-localhost'
             ctxt['haproxy_host'] = '::'
         else:
@@ -736,11 +737,17 @@ class ApacheSSLContext(OSContextGenerator):
         return sorted(list(set(cns)))
 
     def get_network_addresses(self):
-        """For each network configured, return corresponding address and vip
+        """For each network configured, return corresponding address and
-           (if available).
+           hostnamr or vip (if available).
 
         Returns a list of tuples of the form:
 
+            [(address_in_net_a, hostname_in_net_a),
+             (address_in_net_b, hostname_in_net_b),
+             ...]
+
+            or, if no hostnames(s) available:
+
             [(address_in_net_a, vip_in_net_a),
              (address_in_net_b, vip_in_net_b),
              ...]
@@ -752,32 +759,27 @@ class ApacheSSLContext(OSContextGenerator):
              ...]
         """
         addresses = []
-        if config('vip'):
+        for net_type in [INTERNAL, ADMIN, PUBLIC]:
-            vips = config('vip').split()
+            net_config = config(ADDRESS_MAP[net_type]['config'])
-        else:
+            # NOTE(jamespage): Fallback must always be private address
-            vips = []
+            #                  as this is used to bind services on the
-
+            #                  local unit.
-        for net_type in ['os-internal-network', 'os-admin-network',
+            fallback = unit_get("private-address")
-                         'os-public-network']:
+            if net_config:
-            addr = get_address_in_network(config(net_type),
+                addr = get_address_in_network(net_config,
-                                          unit_get('private-address'))
+                                              fallback)
-            if len(vips) > 1 and is_clustered():
-                if not config(net_type):
-                    log("Multiple networks configured but net_type "
-                        "is None (%s)." % net_type, level=WARNING)
-                    continue
-
-                for vip in vips:
-                    if is_address_in_network(config(net_type), vip):
-                        addresses.append((addr, vip))
-                        break
-
-            elif is_clustered() and config('vip'):
-                addresses.append((addr, config('vip')))
             else:
-                addresses.append((addr, addr))
+                try:
+                    addr = network_get_primary_address(
+                        ADDRESS_MAP[net_type]['binding']
+                    )
+                except NotImplementedError:
+                    addr = fallback
 
-        return sorted(addresses)
+            endpoint = resolve_address(net_type)
+            addresses.append((addr, endpoint))
+
+        return sorted(set(addresses))
 
     def __call__(self):
         if isinstance(self.external_ports, six.string_types):
@@ -804,7 +806,7 @@ class ApacheSSLContext(OSContextGenerator):
             self.configure_cert(cn)
 
         addresses = self.get_network_addresses()
-        for address, endpoint in sorted(set(addresses)):
+        for address, endpoint in addresses:
             for api_port in self.external_ports:
                 ext_port = determine_apache_port(api_port,
                                                  singlenode_mode=True)
@@ -1419,14 +1421,26 @@ class NeutronAPIContext(OSContextGenerator):
                 'rel_key': 'report-interval',
                 'default': 30,
             },
+            'enable_qos': {
+                'rel_key': 'enable-qos',
+                'default': False,
+            },
         }
         ctxt = self.get_neutron_options({})
         for rid in relation_ids('neutron-plugin-api'):
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
+                # The l2-population key is used by the context as a way of
+                # checking if the api service on the other end is sending data
+                # in a recent format.
                 if 'l2-population' in rdata:
                     ctxt.update(self.get_neutron_options(rdata))
 
+        if ctxt['enable_qos']:
+            ctxt['extension_drivers'] = 'qos'
+        else:
+            ctxt['extension_drivers'] = ''
+
         return ctxt
 
     def get_neutron_options(self, rdata):

hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg

@@ -48,9 +48,7 @@ listen stats
 {% for service, ports in service_ports.items() -%}
 frontend tcp-in_{{ service }}
     bind *:{{ ports[0] }}
-    {% if ipv6 -%}
     bind :::{{ ports[0] }}
-    {% endif -%}
     {% for frontend in frontends -%}
     acl net_{{ frontend }} dst {{ frontends[frontend]['network'] }}
     use_backend {{ service }}_{{ frontend }} if net_{{ frontend }}

hooks/charmhelpers/contrib/openstack/templating.py

@@ -20,7 +20,8 @@ from charmhelpers.fetch import apt_install, apt_update
 from charmhelpers.core.hookenv import (
     log,
     ERROR,
-    INFO
+    INFO,
+    TRACE
 )
 from charmhelpers.contrib.openstack.utils import OPENSTACK_CODENAMES
 
@@ -80,8 +81,10 @@ def get_loader(templates_dir, os_release):
             loaders.insert(0, FileSystemLoader(tmpl_dir))
         if rel == os_release:
             break
+    # demote this log to the lowest level; we don't really need to see these
+    # lots in production even when debugging.
     log('Creating choice loader with dirs: %s' %
-        [l.searchpath for l in loaders], level=INFO)
+        [l.searchpath for l in loaders], level=TRACE)
     return ChoiceLoader(loaders)
 
 

hooks/charmhelpers/contrib/openstack/utils.py

@@ -186,7 +186,7 @@ SWIFT_CODENAMES = OrderedDict([
     ('ocata',
         ['2.11.0', '2.12.0', '2.13.0']),
     ('pike',
-        ['2.13.0']),
+        ['2.13.0', '2.15.0']),
 ])
 
 # >= Liberty version->codename mapping

hooks/charmhelpers/core/hookenv.py

@@ -43,6 +43,7 @@ ERROR = "ERROR"
 WARNING = "WARNING"
 INFO = "INFO"
 DEBUG = "DEBUG"
+TRACE = "TRACE"
 MARKER = object()
 
 cache = {}

hooks/charmhelpers/core/host.py

@@ -34,7 +34,7 @@ import six
 
 from contextlib import contextmanager
 from collections import OrderedDict
-from .hookenv import log
+from .hookenv import log, DEBUG
 from .fstab import Fstab
 from charmhelpers.osplatform import get_platform
 
@@ -487,13 +487,37 @@ def mkdir(path, owner='root', group='root', perms=0o555, force=False):
 
 def write_file(path, content, owner='root', group='root', perms=0o444):
     """Create or overwrite a file with the contents of a byte string."""
-    log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
     uid = pwd.getpwnam(owner).pw_uid
     gid = grp.getgrnam(group).gr_gid
-    with open(path, 'wb') as target:
+    # lets see if we can grab the file and compare the context, to avoid doing
-        os.fchown(target.fileno(), uid, gid)
+    # a write.
-        os.fchmod(target.fileno(), perms)
+    existing_content = None
-        target.write(content)
+    existing_uid, existing_gid = None, None
+    try:
+        with open(path, 'rb') as target:
+            existing_content = target.read()
+        stat = os.stat(path)
+        existing_uid, existing_gid = stat.st_uid, stat.st_gid
+    except:
+        pass
+    if content != existing_content:
+        log("Writing file {} {}:{} {:o}".format(path, owner, group, perms),
+            level=DEBUG)
+        with open(path, 'wb') as target:
+            os.fchown(target.fileno(), uid, gid)
+            os.fchmod(target.fileno(), perms)
+            target.write(content)
+        return
+    # the contents were the same, but we might still need to change the
+    # ownership.
+    if existing_uid != uid:
+        log("Changing uid on already existing content: {} -> {}"
+            .format(existing_uid, uid), level=DEBUG)
+        os.chown(path, uid, -1)
+    if existing_gid != gid:
+        log("Changing gid on already existing content: {} -> {}"
+            .format(existing_gid, gid), level=DEBUG)
+        os.chown(path, -1, gid)
 
 
 def fstab_remove(mp):

hooks/cinder_hooks.py

@@ -60,6 +60,7 @@ from charmhelpers.core.hookenv import (
     service_name,
     log,
     ERROR,
+    WARNING,
     status_set,
     open_port,
 )
@@ -510,6 +511,14 @@ def ha_joined(relation_id=None):
 
             if iface is not None:
                 vip_key = 'res_cinder_{}_vip'.format(iface)
+                if vip_key in vip_group:
+                    if vip not in resource_params[vip_key]:
+                        vip_key = '{}_{}'.format(vip_key, vip_params)
+                    else:
+                        log("Resource '%s' (vip='%s') already exists in "
+                            "vip group - skipping" % (vip_key, vip), WARNING)
+                        continue
+
                 resources[vip_key] = res_cinder_vip
                 resource_params[vip_key] = (
                     'params {ip}="{vip}" cidr_netmask="{netmask}"'

tests/charmhelpers/core/hookenv.py

@@ -43,6 +43,7 @@ ERROR = "ERROR"
 WARNING = "WARNING"
 INFO = "INFO"
 DEBUG = "DEBUG"
+TRACE = "TRACE"
 MARKER = object()
 
 cache = {}

tests/charmhelpers/core/host.py

@@ -34,7 +34,7 @@ import six
 
 from contextlib import contextmanager
 from collections import OrderedDict
-from .hookenv import log
+from .hookenv import log, DEBUG
 from .fstab import Fstab
 from charmhelpers.osplatform import get_platform
 
@@ -487,13 +487,37 @@ def mkdir(path, owner='root', group='root', perms=0o555, force=False):
 
 def write_file(path, content, owner='root', group='root', perms=0o444):
     """Create or overwrite a file with the contents of a byte string."""
-    log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
     uid = pwd.getpwnam(owner).pw_uid
     gid = grp.getgrnam(group).gr_gid
-    with open(path, 'wb') as target:
+    # lets see if we can grab the file and compare the context, to avoid doing
-        os.fchown(target.fileno(), uid, gid)
+    # a write.
-        os.fchmod(target.fileno(), perms)
+    existing_content = None
-        target.write(content)
+    existing_uid, existing_gid = None, None
+    try:
+        with open(path, 'rb') as target:
+            existing_content = target.read()
+        stat = os.stat(path)
+        existing_uid, existing_gid = stat.st_uid, stat.st_gid
+    except:
+        pass
+    if content != existing_content:
+        log("Writing file {} {}:{} {:o}".format(path, owner, group, perms),
+            level=DEBUG)
+        with open(path, 'wb') as target:
+            os.fchown(target.fileno(), uid, gid)
+            os.fchmod(target.fileno(), perms)
+            target.write(content)
+        return
+    # the contents were the same, but we might still need to change the
+    # ownership.
+    if existing_uid != uid:
+        log("Changing uid on already existing content: {} -> {}"
+            .format(existing_uid, uid), level=DEBUG)
+        os.chown(path, uid, -1)
+    if existing_gid != gid:
+        log("Changing gid on already existing content: {} -> {}"
+            .format(existing_gid, gid), level=DEBUG)
+        os.chown(path, -1, gid)
 
 
 def fstab_remove(mp):

unit_tests/test_cinder_contexts.py

@@ -145,6 +145,8 @@ class TestCinderContext(CharmTestCase):
 
     mod_ch_context = 'charmhelpers.contrib.openstack.context'
 
+    @patch('charmhelpers.contrib.openstack.context.resolve_address')
+    @patch('charmhelpers.contrib.openstack.ip.config')
     @patch('%s.ApacheSSLContext.canonical_names' % (mod_ch_context))
     @patch('%s.ApacheSSLContext.configure_ca' % (mod_ch_context))
     @patch('%s.config' % (mod_ch_context))
@@ -161,9 +163,12 @@ class TestCinderContext(CharmTestCase):
                                                 mock_is_clustered,
                                                 mock_hookenv,
                                                 mock_configure_ca,
-                                                mock_cfg_canonical_names):
+                                                mock_cfg_canonical_names,
+                                                mock_ip_config,
+                                                mock_ip_network_get):
         mock_https.return_value = True
         mock_unit_get.return_value = '1.2.3.4'
+        mock_ip_network_get.return_value = '1.2.3.4'
         mock_determine_api_port.return_value = '12'
         mock_determine_apache_port.return_value = '34'
         mock_is_clustered.return_value = False