Support for neutron context without credentials

If the compute node is deployed in a cell it will get service
credentials via a direct relationship with keystone and not
from a nova-cloud-controller relation. This change adds support
for having a complete CloudComputeContext without having
service credentials.

Change-Id: Id39262c53f207bb61b0a60954d5a077562e4e6f3

hooks/charmhelpers/contrib/hahelpers/apache.py

@@ -23,8 +23,8 @@
 #
 
 import os
-import subprocess
 
+from charmhelpers.core import host
 from charmhelpers.core.hookenv import (
     config as config_get,
     relation_get,
@@ -83,14 +83,4 @@ def retrieve_ca_cert(cert_file):
 
 
 def install_ca_cert(ca_cert):
-    if ca_cert:
-        cert_file = ('/usr/local/share/ca-certificates/'
-                     'keystone_juju_ca_cert.crt')
-        old_cert = retrieve_ca_cert(cert_file)
-        if old_cert and old_cert == ca_cert:
-            log("CA cert is the same as installed version", level=INFO)
-        else:
-            log("Installing new CA cert", level=INFO)
-            with open(cert_file, 'wb') as crt:
-                crt.write(ca_cert)
-            subprocess.check_call(['update-ca-certificates', '--fresh'])
+    host.install_ca_cert(ca_cert, 'keystone_juju_ca_cert')

hooks/charmhelpers/contrib/hardening/apache/checks/config.py

@@ -14,6 +14,7 @@
 
 import os
 import re
+import six
 import subprocess
 
 
@@ -95,6 +96,8 @@ class ApacheConfContext(object):
         ctxt = settings['hardening']
 
         out = subprocess.check_output(['apache2', '-v'])
+        if six.PY3:
+            out = out.decode('utf-8')
         ctxt['apache_version'] = re.search(r'.+version: Apache/(.+?)\s.+',
                                            out).group(1)
         ctxt['apache_icondir'] = '/usr/share/apache2/icons/'

hooks/charmhelpers/contrib/hardening/audits/apache.py

@@ -15,7 +15,7 @@
 import re
 import subprocess
 
-from six import string_types
+import six
 
 from charmhelpers.core.hookenv import (
     log,
@@ -35,7 +35,7 @@ class DisabledModuleAudit(BaseAudit):
     def __init__(self, modules):
         if modules is None:
             self.modules = []
-        elif isinstance(modules, string_types):
+        elif isinstance(modules, six.string_types):
             self.modules = [modules]
         else:
             self.modules = modules
@@ -69,6 +69,8 @@ class DisabledModuleAudit(BaseAudit):
     def _get_loaded_modules():
         """Returns the modules which are enabled in Apache."""
         output = subprocess.check_output(['apache2ctl', '-M'])
+        if six.PY3:
+            output = output.decode('utf-8')
         modules = []
         for line in output.splitlines():
             # Each line of the enabled module output looks like:

hooks/charmhelpers/contrib/openstack/amulet/utils.py

@@ -618,12 +618,12 @@ class OpenStackAmuletUtils(AmuletUtils):
         return self.authenticate_keystone(keystone_ip, user, password,
                                           project_name=tenant)
 
-    def authenticate_glance_admin(self, keystone):
+    def authenticate_glance_admin(self, keystone, force_v1_client=False):
         """Authenticates admin user with glance."""
         self.log.debug('Authenticating glance admin...')
         ep = keystone.service_catalog.url_for(service_type='image',
                                               interface='adminURL')
-        if keystone.session:
+        if not force_v1_client and keystone.session:
             return glance_clientv2.Client("2", session=keystone.session)
         else:
             return glance_client.Client(ep, token=keystone.auth_token)
@@ -680,18 +680,30 @@ class OpenStackAmuletUtils(AmuletUtils):
             nova.flavors.create(name, ram, vcpus, disk, flavorid,
                                 ephemeral, swap, rxtx_factor, is_public)
 
-    def create_cirros_image(self, glance, image_name):
-        """Download the latest cirros image and upload it to glance,
-        validate and return a resource pointer.
+    def glance_create_image(self, glance, image_name, image_url,
+                            download_dir='tests',
+                            hypervisor_type=None,
+                            disk_format='qcow2',
+                            architecture='x86_64',
+                            container_format='bare'):
+        """Download an image and upload it to glance, validate its status
+        and return an image object pointer. KVM defaults, can override for
+        LXD.
 
-        :param glance: pointer to authenticated glance connection
+        :param glance: pointer to authenticated glance api connection
         :param image_name: display name for new image
+        :param image_url: url to retrieve
+        :param download_dir: directory to store downloaded image file
+        :param hypervisor_type: glance image hypervisor property
+        :param disk_format: glance image disk format
+        :param architecture: glance image architecture property
+        :param container_format: glance image container format
         :returns: glance image pointer
         """
-        self.log.debug('Creating glance cirros image '
-                       '({})...'.format(image_name))
+        self.log.debug('Creating glance image ({}) from '
+                       '{}...'.format(image_name, image_url))
 
-        # Download cirros image
+        # Download image
         http_proxy = os.getenv('AMULET_HTTP_PROXY')
         self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
         if http_proxy:
@@ -700,31 +712,34 @@ class OpenStackAmuletUtils(AmuletUtils):
         else:
             opener = urllib.FancyURLopener()
 
-        f = opener.open('http://download.cirros-cloud.net/version/released')
-        version = f.read().strip()
-        cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
-        local_path = os.path.join('tests', cirros_img)
-
-        if not os.path.exists(local_path):
-            cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
-                                                  version, cirros_img)
-            opener.retrieve(cirros_url, local_path)
-        f.close()
+        abs_file_name = os.path.join(download_dir, image_name)
+        if not os.path.exists(abs_file_name):
+            opener.retrieve(image_url, abs_file_name)
 
         # Create glance image
+        glance_properties = {
+            'architecture': architecture,
+        }
+        if hypervisor_type:
+            glance_properties['hypervisor_type'] = hypervisor_type
+        # Create glance image
         if float(glance.version) < 2.0:
-            with open(local_path) as fimage:
-                image = glance.images.create(name=image_name, is_public=True,
-                                             disk_format='qcow2',
-                                             container_format='bare',
-                                             data=fimage)
+            with open(abs_file_name) as f:
+                image = glance.images.create(
+                    name=image_name,
+                    is_public=True,
+                    disk_format=disk_format,
+                    container_format=container_format,
+                    properties=glance_properties,
+                    data=f)
         else:
             image = glance.images.create(
                 name=image_name,
-                disk_format="qcow2",
                 visibility="public",
-                container_format="bare")
-            glance.images.upload(image.id, open(local_path, 'rb'))
+                disk_format=disk_format,
+                container_format=container_format)
+            glance.images.upload(image.id, open(abs_file_name, 'rb'))
+            glance.images.update(image.id, **glance_properties)
 
         # Wait for image to reach active status
         img_id = image.id
@@ -753,15 +768,54 @@ class OpenStackAmuletUtils(AmuletUtils):
                         val_img_stat, val_img_cfmt, val_img_dfmt))
 
         if val_img_name == image_name and val_img_stat == 'active' \
-                and val_img_pub is True and val_img_cfmt == 'bare' \
-                and val_img_dfmt == 'qcow2':
+                and val_img_pub is True and val_img_cfmt == container_format \
+                and val_img_dfmt == disk_format:
             self.log.debug(msg_attr)
         else:
-            msg = ('Volume validation failed, {}'.format(msg_attr))
+            msg = ('Image validation failed, {}'.format(msg_attr))
             amulet.raise_status(amulet.FAIL, msg=msg)
 
         return image
 
+    def create_cirros_image(self, glance, image_name, hypervisor_type=None):
+        """Download the latest cirros image and upload it to glance,
+        validate and return a resource pointer.
+
+        :param glance: pointer to authenticated glance connection
+        :param image_name: display name for new image
+        :param hypervisor_type: glance image hypervisor property
+        :returns: glance image pointer
+        """
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'glance_create_image instead of '
+                      'create_cirros_image.')
+
+        self.log.debug('Creating glance cirros image '
+                       '({})...'.format(image_name))
+
+        # Get cirros image URL
+        http_proxy = os.getenv('AMULET_HTTP_PROXY')
+        self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
+        if http_proxy:
+            proxies = {'http': http_proxy}
+            opener = urllib.FancyURLopener(proxies)
+        else:
+            opener = urllib.FancyURLopener()
+
+        f = opener.open('http://download.cirros-cloud.net/version/released')
+        version = f.read().strip()
+        cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
+        cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
+                                              version, cirros_img)
+        f.close()
+
+        return self.glance_create_image(
+            glance,
+            image_name,
+            cirros_url,
+            hypervisor_type=hypervisor_type)
+
     def delete_image(self, glance, image):
         """Delete the specified image."""
 

hooks/charmhelpers/contrib/openstack/context.py

@@ -1523,10 +1523,6 @@ class NeutronAPIContext(OSContextGenerator):
                 'rel_key': 'enable-nsg-logging',
                 'default': False,
             },
-            'nsg_log_output_base': {
-                'rel_key': 'nsg-log-output-base',
-                'default': None,
-            },
         }
         ctxt = self.get_neutron_options({})
         for rid in relation_ids('neutron-plugin-api'):
@@ -1901,7 +1897,7 @@ class EnsureDirContext(OSContextGenerator):
     Some software requires a user to create a target directory to be
     scanned for drop-in files with a specific format. This is why this
     context is needed to do that before rendering a template.
-   '''
+    '''
 
     def __init__(self, dirname, **kwargs):
         '''Used merely to ensure that a given directory exists.'''
@@ -1911,3 +1907,23 @@ class EnsureDirContext(OSContextGenerator):
     def __call__(self):
         mkdir(self.dirname, **self.kwargs)
         return {}
+
+
+class VersionsContext(OSContextGenerator):
+    """Context to return the openstack and operating system versions.
+
+    """
+    def __init__(self, pkg='python-keystone'):
+        """Initialise context.
+
+        :param pkg: Package to extrapolate openstack version from.
+        :type pkg: str
+        """
+        self.pkg = pkg
+
+    def __call__(self):
+        ostack = os_release(self.pkg, base='icehouse')
+        osystem = lsb_release()['DISTRIB_CODENAME'].lower()
+        return {
+            'openstack_release': ostack,
+            'operating_system_release': osystem}

hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka

@@ -1,12 +1,14 @@
 {% if auth_host -%}
 [keystone_authtoken]
-auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
-auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
 auth_type = password
 {% if api_version == "3" -%}
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/v3
+auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v3
 project_domain_name = {{ admin_domain_name }}
 user_domain_name = {{ admin_domain_name }}
 {% else -%}
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
+auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
 project_domain_name = default
 user_domain_name = default
 {% endif -%}

hooks/charmhelpers/core/host.py

@@ -34,7 +34,7 @@ import six
 
 from contextlib import contextmanager
 from collections import OrderedDict
-from .hookenv import log, DEBUG, local_unit
+from .hookenv import log, INFO, DEBUG, local_unit, charm_name
 from .fstab import Fstab
 from charmhelpers.osplatform import get_platform
 
@@ -1040,3 +1040,27 @@ def modulo_distribution(modulo=3, wait=30, non_zero_wait=False):
         return modulo * wait
     else:
         return calculated_wait_time
+
+
+def install_ca_cert(ca_cert, name=None):
+    """
+    Install the given cert as a trusted CA.
+
+    The ``name`` is the stem of the filename where the cert is written, and if
+    not provided, it will default to ``juju-{charm_name}``.
+
+    If the cert is empty or None, or is unchanged, nothing is done.
+    """
+    if not ca_cert:
+        return
+    if not isinstance(ca_cert, bytes):
+        ca_cert = ca_cert.encode('utf8')
+    if not name:
+        name = 'juju-{}'.format(charm_name())
+    cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name)
+    new_hash = hashlib.md5(ca_cert).hexdigest()
+    if file_hash(cert_file) == new_hash:
+        return
+    log("Installing new CA cert at: {}".format(cert_file), level=INFO)
+    write_file(cert_file, ca_cert)
+    subprocess.check_call(['update-ca-certificates', '--fresh'])

hooks/nova_compute_context.py

@@ -474,6 +474,27 @@ class CloudComputeContext(context.OSContextGenerator):
 
         return neutron_ctxt
 
+    def neutron_context_no_auth_data(self):
+        """If the charm has a cloud-credentials relation then a subset
+        of data is needed to complete this context."""
+        neutron_ctxt = {'neutron_url': None}
+        for rid in relation_ids('cloud-compute'):
+            for unit in related_units(rid):
+                rel = {'rid': rid, 'unit': unit}
+
+                url = _neutron_url(**rel)
+                if not url:
+                    # only bother with units that have a neutron url set.
+                    continue
+
+                neutron_ctxt = {
+                    'neutron_auth_strategy': 'keystone',
+                    'neutron_plugin': _neutron_plugin(),
+                    'neutron_url': url,
+                }
+
+        return neutron_ctxt
+
     def volume_context(self):
         # provide basic validation that the volume manager is supported on the
         # given openstack release (nova-volume is only supported for E and F)
@@ -498,6 +519,10 @@ class CloudComputeContext(context.OSContextGenerator):
         elif self.network_manager == 'neutron':
             ctxt = self.neutron_context()
 
+            # If charm has a cloud-credentials relation then auth data is not
+            # needed.
+            if relation_ids('cloud-credentials') and not ctxt:
+                ctxt = self.neutron_context_no_auth_data()
         _save_flag_file(path='/etc/nova/nm.conf', data=self.network_manager)
 
         log('Generated config context for %s network manager.' %
@@ -520,22 +545,28 @@ class CloudComputeContext(context.OSContextGenerator):
         ctxt = {}
 
         net_manager = self.network_manager_context()
+
         if net_manager:
-            ctxt['network_manager'] = self.network_manager
-            ctxt['network_manager_config'] = net_manager
-            # This is duplicating information in the context to enable
-            # common keystone fragment to be used in template
-            ctxt['service_protocol'] = net_manager.get('service_protocol')
-            ctxt['service_host'] = net_manager.get('keystone_host')
-            ctxt['service_port'] = net_manager.get('service_port')
-            ctxt['admin_tenant_name'] = net_manager.get(
-                'neutron_admin_tenant_name')
-            ctxt['admin_user'] = net_manager.get('neutron_admin_username')
-            ctxt['admin_password'] = net_manager.get('neutron_admin_password')
-            ctxt['auth_protocol'] = net_manager.get('auth_protocol')
-            ctxt['auth_host'] = net_manager.get('keystone_host')
-            ctxt['auth_port'] = net_manager.get('auth_port')
-            ctxt['api_version'] = net_manager.get('api_version')
+            if net_manager.get('neutron_admin_password'):
+                ctxt['network_manager'] = self.network_manager
+                ctxt['network_manager_config'] = net_manager
+                # This is duplicating information in the context to enable
+                # common keystone fragment to be used in template
+                ctxt['service_protocol'] = net_manager.get('service_protocol')
+                ctxt['service_host'] = net_manager.get('keystone_host')
+                ctxt['service_port'] = net_manager.get('service_port')
+                ctxt['admin_tenant_name'] = net_manager.get(
+                    'neutron_admin_tenant_name')
+                ctxt['admin_user'] = net_manager.get('neutron_admin_username')
+                ctxt['admin_password'] = net_manager.get(
+                    'neutron_admin_password')
+                ctxt['auth_protocol'] = net_manager.get('auth_protocol')
+                ctxt['auth_host'] = net_manager.get('keystone_host')
+                ctxt['auth_port'] = net_manager.get('auth_port')
+                ctxt['api_version'] = net_manager.get('api_version')
+            else:
+                ctxt['network_manager'] = self.network_manager
+                ctxt['network_manager_config'] = net_manager
 
         net_dev_mtu = config('network-device-mtu')
         if net_dev_mtu:
@@ -552,7 +583,10 @@ class CloudComputeContext(context.OSContextGenerator):
         if region:
             ctxt['region'] = region
 
-        return ctxt
+        if self.context_complete(ctxt):
+            return ctxt
+
+        return {}
 
 
 class InstanceConsoleContext(context.OSContextGenerator):

hooks/nova_compute_utils.py

@@ -252,6 +252,7 @@ LIBVIRT_URIS = {
 REQUIRED_INTERFACES = {
     'messaging': ['amqp'],
     'image': ['image-service'],
+    'compute': ['cloud-compute'],
 }
 
 

tests/basic_deployment.py

@@ -178,8 +178,18 @@ class NovaBasicDeployment(OpenStackAmuletDeployment):
             self.keystone_sentry,
             openstack_release=self._get_openstack_release())
 
+        force_v1_client = False
+        if self._get_openstack_release() == self.trusty_icehouse:
+            # Updating image properties (such as arch or hypervisor) using the
+            # v2 api in icehouse results in:
+            # https://bugs.launchpad.net/python-glanceclient/+bug/1371559
+            u.log.debug('Forcing glance to use v1 api')
+            force_v1_client = True
+
         # Authenticate admin with glance endpoint
-        self.glance = u.authenticate_glance_admin(self.keystone)
+        self.glance = u.authenticate_glance_admin(
+            self.keystone,
+            force_v1_client=force_v1_client)
 
         # Authenticate admin with nova endpoint
         self.nova = nova_client.Client(2, session=self.keystone_session)

unit_tests/test_nova_compute_contexts.py

@@ -144,16 +144,6 @@ class NovaComputeContextTests(CharmTestCase):
                 'ec2_dmz_host': 'novaapihost',
                 'flat_interface': 'eth1'
             },
-            'service_protocol': None,
-            'service_host': None,
-            'service_port': None,
-            'admin_tenant_name': None,
-            'admin_user': None,
-            'admin_password': None,
-            'auth_protocol': None,
-            'auth_host': None,
-            'auth_port': None,
-            'api_version': None,
         }
         self.assertEqual(ex_ctxt, cloud_compute())