diff --git a/hooks/charmhelpers/contrib/hahelpers/apache.py b/hooks/charmhelpers/contrib/hahelpers/apache.py index 605a1bec..2c1e371e 100644 --- a/hooks/charmhelpers/contrib/hahelpers/apache.py +++ b/hooks/charmhelpers/contrib/hahelpers/apache.py @@ -23,8 +23,8 @@ # import os -import subprocess +from charmhelpers.core import host from charmhelpers.core.hookenv import ( config as config_get, relation_get, @@ -83,14 +83,4 @@ def retrieve_ca_cert(cert_file): def install_ca_cert(ca_cert): - if ca_cert: - cert_file = ('/usr/local/share/ca-certificates/' - 'keystone_juju_ca_cert.crt') - old_cert = retrieve_ca_cert(cert_file) - if old_cert and old_cert == ca_cert: - log("CA cert is the same as installed version", level=INFO) - else: - log("Installing new CA cert", level=INFO) - with open(cert_file, 'wb') as crt: - crt.write(ca_cert) - subprocess.check_call(['update-ca-certificates', '--fresh']) + host.install_ca_cert(ca_cert, 'keystone_juju_ca_cert') diff --git a/hooks/charmhelpers/contrib/hardening/apache/checks/config.py b/hooks/charmhelpers/contrib/hardening/apache/checks/config.py index 06482aac..341da9ee 100644 --- a/hooks/charmhelpers/contrib/hardening/apache/checks/config.py +++ b/hooks/charmhelpers/contrib/hardening/apache/checks/config.py @@ -14,6 +14,7 @@ import os import re +import six import subprocess @@ -95,6 +96,8 @@ class ApacheConfContext(object): ctxt = settings['hardening'] out = subprocess.check_output(['apache2', '-v']) + if six.PY3: + out = out.decode('utf-8') ctxt['apache_version'] = re.search(r'.+version: Apache/(.+?)\s.+', out).group(1) ctxt['apache_icondir'] = '/usr/share/apache2/icons/' diff --git a/hooks/charmhelpers/contrib/hardening/audits/apache.py b/hooks/charmhelpers/contrib/hardening/audits/apache.py index d32bf44e..04825f5a 100644 --- a/hooks/charmhelpers/contrib/hardening/audits/apache.py +++ b/hooks/charmhelpers/contrib/hardening/audits/apache.py @@ -15,7 +15,7 @@ import re import subprocess -from six import string_types +import six from charmhelpers.core.hookenv import ( log, @@ -35,7 +35,7 @@ class DisabledModuleAudit(BaseAudit): def __init__(self, modules): if modules is None: self.modules = [] - elif isinstance(modules, string_types): + elif isinstance(modules, six.string_types): self.modules = [modules] else: self.modules = modules @@ -69,6 +69,8 @@ class DisabledModuleAudit(BaseAudit): def _get_loaded_modules(): """Returns the modules which are enabled in Apache.""" output = subprocess.check_output(['apache2ctl', '-M']) + if six.PY3: + output = output.decode('utf-8') modules = [] for line in output.splitlines(): # Each line of the enabled module output looks like: diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 936b4036..9133e9b3 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -618,12 +618,12 @@ class OpenStackAmuletUtils(AmuletUtils): return self.authenticate_keystone(keystone_ip, user, password, project_name=tenant) - def authenticate_glance_admin(self, keystone): + def authenticate_glance_admin(self, keystone, force_v1_client=False): """Authenticates admin user with glance.""" self.log.debug('Authenticating glance admin...') ep = keystone.service_catalog.url_for(service_type='image', interface='adminURL') - if keystone.session: + if not force_v1_client and keystone.session: return glance_clientv2.Client("2", session=keystone.session) else: return glance_client.Client(ep, token=keystone.auth_token) @@ -680,18 +680,30 @@ class OpenStackAmuletUtils(AmuletUtils): nova.flavors.create(name, ram, vcpus, disk, flavorid, ephemeral, swap, rxtx_factor, is_public) - def create_cirros_image(self, glance, image_name): - """Download the latest cirros image and upload it to glance, - validate and return a resource pointer. + def glance_create_image(self, glance, image_name, image_url, + download_dir='tests', + hypervisor_type=None, + disk_format='qcow2', + architecture='x86_64', + container_format='bare'): + """Download an image and upload it to glance, validate its status + and return an image object pointer. KVM defaults, can override for + LXD. - :param glance: pointer to authenticated glance connection + :param glance: pointer to authenticated glance api connection :param image_name: display name for new image + :param image_url: url to retrieve + :param download_dir: directory to store downloaded image file + :param hypervisor_type: glance image hypervisor property + :param disk_format: glance image disk format + :param architecture: glance image architecture property + :param container_format: glance image container format :returns: glance image pointer """ - self.log.debug('Creating glance cirros image ' - '({})...'.format(image_name)) + self.log.debug('Creating glance image ({}) from ' + '{}...'.format(image_name, image_url)) - # Download cirros image + # Download image http_proxy = os.getenv('AMULET_HTTP_PROXY') self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) if http_proxy: @@ -700,31 +712,34 @@ class OpenStackAmuletUtils(AmuletUtils): else: opener = urllib.FancyURLopener() - f = opener.open('http://download.cirros-cloud.net/version/released') - version = f.read().strip() - cirros_img = 'cirros-{}-x86_64-disk.img'.format(version) - local_path = os.path.join('tests', cirros_img) - - if not os.path.exists(local_path): - cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net', - version, cirros_img) - opener.retrieve(cirros_url, local_path) - f.close() + abs_file_name = os.path.join(download_dir, image_name) + if not os.path.exists(abs_file_name): + opener.retrieve(image_url, abs_file_name) # Create glance image + glance_properties = { + 'architecture': architecture, + } + if hypervisor_type: + glance_properties['hypervisor_type'] = hypervisor_type + # Create glance image if float(glance.version) < 2.0: - with open(local_path) as fimage: - image = glance.images.create(name=image_name, is_public=True, - disk_format='qcow2', - container_format='bare', - data=fimage) + with open(abs_file_name) as f: + image = glance.images.create( + name=image_name, + is_public=True, + disk_format=disk_format, + container_format=container_format, + properties=glance_properties, + data=f) else: image = glance.images.create( name=image_name, - disk_format="qcow2", visibility="public", - container_format="bare") - glance.images.upload(image.id, open(local_path, 'rb')) + disk_format=disk_format, + container_format=container_format) + glance.images.upload(image.id, open(abs_file_name, 'rb')) + glance.images.update(image.id, **glance_properties) # Wait for image to reach active status img_id = image.id @@ -753,15 +768,54 @@ class OpenStackAmuletUtils(AmuletUtils): val_img_stat, val_img_cfmt, val_img_dfmt)) if val_img_name == image_name and val_img_stat == 'active' \ - and val_img_pub is True and val_img_cfmt == 'bare' \ - and val_img_dfmt == 'qcow2': + and val_img_pub is True and val_img_cfmt == container_format \ + and val_img_dfmt == disk_format: self.log.debug(msg_attr) else: - msg = ('Volume validation failed, {}'.format(msg_attr)) + msg = ('Image validation failed, {}'.format(msg_attr)) amulet.raise_status(amulet.FAIL, msg=msg) return image + def create_cirros_image(self, glance, image_name, hypervisor_type=None): + """Download the latest cirros image and upload it to glance, + validate and return a resource pointer. + + :param glance: pointer to authenticated glance connection + :param image_name: display name for new image + :param hypervisor_type: glance image hypervisor property + :returns: glance image pointer + """ + # /!\ DEPRECATION WARNING + self.log.warn('/!\\ DEPRECATION WARNING: use ' + 'glance_create_image instead of ' + 'create_cirros_image.') + + self.log.debug('Creating glance cirros image ' + '({})...'.format(image_name)) + + # Get cirros image URL + http_proxy = os.getenv('AMULET_HTTP_PROXY') + self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) + if http_proxy: + proxies = {'http': http_proxy} + opener = urllib.FancyURLopener(proxies) + else: + opener = urllib.FancyURLopener() + + f = opener.open('http://download.cirros-cloud.net/version/released') + version = f.read().strip() + cirros_img = 'cirros-{}-x86_64-disk.img'.format(version) + cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net', + version, cirros_img) + f.close() + + return self.glance_create_image( + glance, + image_name, + cirros_url, + hypervisor_type=hypervisor_type) + def delete_image(self, glance, image): """Delete the specified image.""" diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 92cb742e..171890e7 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -1523,10 +1523,6 @@ class NeutronAPIContext(OSContextGenerator): 'rel_key': 'enable-nsg-logging', 'default': False, }, - 'nsg_log_output_base': { - 'rel_key': 'nsg-log-output-base', - 'default': None, - }, } ctxt = self.get_neutron_options({}) for rid in relation_ids('neutron-plugin-api'): @@ -1901,7 +1897,7 @@ class EnsureDirContext(OSContextGenerator): Some software requires a user to create a target directory to be scanned for drop-in files with a specific format. This is why this context is needed to do that before rendering a template. - ''' + ''' def __init__(self, dirname, **kwargs): '''Used merely to ensure that a given directory exists.''' @@ -1911,3 +1907,23 @@ class EnsureDirContext(OSContextGenerator): def __call__(self): mkdir(self.dirname, **self.kwargs) return {} + + +class VersionsContext(OSContextGenerator): + """Context to return the openstack and operating system versions. + + """ + def __init__(self, pkg='python-keystone'): + """Initialise context. + + :param pkg: Package to extrapolate openstack version from. + :type pkg: str + """ + self.pkg = pkg + + def __call__(self): + ostack = os_release(self.pkg, base='icehouse') + osystem = lsb_release()['DISTRIB_CODENAME'].lower() + return { + 'openstack_release': ostack, + 'operating_system_release': osystem} diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka index 8e6889e0..c281868b 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka +++ b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka @@ -1,12 +1,14 @@ {% if auth_host -%} [keystone_authtoken] -auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }} -auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }} auth_type = password {% if api_version == "3" -%} +auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/v3 +auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v3 project_domain_name = {{ admin_domain_name }} user_domain_name = {{ admin_domain_name }} {% else -%} +auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }} +auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }} project_domain_name = default user_domain_name = default {% endif -%} diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index e9fd38a0..0ebfdbd1 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -34,7 +34,7 @@ import six from contextlib import contextmanager from collections import OrderedDict -from .hookenv import log, DEBUG, local_unit +from .hookenv import log, INFO, DEBUG, local_unit, charm_name from .fstab import Fstab from charmhelpers.osplatform import get_platform @@ -1040,3 +1040,27 @@ def modulo_distribution(modulo=3, wait=30, non_zero_wait=False): return modulo * wait else: return calculated_wait_time + + +def install_ca_cert(ca_cert, name=None): + """ + Install the given cert as a trusted CA. + + The ``name`` is the stem of the filename where the cert is written, and if + not provided, it will default to ``juju-{charm_name}``. + + If the cert is empty or None, or is unchanged, nothing is done. + """ + if not ca_cert: + return + if not isinstance(ca_cert, bytes): + ca_cert = ca_cert.encode('utf8') + if not name: + name = 'juju-{}'.format(charm_name()) + cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name) + new_hash = hashlib.md5(ca_cert).hexdigest() + if file_hash(cert_file) == new_hash: + return + log("Installing new CA cert at: {}".format(cert_file), level=INFO) + write_file(cert_file, ca_cert) + subprocess.check_call(['update-ca-certificates', '--fresh']) diff --git a/hooks/nova_compute_context.py b/hooks/nova_compute_context.py index 739828e4..61922c18 100644 --- a/hooks/nova_compute_context.py +++ b/hooks/nova_compute_context.py @@ -474,6 +474,27 @@ class CloudComputeContext(context.OSContextGenerator): return neutron_ctxt + def neutron_context_no_auth_data(self): + """If the charm has a cloud-credentials relation then a subset + of data is needed to complete this context.""" + neutron_ctxt = {'neutron_url': None} + for rid in relation_ids('cloud-compute'): + for unit in related_units(rid): + rel = {'rid': rid, 'unit': unit} + + url = _neutron_url(**rel) + if not url: + # only bother with units that have a neutron url set. + continue + + neutron_ctxt = { + 'neutron_auth_strategy': 'keystone', + 'neutron_plugin': _neutron_plugin(), + 'neutron_url': url, + } + + return neutron_ctxt + def volume_context(self): # provide basic validation that the volume manager is supported on the # given openstack release (nova-volume is only supported for E and F) @@ -498,6 +519,10 @@ class CloudComputeContext(context.OSContextGenerator): elif self.network_manager == 'neutron': ctxt = self.neutron_context() + # If charm has a cloud-credentials relation then auth data is not + # needed. + if relation_ids('cloud-credentials') and not ctxt: + ctxt = self.neutron_context_no_auth_data() _save_flag_file(path='/etc/nova/nm.conf', data=self.network_manager) log('Generated config context for %s network manager.' % @@ -520,22 +545,28 @@ class CloudComputeContext(context.OSContextGenerator): ctxt = {} net_manager = self.network_manager_context() + if net_manager: - ctxt['network_manager'] = self.network_manager - ctxt['network_manager_config'] = net_manager - # This is duplicating information in the context to enable - # common keystone fragment to be used in template - ctxt['service_protocol'] = net_manager.get('service_protocol') - ctxt['service_host'] = net_manager.get('keystone_host') - ctxt['service_port'] = net_manager.get('service_port') - ctxt['admin_tenant_name'] = net_manager.get( - 'neutron_admin_tenant_name') - ctxt['admin_user'] = net_manager.get('neutron_admin_username') - ctxt['admin_password'] = net_manager.get('neutron_admin_password') - ctxt['auth_protocol'] = net_manager.get('auth_protocol') - ctxt['auth_host'] = net_manager.get('keystone_host') - ctxt['auth_port'] = net_manager.get('auth_port') - ctxt['api_version'] = net_manager.get('api_version') + if net_manager.get('neutron_admin_password'): + ctxt['network_manager'] = self.network_manager + ctxt['network_manager_config'] = net_manager + # This is duplicating information in the context to enable + # common keystone fragment to be used in template + ctxt['service_protocol'] = net_manager.get('service_protocol') + ctxt['service_host'] = net_manager.get('keystone_host') + ctxt['service_port'] = net_manager.get('service_port') + ctxt['admin_tenant_name'] = net_manager.get( + 'neutron_admin_tenant_name') + ctxt['admin_user'] = net_manager.get('neutron_admin_username') + ctxt['admin_password'] = net_manager.get( + 'neutron_admin_password') + ctxt['auth_protocol'] = net_manager.get('auth_protocol') + ctxt['auth_host'] = net_manager.get('keystone_host') + ctxt['auth_port'] = net_manager.get('auth_port') + ctxt['api_version'] = net_manager.get('api_version') + else: + ctxt['network_manager'] = self.network_manager + ctxt['network_manager_config'] = net_manager net_dev_mtu = config('network-device-mtu') if net_dev_mtu: @@ -552,7 +583,10 @@ class CloudComputeContext(context.OSContextGenerator): if region: ctxt['region'] = region - return ctxt + if self.context_complete(ctxt): + return ctxt + + return {} class InstanceConsoleContext(context.OSContextGenerator): diff --git a/hooks/nova_compute_utils.py b/hooks/nova_compute_utils.py index 567afe24..edfc985d 100644 --- a/hooks/nova_compute_utils.py +++ b/hooks/nova_compute_utils.py @@ -252,6 +252,7 @@ LIBVIRT_URIS = { REQUIRED_INTERFACES = { 'messaging': ['amqp'], 'image': ['image-service'], + 'compute': ['cloud-compute'], } diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index d9fec39d..197e629f 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -178,8 +178,18 @@ class NovaBasicDeployment(OpenStackAmuletDeployment): self.keystone_sentry, openstack_release=self._get_openstack_release()) + force_v1_client = False + if self._get_openstack_release() == self.trusty_icehouse: + # Updating image properties (such as arch or hypervisor) using the + # v2 api in icehouse results in: + # https://bugs.launchpad.net/python-glanceclient/+bug/1371559 + u.log.debug('Forcing glance to use v1 api') + force_v1_client = True + # Authenticate admin with glance endpoint - self.glance = u.authenticate_glance_admin(self.keystone) + self.glance = u.authenticate_glance_admin( + self.keystone, + force_v1_client=force_v1_client) # Authenticate admin with nova endpoint self.nova = nova_client.Client(2, session=self.keystone_session) diff --git a/unit_tests/test_nova_compute_contexts.py b/unit_tests/test_nova_compute_contexts.py index af4c2c83..803111ef 100644 --- a/unit_tests/test_nova_compute_contexts.py +++ b/unit_tests/test_nova_compute_contexts.py @@ -144,16 +144,6 @@ class NovaComputeContextTests(CharmTestCase): 'ec2_dmz_host': 'novaapihost', 'flat_interface': 'eth1' }, - 'service_protocol': None, - 'service_host': None, - 'service_port': None, - 'admin_tenant_name': None, - 'admin_user': None, - 'admin_password': None, - 'auth_protocol': None, - 'auth_host': None, - 'auth_port': None, - 'api_version': None, } self.assertEqual(ex_ctxt, cloud_compute())