openstack
/
charm-glance
Code Issues Proposed changes

V3 authtoken update and glance v1 icehouse 

Update the keystone authtoken template section to explicitly
specify v3 API when it is in use. This prevents errors that
result in "Could not find versioned identity endpoints when
attempting to authenticate".

Along with this change we remove the tests that verify the
keystone_authtoken section as it's generally agreed that
functional testing should catch issues with this config.
Also update requirements to fix pyyaml dep of charm-tools.

Finally get glance v1 client for icehouse. Previously the
image virt type was qemu and the compute node virt type was
kvm. This works for deployments prior to rocky but in rocky
this causes the image type filter to return no valid hosts.
An update to charmhelpers has removed the default behaviour
of setting the virt type to 'qemu' by default. Due to a bug
in icehouse updating glance image properties using the v2 api
fails (See Bug #1371559) so for icehouse deploys get a v1
client.

Change-Id: I4ca604c674bda5d5f7daca6a3e9d13c8b4bd4efa
Closes-Bug: #1794637
This commit is contained in:
Corey Bryant 2018-09-27 12:46:49 +00:00
parent 0720f3bc3a
commit 981b86a3a1
10 changed files with 2807 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
charmhelpers/contrib/hahelpers/apache.py
View File

@ -23,8 +23,8 @@
#
import os
import subprocess
from charmhelpers.core import host
from charmhelpers.core.hookenv import (
config as config_get,
relation_get,
@ -83,14 +83,4 @@ def retrieve_ca_cert(cert_file):
def install_ca_cert(ca_cert):
if ca_cert:
cert_file = ('/usr/local/share/ca-certificates/'
'keystone_juju_ca_cert.crt')
old_cert = retrieve_ca_cert(cert_file)
if old_cert and old_cert == ca_cert:
log("CA cert is the same as installed version", level=INFO)
else:
log("Installing new CA cert", level=INFO)
with open(cert_file, 'wb') as crt:
crt.write(ca_cert)
subprocess.check_call(['update-ca-certificates', '--fresh'])
host.install_ca_cert(ca_cert, 'keystone_juju_ca_cert')

3
charmhelpers/contrib/hardening/apache/checks/config.py
View File

@ -14,6 +14,7 @@
import os
import re
import six
import subprocess
@ -95,6 +96,8 @@ class ApacheConfContext(object):
ctxt = settings['hardening']
out = subprocess.check_output(['apache2', '-v'])
if six.PY3:
out = out.decode('utf-8')
ctxt['apache_version'] = re.search(r'.+version: Apache/(.+?)\s.+',
out).group(1)
ctxt['apache_icondir'] = '/usr/share/apache2/icons/'

6
charmhelpers/contrib/hardening/audits/apache.py
View File

@ -15,7 +15,7 @@
import re
import subprocess
from six import string_types
import six
from charmhelpers.core.hookenv import (
log,
@ -35,7 +35,7 @@ class DisabledModuleAudit(BaseAudit):
def __init__(self, modules):
if modules is None:
self.modules = []
elif isinstance(modules, string_types):
elif isinstance(modules, six.string_types):
self.modules = [modules]
else:
self.modules = modules
@ -69,6 +69,8 @@ class DisabledModuleAudit(BaseAudit):
def _get_loaded_modules():
"""Returns the modules which are enabled in Apache."""
output = subprocess.check_output(['apache2ctl', '-M'])
if six.PY3:
output = output.decode('utf-8')
modules = []
for line in output.splitlines():
# Each line of the enabled module output looks like:

114
charmhelpers/contrib/openstack/amulet/utils.py
View File

@ -618,12 +618,12 @@ class OpenStackAmuletUtils(AmuletUtils):
return self.authenticate_keystone(keystone_ip, user, password,
project_name=tenant)
def authenticate_glance_admin(self, keystone):
def authenticate_glance_admin(self, keystone, force_v1_client=False):
"""Authenticates admin user with glance."""
self.log.debug('Authenticating glance admin...')
ep = keystone.service_catalog.url_for(service_type='image',
interface='adminURL')
if keystone.session:
if not force_v1_client and keystone.session:
return glance_clientv2.Client("2", session=keystone.session)
else:
return glance_client.Client(ep, token=keystone.auth_token)
@ -680,18 +680,30 @@ class OpenStackAmuletUtils(AmuletUtils):
nova.flavors.create(name, ram, vcpus, disk, flavorid,
ephemeral, swap, rxtx_factor, is_public)
def create_cirros_image(self, glance, image_name):
"""Download the latest cirros image and upload it to glance,
validate and return a resource pointer.
def glance_create_image(self, glance, image_name, image_url,
download_dir='tests',
hypervisor_type=None,
disk_format='qcow2',
architecture='x86_64',
container_format='bare'):
"""Download an image and upload it to glance, validate its status
and return an image object pointer. KVM defaults, can override for
LXD.
:param glance: pointer to authenticated glance connection
:param glance: pointer to authenticated glance api connection
:param image_name: display name for new image
:param image_url: url to retrieve
:param download_dir: directory to store downloaded image file
:param hypervisor_type: glance image hypervisor property
:param disk_format: glance image disk format
:param architecture: glance image architecture property
:param container_format: glance image container format
:returns: glance image pointer
"""
self.log.debug('Creating glance cirros image '
'({})...'.format(image_name))
self.log.debug('Creating glance image ({}) from '
'{}...'.format(image_name, image_url))
# Download cirros image
# Download image
http_proxy = os.getenv('AMULET_HTTP_PROXY')
self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
if http_proxy:
@ -700,31 +712,34 @@ class OpenStackAmuletUtils(AmuletUtils):
else:
opener = urllib.FancyURLopener()
f = opener.open('http://download.cirros-cloud.net/version/released')
version = f.read().strip()
cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
local_path = os.path.join('tests', cirros_img)
if not os.path.exists(local_path):
cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
version, cirros_img)
opener.retrieve(cirros_url, local_path)
f.close()
abs_file_name = os.path.join(download_dir, image_name)
if not os.path.exists(abs_file_name):
opener.retrieve(image_url, abs_file_name)
# Create glance image
glance_properties = {
'architecture': architecture,
}
if hypervisor_type:
glance_properties['hypervisor_type'] = hypervisor_type
# Create glance image
if float(glance.version) < 2.0:
with open(local_path) as fimage:
image = glance.images.create(name=image_name, is_public=True,
disk_format='qcow2',
container_format='bare',
data=fimage)
with open(abs_file_name) as f:
image = glance.images.create(
name=image_name,
is_public=True,
disk_format=disk_format,
container_format=container_format,
properties=glance_properties,
data=f)
else:
image = glance.images.create(
name=image_name,
disk_format="qcow2",
visibility="public",
container_format="bare")
glance.images.upload(image.id, open(local_path, 'rb'))
disk_format=disk_format,
container_format=container_format)
glance.images.upload(image.id, open(abs_file_name, 'rb'))
glance.images.update(image.id, **glance_properties)
# Wait for image to reach active status
img_id = image.id
@ -753,15 +768,54 @@ class OpenStackAmuletUtils(AmuletUtils):
val_img_stat, val_img_cfmt, val_img_dfmt))
if val_img_name == image_name and val_img_stat == 'active' \
and val_img_pub is True and val_img_cfmt == 'bare' \
and val_img_dfmt == 'qcow2':
and val_img_pub is True and val_img_cfmt == container_format \
and val_img_dfmt == disk_format:
self.log.debug(msg_attr)
else:
msg = ('Volume validation failed, {}'.format(msg_attr))
msg = ('Image validation failed, {}'.format(msg_attr))
amulet.raise_status(amulet.FAIL, msg=msg)
return image
def create_cirros_image(self, glance, image_name, hypervisor_type=None):
"""Download the latest cirros image and upload it to glance,
validate and return a resource pointer.
:param glance: pointer to authenticated glance connection
:param image_name: display name for new image
:param hypervisor_type: glance image hypervisor property
:returns: glance image pointer
"""
# /!\ DEPRECATION WARNING
self.log.warn('/!\\ DEPRECATION WARNING: use '
'glance_create_image instead of '
'create_cirros_image.')
self.log.debug('Creating glance cirros image '
'({})...'.format(image_name))
# Get cirros image URL
http_proxy = os.getenv('AMULET_HTTP_PROXY')
self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
if http_proxy:
proxies = {'http': http_proxy}
opener = urllib.FancyURLopener(proxies)
else:
opener = urllib.FancyURLopener()
f = opener.open('http://download.cirros-cloud.net/version/released')
version = f.read().strip()
cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
version, cirros_img)
f.close()
return self.glance_create_image(
glance,
image_name,
cirros_url,
hypervisor_type=hypervisor_type)
def delete_image(self, glance, image):
"""Delete the specified image."""

26
charmhelpers/contrib/openstack/context.py
View File

@ -1523,10 +1523,6 @@ class NeutronAPIContext(OSContextGenerator):
'rel_key': 'enable-nsg-logging',
'default': False,
},
'nsg_log_output_base': {
'rel_key': 'nsg-log-output-base',
'default': None,
},
}
ctxt = self.get_neutron_options({})
for rid in relation_ids('neutron-plugin-api'):
@ -1901,7 +1897,7 @@ class EnsureDirContext(OSContextGenerator):
Some software requires a user to create a target directory to be
scanned for drop-in files with a specific format. This is why this
context is needed to do that before rendering a template.
'''
'''
def __init__(self, dirname, **kwargs):
'''Used merely to ensure that a given directory exists.'''
@ -1911,3 +1907,23 @@ class EnsureDirContext(OSContextGenerator):
def __call__(self):
mkdir(self.dirname, **self.kwargs)
return {}
class VersionsContext(OSContextGenerator):
"""Context to return the openstack and operating system versions.
"""
def __init__(self, pkg='python-keystone'):
"""Initialise context.
:param pkg: Package to extrapolate openstack version from.
:type pkg: str
"""
self.pkg = pkg
def __call__(self):
ostack = os_release(self.pkg, base='icehouse')
osystem = lsb_release()['DISTRIB_CODENAME'].lower()
return {
'openstack_release': ostack,
'operating_system_release': osystem}

6
charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka
View File

@ -1,12 +1,14 @@
{% if auth_host -%}
[keystone_authtoken]
auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
auth_type = password
{% if api_version == "3" -%}
auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/v3
auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v3
project_domain_name = {{ admin_domain_name }}
user_domain_name = {{ admin_domain_name }}
{% else -%}
auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
project_domain_name = default
user_domain_name = default
{% endif -%}

26
charmhelpers/core/host.py
View File

@ -34,7 +34,7 @@ import six
from contextlib import contextmanager
from collections import OrderedDict
from .hookenv import log, DEBUG, local_unit
from .hookenv import log, INFO, DEBUG, local_unit, charm_name
from .fstab import Fstab
from charmhelpers.osplatform import get_platform
@ -1040,3 +1040,27 @@ def modulo_distribution(modulo=3, wait=30, non_zero_wait=False):
return modulo * wait
else:
return calculated_wait_time
def install_ca_cert(ca_cert, name=None):
"""
Install the given cert as a trusted CA.
The ``name`` is the stem of the filename where the cert is written, and if
not provided, it will default to ``juju-{charm_name}``.
If the cert is empty or None, or is unchanged, nothing is done.
"""
if not ca_cert:
return
if not isinstance(ca_cert, bytes):
ca_cert = ca_cert.encode('utf8')
if not name:
name = 'juju-{}'.format(charm_name())
cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name)
new_hash = hashlib.md5(ca_cert).hexdigest()
if file_hash(cert_file) == new_hash:
return
log("Installing new CA cert at: {}".format(cert_file), level=INFO)
write_file(cert_file, ca_cert)
subprocess.check_call(['update-ca-certificates', '--fresh'])

97
tests/basic_deployment.py
View File

@ -125,8 +125,18 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment):
self.keystone_sentry,
openstack_release=self._get_openstack_release())
force_v1_client = False
if self._get_openstack_release() == self.trusty_icehouse:
# Updating image properties (such as arch or hypervisor) using the
# v2 api in icehouse results in:
# https://bugs.launchpad.net/python-glanceclient/+bug/1371559
u.log.debug('Forcing glance to use v1 api')
force_v1_client = True
# Authenticate admin with glance endpoint
self.glance = u.authenticate_glance_admin(self.keystone)
self.glance = u.authenticate_glance_admin(
self.keystone,
force_v1_client=force_v1_client)
def test_100_services(self):
"""Verify that the expected services are running on the
@ -361,90 +371,12 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment):
message = u.relation_error('glance amqp', ret)
amulet.raise_status(amulet.FAIL, msg=message)
def _get_keystone_authtoken_expected_dict(self, rel_ks_gl):
"""Return expected authtoken dict for OS release"""
auth_uri = ('http://%s:%s/' %
(rel_ks_gl['auth_host'], rel_ks_gl['service_port']))
auth_url = ('http://%s:%s/' %
(rel_ks_gl['auth_host'], rel_ks_gl['auth_port']))
if self._get_openstack_release() >= self.xenial_queens:
expected = {
'keystone_authtoken': {
'auth_uri': auth_uri.rstrip('/'),
'auth_url': auth_url.rstrip('/'),
'auth_type': 'password',
'project_domain_name': 'service_domain',
'user_domain_name': 'service_domain',
'project_name': 'services',
'username': rel_ks_gl['service_username'],
'password': rel_ks_gl['service_password'],
'signing_dir': '/var/cache/glance'
}
}
elif self._get_openstack_release() >= self.trusty_mitaka:
expected = {
'keystone_authtoken': {
'auth_uri': auth_uri.rstrip('/'),
'auth_url': auth_url.rstrip('/'),
'auth_type': 'password',
'project_domain_name': 'default',
'user_domain_name': 'default',
'project_name': 'services',
'username': rel_ks_gl['service_username'],
'password': rel_ks_gl['service_password'],
'signing_dir': '/var/cache/glance'
}
}
elif self._get_openstack_release() >= self.trusty_liberty:
expected = {
'keystone_authtoken': {
'auth_uri': auth_uri.rstrip('/'),
'auth_url': auth_url.rstrip('/'),
'auth_plugin': 'password',
'project_domain_id': 'default',
'user_domain_id': 'default',
'project_name': 'services',
'username': rel_ks_gl['service_username'],
'password': rel_ks_gl['service_password'],
'signing_dir': '/var/cache/glance'
}
}
elif self._get_openstack_release() >= self.trusty_kilo:
expected = {
'keystone_authtoken': {
'project_name': 'services',
'username': 'glance',
'password': rel_ks_gl['service_password'],
'auth_uri': u.valid_url,
'auth_url': u.valid_url,
'signing_dir': '/var/cache/glance',
}
}
else:
expected = {
'keystone_authtoken': {
'auth_uri': u.valid_url,
'auth_host': rel_ks_gl['auth_host'],
'auth_port': rel_ks_gl['auth_port'],
'auth_protocol': rel_ks_gl['auth_protocol'],
'admin_tenant_name': 'services',
'admin_user': 'glance',
'admin_password': rel_ks_gl['service_password'],
'signing_dir': '/var/cache/glance',
}
}
return expected
def test_300_glance_api_default_config(self):
"""Verify default section configs in glance-api.conf and
compare some of the parameters to relation data."""
u.log.debug('Checking glance api config file...')
unit = self.glance_sentry
unit_ks = self.keystone_sentry
rel_mq_gl = self.rabbitmq_sentry.relation('amqp', 'glance:amqp')
rel_ks_gl = unit_ks.relation('identity-service',
'glance:identity-service')
rel_my_gl = self.pxc_sentry.relation('shared-db', 'glance:shared-db')
db_uri = "mysql://{}:{}@{}/{}".format('glance', rel_my_gl['password'],
rel_my_gl['db_host'], 'glance')
@ -469,8 +401,6 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment):
},
}
expected.update(self._get_keystone_authtoken_expected_dict(rel_ks_gl))
if self._get_openstack_release() >= self.trusty_kilo:
# Kilo or later
expected['oslo_messaging_rabbit'] = {
@ -529,9 +459,6 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment):
"""Verify configs in glance-registry.conf"""
u.log.debug('Checking glance registry config file...')
unit = self.glance_sentry
unit_ks = self.keystone_sentry
rel_ks_gl = unit_ks.relation('identity-service',
'glance:identity-service')
rel_my_gl = self.pxc_sentry.relation('shared-db', 'glance:shared-db')
db_uri = "mysql://{}:{}@{}/{}".format('glance', rel_my_gl['password'],
rel_my_gl['db_host'], 'glance')
@ -561,8 +488,6 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment):
'connection': db_uri
}
expected.update(self._get_keystone_authtoken_expected_dict(rel_ks_gl))
for section, pairs in expected.iteritems():
ret = u.validate_config_data(unit, conf, section, pairs)
if ret:

1587
tests/charmhelpers/contrib/openstack/amulet/utils.py Normal file
View File

File diff suppressed because it is too large Load Diff

1066
tests/charmhelpers/core/host.py Normal file
View File

File diff suppressed because it is too large Load Diff