From f0f70a942bc76e98c74894bdfd9015865c2c7d44 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 4 Dec 2017 16:32:36 +0000 Subject: [PATCH] Use volumev3 for internal endpoint in >= Pike volumev2 api is deprecated for >= Pike so select volumev3 for internal endpoint when using >= Pike. Also moved cinder_catalog_info config to [glance_store] section which should have been done a long time ago. Change-Id: Ia9c002199d939ba20c029b70d2eb3c6b6cdc51a6 Partial-Bug: #1733566 --- .../contrib/openstack/amulet/deployment.py | 12 ++++++++---- charmhelpers/contrib/openstack/amulet/utils.py | 9 ++++++--- charmhelpers/contrib/openstack/context.py | 14 +++++++++++++- charmhelpers/contrib/openstack/utils.py | 2 ++ charmhelpers/contrib/storage/linux/ceph.py | 8 ++++---- charmhelpers/core/host.py | 2 ++ charmhelpers/core/host_factory/ubuntu.py | 1 + hooks/glance_utils.py | 2 +- templates/mitaka/glance-api.conf | 7 +++---- .../contrib/openstack/amulet/deployment.py | 12 ++++++++---- .../charmhelpers/contrib/openstack/amulet/utils.py | 9 ++++++--- tests/charmhelpers/core/host.py | 2 ++ tests/charmhelpers/core/host_factory/ubuntu.py | 1 + 13 files changed, 57 insertions(+), 24 deletions(-) diff --git a/charmhelpers/contrib/openstack/amulet/deployment.py b/charmhelpers/contrib/openstack/amulet/deployment.py index e37f2834..5afbbd87 100644 --- a/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/charmhelpers/contrib/openstack/amulet/deployment.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +import os import re import sys import six @@ -185,7 +186,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.d.configure(service, config) def _auto_wait_for_status(self, message=None, exclude_services=None, - include_only=None, timeout=1800): + include_only=None, timeout=None): """Wait for all units to have a specific extended status, except for any defined as excluded. Unless specified via message, any status containing any case of 'ready' will be considered a match. @@ -215,7 +216,10 @@ class OpenStackAmuletDeployment(AmuletDeployment): :param timeout: Maximum time in seconds to wait for status match :returns: None. Raises if timeout is hit. """ - self.log.info('Waiting for extended status on units...') + if not timeout: + timeout = int(os.environ.get('AMULET_SETUP_TIMEOUT', 1800)) + self.log.info('Waiting for extended status on units for {}s...' + ''.format(timeout)) all_services = self.d.services.keys() @@ -252,9 +256,9 @@ class OpenStackAmuletDeployment(AmuletDeployment): service_messages = {service: message for service in services} # Check for idleness - self.d.sentry.wait() + self.d.sentry.wait(timeout=timeout) # Check for error states and bail early - self.d.sentry.wait_for_status(self.d.juju_env, services) + self.d.sentry.wait_for_status(self.d.juju_env, services, timeout=timeout) # Check for ready messages self.d.sentry.wait_for_messages(service_messages, timeout=timeout) diff --git a/charmhelpers/contrib/openstack/amulet/utils.py b/charmhelpers/contrib/openstack/amulet/utils.py index b71b2b19..87f364d1 100644 --- a/charmhelpers/contrib/openstack/amulet/utils.py +++ b/charmhelpers/contrib/openstack/amulet/utils.py @@ -858,9 +858,12 @@ class OpenStackAmuletUtils(AmuletUtils): :returns: List of pool name, object count, kb disk space used """ df = self.get_ceph_df(sentry_unit) - pool_name = df['pools'][pool_id]['name'] - obj_count = df['pools'][pool_id]['stats']['objects'] - kb_used = df['pools'][pool_id]['stats']['kb_used'] + for pool in df['pools']: + if pool['id'] == pool_id: + pool_name = pool['name'] + obj_count = pool['stats']['objects'] + kb_used = pool['stats']['kb_used'] + self.log.debug('Ceph {} pool (ID {}): {} objects, ' '{} kb used'.format(pool_name, pool_id, obj_count, kb_used)) diff --git a/charmhelpers/contrib/openstack/context.py b/charmhelpers/contrib/openstack/context.py index e6c0e9fe..70850c1b 100644 --- a/charmhelpers/contrib/openstack/context.py +++ b/charmhelpers/contrib/openstack/context.py @@ -101,6 +101,8 @@ from charmhelpers.contrib.openstack.utils import ( git_determine_python_path, enable_memcache, snap_install_requested, + CompareOpenStackReleases, + os_release, ) from charmhelpers.core.unitdata import kv @@ -1566,8 +1568,18 @@ class InternalEndpointContext(OSContextGenerator): endpoints by default so this allows admins to optionally use internal endpoints. """ + def __init__(self, ost_rel_check_pkg_name): + self.ost_rel_check_pkg_name = ost_rel_check_pkg_name + def __call__(self): - return {'use_internal_endpoints': config('use-internal-endpoints')} + ctxt = {'use_internal_endpoints': config('use-internal-endpoints')} + rel = os_release(self.ost_rel_check_pkg_name, base='icehouse') + if CompareOpenStackReleases(rel) >= 'pike': + ctxt['volume_api_version'] = '3' + else: + ctxt['volume_api_version'] = '2' + + return ctxt class AppArmorContext(OSContextGenerator): diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index 8a541d40..9e5af342 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -392,6 +392,8 @@ def get_swift_codename(version): releases = UBUNTU_OPENSTACK_RELEASE release = [k for k, v in six.iteritems(releases) if codename in v] ret = subprocess.check_output(['apt-cache', 'policy', 'swift']) + if six.PY3: + ret = ret.decode('UTF-8') if codename in ret or release[0] in ret: return codename elif len(codenames) == 1: diff --git a/charmhelpers/contrib/storage/linux/ceph.py b/charmhelpers/contrib/storage/linux/ceph.py index 39231612..0d9bacfd 100644 --- a/charmhelpers/contrib/storage/linux/ceph.py +++ b/charmhelpers/contrib/storage/linux/ceph.py @@ -377,12 +377,12 @@ def get_mon_map(service): try: return json.loads(mon_status) except ValueError as v: - log("Unable to parse mon_status json: {}. Error: {}".format( - mon_status, v.message)) + log("Unable to parse mon_status json: {}. Error: {}" + .format(mon_status, str(v))) raise except CalledProcessError as e: - log("mon_status command failed with message: {}".format( - e.message)) + log("mon_status command failed with message: {}" + .format(str(e))) raise diff --git a/charmhelpers/core/host.py b/charmhelpers/core/host.py index 5cc5c86b..fd14d60f 100644 --- a/charmhelpers/core/host.py +++ b/charmhelpers/core/host.py @@ -549,6 +549,8 @@ def write_file(path, content, owner='root', group='root', perms=0o444): with open(path, 'wb') as target: os.fchown(target.fileno(), uid, gid) os.fchmod(target.fileno(), perms) + if six.PY3 and isinstance(content, six.string_types): + content = content.encode('UTF-8') target.write(content) return # the contents were the same, but we might still need to change the diff --git a/charmhelpers/core/host_factory/ubuntu.py b/charmhelpers/core/host_factory/ubuntu.py index d8dc378a..99451b59 100644 --- a/charmhelpers/core/host_factory/ubuntu.py +++ b/charmhelpers/core/host_factory/ubuntu.py @@ -20,6 +20,7 @@ UBUNTU_RELEASES = ( 'yakkety', 'zesty', 'artful', + 'bionic', ) diff --git a/hooks/glance_utils.py b/hooks/glance_utils.py index 66b78b76..7506c4ce 100644 --- a/hooks/glance_utils.py +++ b/hooks/glance_utils.py @@ -201,7 +201,7 @@ CONFIG_FILES = OrderedDict([ context.OSConfigFlagContext( charm_flag='api-config-flags', template_flag='api_config_flags'), - context.InternalEndpointContext(), + context.InternalEndpointContext('glance-common'), context.SubordinateConfigContext( interface=['storage-backend'], service=['glance-api'], diff --git a/templates/mitaka/glance-api.conf b/templates/mitaka/glance-api.conf index 9bee3908..c9539a34 100644 --- a/templates/mitaka/glance-api.conf +++ b/templates/mitaka/glance-api.conf @@ -5,10 +5,6 @@ debug = {{ debug }} workers = {{ workers }} bind_host = {{ bind_host }} -{% if use_internal_endpoints -%} -cinder_catalog_info = volumev2::internalURL -{% endif %} - {% if ext -%} bind_port = {{ ext }} {% elif bind_port -%} @@ -42,6 +38,9 @@ image_cache_dir = /var/lib/glance/image-cache/ db_enforce_mysql_charset = False [glance_store] +{%- if use_internal_endpoints %} +catalog_info = volumev{{volume_api_version}}:cinderv{{volume_api_version}}:internalURL +{%- endif %} filesystem_store_datadir = /var/lib/glance/images/ stores = {{ known_stores }} {% if rbd_pool -%} diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index e37f2834..5afbbd87 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +import os import re import sys import six @@ -185,7 +186,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.d.configure(service, config) def _auto_wait_for_status(self, message=None, exclude_services=None, - include_only=None, timeout=1800): + include_only=None, timeout=None): """Wait for all units to have a specific extended status, except for any defined as excluded. Unless specified via message, any status containing any case of 'ready' will be considered a match. @@ -215,7 +216,10 @@ class OpenStackAmuletDeployment(AmuletDeployment): :param timeout: Maximum time in seconds to wait for status match :returns: None. Raises if timeout is hit. """ - self.log.info('Waiting for extended status on units...') + if not timeout: + timeout = int(os.environ.get('AMULET_SETUP_TIMEOUT', 1800)) + self.log.info('Waiting for extended status on units for {}s...' + ''.format(timeout)) all_services = self.d.services.keys() @@ -252,9 +256,9 @@ class OpenStackAmuletDeployment(AmuletDeployment): service_messages = {service: message for service in services} # Check for idleness - self.d.sentry.wait() + self.d.sentry.wait(timeout=timeout) # Check for error states and bail early - self.d.sentry.wait_for_status(self.d.juju_env, services) + self.d.sentry.wait_for_status(self.d.juju_env, services, timeout=timeout) # Check for ready messages self.d.sentry.wait_for_messages(service_messages, timeout=timeout) diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index b71b2b19..87f364d1 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -858,9 +858,12 @@ class OpenStackAmuletUtils(AmuletUtils): :returns: List of pool name, object count, kb disk space used """ df = self.get_ceph_df(sentry_unit) - pool_name = df['pools'][pool_id]['name'] - obj_count = df['pools'][pool_id]['stats']['objects'] - kb_used = df['pools'][pool_id]['stats']['kb_used'] + for pool in df['pools']: + if pool['id'] == pool_id: + pool_name = pool['name'] + obj_count = pool['stats']['objects'] + kb_used = pool['stats']['kb_used'] + self.log.debug('Ceph {} pool (ID {}): {} objects, ' '{} kb used'.format(pool_name, pool_id, obj_count, kb_used)) diff --git a/tests/charmhelpers/core/host.py b/tests/charmhelpers/core/host.py index 5cc5c86b..fd14d60f 100644 --- a/tests/charmhelpers/core/host.py +++ b/tests/charmhelpers/core/host.py @@ -549,6 +549,8 @@ def write_file(path, content, owner='root', group='root', perms=0o444): with open(path, 'wb') as target: os.fchown(target.fileno(), uid, gid) os.fchmod(target.fileno(), perms) + if six.PY3 and isinstance(content, six.string_types): + content = content.encode('UTF-8') target.write(content) return # the contents were the same, but we might still need to change the diff --git a/tests/charmhelpers/core/host_factory/ubuntu.py b/tests/charmhelpers/core/host_factory/ubuntu.py index d8dc378a..99451b59 100644 --- a/tests/charmhelpers/core/host_factory/ubuntu.py +++ b/tests/charmhelpers/core/host_factory/ubuntu.py @@ -20,6 +20,7 @@ UBUNTU_RELEASES = ( 'yakkety', 'zesty', 'artful', + 'bionic', )