From 527261db5cb0c81de395cd40071b58ecdd4596b2 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 10 Jun 2022 20:14:47 +0000 Subject: [PATCH] Add Kinetic and Zed support * sync charm-helpers to classic charms * change openstack-origin/source default to zed * align testing with zed * add new zed bundles * add zed bundles to tests.yaml * add zed tests to osci.yaml and .zuul.yaml * update build-on and run-on bases * add bindep.txt for py310 * sync tox.ini and requirements.txt for ruamel * use charmcraft_channel 2.0/stable * drop reactive plugin overrides * move interface/layer env vars to charmcraft.yaml Change-Id: I1a250a0591c15971840bbd6130c7cff3822eed4f --- .zuul.yaml | 2 +- bindep.txt | 3 + charmcraft.yaml | 5 +- charmhelpers/__init__.py | 17 +-- charmhelpers/cli/__init__.py | 13 +- charmhelpers/contrib/charmsupport/nrpe.py | 40 +++++- charmhelpers/contrib/hahelpers/cluster.py | 15 +- charmhelpers/contrib/network/ip.py | 25 +--- .../files/check_deferred_restarts.py | 128 ++++++++++++++++++ charmhelpers/contrib/openstack/policyd.py | 46 +------ charmhelpers/contrib/openstack/utils.py | 116 ++++++++++------ charmhelpers/contrib/python.py | 2 - charmhelpers/contrib/storage/linux/ceph.py | 99 +++++++------- .../contrib/storage/linux/loopback.py | 10 +- charmhelpers/contrib/templating/contexts.py | 12 +- charmhelpers/contrib/templating/jinja.py | 8 +- charmhelpers/core/hookenv.py | 81 ++++++----- charmhelpers/core/host.py | 47 +++++-- charmhelpers/core/host_factory/ubuntu.py | 1 + charmhelpers/core/services/base.py | 10 +- charmhelpers/core/services/helpers.py | 4 +- charmhelpers/core/strutils.py | 9 +- charmhelpers/core/templating.py | 11 +- charmhelpers/fetch/__init__.py | 10 +- charmhelpers/fetch/archiveurl.py | 60 ++++---- charmhelpers/fetch/centos.py | 7 +- charmhelpers/fetch/python/debug.py | 2 - charmhelpers/fetch/python/packages.py | 14 +- charmhelpers/fetch/ubuntu.py | 47 +++---- charmhelpers/fetch/ubuntu_apt_pkg.py | 25 +++- metadata.yaml | 1 - osci.yaml | 4 +- requirements.txt | 11 +- test-requirements.txt | 17 +-- .../{focal-yoga.yaml => jammy-zed.yaml} | 4 +- tests/bundles/kinetic-zed.yaml | 57 ++++++++ tests/tests.yaml | 8 +- tox.ini | 29 +--- 38 files changed, 587 insertions(+), 413 deletions(-) create mode 100644 bindep.txt create mode 100755 charmhelpers/contrib/openstack/files/check_deferred_restarts.py rename tests/bundles/{focal-yoga.yaml => jammy-zed.yaml} (93%) create mode 100644 tests/bundles/kinetic-zed.yaml diff --git a/.zuul.yaml b/.zuul.yaml index 7ffc71c..23bf5f6 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,4 +1,4 @@ - project: templates: - - openstack-python3-charm-yoga-jobs + - openstack-python3-charm-zed-jobs - openstack-cover-jobs diff --git a/bindep.txt b/bindep.txt new file mode 100644 index 0000000..bdbe8d5 --- /dev/null +++ b/bindep.txt @@ -0,0 +1,3 @@ +libffi-dev [platform:dpkg] +libxml2-dev [platform:dpkg] +libxslt1-dev [platform:dpkg] diff --git a/charmcraft.yaml b/charmcraft.yaml index ea1d266..a89bb84 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -23,13 +23,10 @@ parts: bases: - build-on: - name: ubuntu - channel: "20.04" + channel: "22.04" architectures: - amd64 run-on: - - name: ubuntu - channel: "20.04" - architectures: [amd64, s390x, ppc64el, arm64] - name: ubuntu channel: "22.04" architectures: [amd64, s390x, ppc64el, arm64] diff --git a/charmhelpers/__init__.py b/charmhelpers/__init__.py index 1f57ed2..ddf3045 100644 --- a/charmhelpers/__init__.py +++ b/charmhelpers/__init__.py @@ -14,30 +14,15 @@ # Bootstrap charm-helpers, installing its dependencies if necessary using # only standard libraries. -from __future__ import print_function -from __future__ import absolute_import - import functools import inspect import subprocess -import sys -try: - import six # NOQA:F401 -except ImportError: - if sys.version_info.major == 2: - subprocess.check_call(['apt-get', 'install', '-y', 'python-six']) - else: - subprocess.check_call(['apt-get', 'install', '-y', 'python3-six']) - import six # NOQA:F401 try: import yaml # NOQA:F401 except ImportError: - if sys.version_info.major == 2: - subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml']) - else: - subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml']) + subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml']) import yaml # NOQA:F401 diff --git a/charmhelpers/cli/__init__.py b/charmhelpers/cli/__init__.py index 74ea729..2b0c4b7 100644 --- a/charmhelpers/cli/__init__.py +++ b/charmhelpers/cli/__init__.py @@ -16,9 +16,6 @@ import inspect import argparse import sys -import six -from six.moves import zip - import charmhelpers.core.unitdata @@ -149,10 +146,7 @@ class CommandLine(object): def run(self): "Run cli, processing arguments and executing subcommands." arguments = self.argument_parser.parse_args() - if six.PY2: - argspec = inspect.getargspec(arguments.func) - else: - argspec = inspect.getfullargspec(arguments.func) + argspec = inspect.getfullargspec(arguments.func) vargs = [] for arg in argspec.args: vargs.append(getattr(arguments, arg)) @@ -177,10 +171,7 @@ def describe_arguments(func): Analyze a function's signature and return a data structure suitable for passing in as arguments to an argparse parser's add_argument() method.""" - if six.PY2: - argspec = inspect.getargspec(func) - else: - argspec = inspect.getfullargspec(func) + argspec = inspect.getfullargspec(func) # we should probably raise an exception somewhere if func includes **kwargs if argspec.defaults: positional_args = argspec.args[:-len(argspec.defaults)] diff --git a/charmhelpers/contrib/charmsupport/nrpe.py b/charmhelpers/contrib/charmsupport/nrpe.py index 8d1753c..bad7a53 100644 --- a/charmhelpers/contrib/charmsupport/nrpe.py +++ b/charmhelpers/contrib/charmsupport/nrpe.py @@ -28,6 +28,7 @@ import subprocess import yaml from charmhelpers.core.hookenv import ( + application_name, config, hook_name, local_unit, @@ -174,7 +175,8 @@ define service {{ if os.path.exists(os.path.join(path, parts[0])): command = os.path.join(path, parts[0]) if len(parts) > 1: - command += " " + " ".join(parts[1:]) + safe_args = [shlex.quote(arg) for arg in parts[1:]] + command += " " + " ".join(safe_args) return command log('Check command not found: {}'.format(parts[0])) return '' @@ -520,3 +522,39 @@ def remove_deprecated_check(nrpe, deprecated_services): for dep_svc in deprecated_services: log('Deprecated service: {}'.format(dep_svc)) nrpe.remove_check(shortname=dep_svc) + + +def add_deferred_restarts_check(nrpe): + """ + Add NRPE check for services with deferred restarts. + + :param NRPE nrpe: NRPE object to add check to + """ + unit_name = local_unit().replace('/', '-') + shortname = unit_name + '_deferred_restarts' + check_cmd = 'check_deferred_restarts.py --application {}'.format( + application_name()) + + log('Adding deferred restarts nrpe check: {}'.format(shortname)) + nrpe.add_check( + shortname=shortname, + description='Check deferred service restarts {}'.format(unit_name), + check_cmd=check_cmd) + + +def remove_deferred_restarts_check(nrpe): + """ + Remove NRPE check for services with deferred service restarts. + + :param NRPE nrpe: NRPE object to remove check from + """ + unit_name = local_unit().replace('/', '-') + shortname = unit_name + '_deferred_restarts' + check_cmd = 'check_deferred_restarts.py --application {}'.format( + application_name()) + + log('Removing deferred restarts nrpe check: {}'.format(shortname)) + nrpe.remove_check( + shortname=shortname, + description='Check deferred service restarts {}'.format(unit_name), + check_cmd=check_cmd) diff --git a/charmhelpers/contrib/hahelpers/cluster.py b/charmhelpers/contrib/hahelpers/cluster.py index f0b629a..146beba 100644 --- a/charmhelpers/contrib/hahelpers/cluster.py +++ b/charmhelpers/contrib/hahelpers/cluster.py @@ -32,8 +32,6 @@ import time from socket import gethostname as get_unit_hostname -import six - from charmhelpers.core.hookenv import ( log, relation_ids, @@ -125,16 +123,16 @@ def is_crm_dc(): """ cmd = ['crm', 'status'] try: - status = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - if not isinstance(status, six.text_type): - status = six.text_type(status, "utf-8") + status = subprocess.check_output( + cmd, stderr=subprocess.STDOUT).decode('utf-8') except subprocess.CalledProcessError as ex: raise CRMDCNotFound(str(ex)) current_dc = '' for line in status.split('\n'): if line.startswith('Current DC'): - # Current DC: juju-lytrusty-machine-2 (168108163) - partition with quorum + # Current DC: juju-lytrusty-machine-2 (168108163) + # - partition with quorum current_dc = line.split(':')[1].split()[0] if current_dc == get_unit_hostname(): return True @@ -158,9 +156,8 @@ def is_crm_leader(resource, retry=False): return is_crm_dc() cmd = ['crm', 'resource', 'show', resource] try: - status = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - if not isinstance(status, six.text_type): - status = six.text_type(status, "utf-8") + status = subprocess.check_output( + cmd, stderr=subprocess.STDOUT).decode('utf-8') except subprocess.CalledProcessError: status = None diff --git a/charmhelpers/contrib/network/ip.py b/charmhelpers/contrib/network/ip.py index b356d64..f8edf37 100644 --- a/charmhelpers/contrib/network/ip.py +++ b/charmhelpers/contrib/network/ip.py @@ -15,7 +15,6 @@ import glob import re import subprocess -import six import socket from functools import partial @@ -39,20 +38,14 @@ try: import netifaces except ImportError: apt_update(fatal=True) - if six.PY2: - apt_install('python-netifaces', fatal=True) - else: - apt_install('python3-netifaces', fatal=True) + apt_install('python3-netifaces', fatal=True) import netifaces try: import netaddr except ImportError: apt_update(fatal=True) - if six.PY2: - apt_install('python-netaddr', fatal=True) - else: - apt_install('python3-netaddr', fatal=True) + apt_install('python3-netaddr', fatal=True) import netaddr @@ -462,22 +455,19 @@ def ns_query(address): try: import dns.resolver except ImportError: - if six.PY2: - apt_install('python-dnspython', fatal=True) - else: - apt_install('python3-dnspython', fatal=True) + apt_install('python3-dnspython', fatal=True) import dns.resolver if isinstance(address, dns.name.Name): rtype = 'PTR' - elif isinstance(address, six.string_types): + elif isinstance(address, str): rtype = 'A' else: return None try: answers = dns.resolver.query(address, rtype) - except dns.resolver.NXDOMAIN: + except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers): return None if answers: @@ -513,10 +503,7 @@ def get_hostname(address, fqdn=True): try: import dns.reversename except ImportError: - if six.PY2: - apt_install("python-dnspython", fatal=True) - else: - apt_install("python3-dnspython", fatal=True) + apt_install("python3-dnspython", fatal=True) import dns.reversename rev = dns.reversename.from_address(address) diff --git a/charmhelpers/contrib/openstack/files/check_deferred_restarts.py b/charmhelpers/contrib/openstack/files/check_deferred_restarts.py new file mode 100755 index 0000000..5f392b3 --- /dev/null +++ b/charmhelpers/contrib/openstack/files/check_deferred_restarts.py @@ -0,0 +1,128 @@ +#!/usr/bin/python3 + +# Copyright 2014-2022 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Checks for services with deferred restarts. + +This Nagios check will parse /var/lib/policy-rd.d/ +to find any restarts that are currently deferred. +""" + +import argparse +import glob +import sys +import yaml + + +DEFERRED_EVENTS_DIR = '/var/lib/policy-rc.d' + + +def get_deferred_events(): + """Return a list of deferred events dicts from policy-rc.d files. + + Events are read from DEFERRED_EVENTS_DIR and are of the form: + { + action: restart, + policy_requestor_name: rabbitmq-server, + policy_requestor_type: charm, + reason: 'Pkg update', + service: rabbitmq-server, + time: 1614328743 + } + + :raises OSError: Raised in case of a system error while reading a policy file + :raises yaml.YAMLError: Raised if parsing a policy file fails + + :returns: List of deferred event dictionaries + :rtype: list + """ + deferred_events_files = glob.glob( + '{}/*.deferred'.format(DEFERRED_EVENTS_DIR)) + + deferred_events = [] + for event_file in deferred_events_files: + with open(event_file, 'r') as f: + event = yaml.safe_load(f) + deferred_events.append(event) + + return deferred_events + + +def get_deferred_restart_services(application=None): + """Returns a list of services with deferred restarts. + + :param str application: Name of the application that blocked the service restart. + If application is None, all services with deferred restarts + are returned. Services which are blocked by a non-charm + requestor are always returned. + + :raises OSError: Raised in case of a system error while reading a policy file + :raises yaml.YAMLError: Raised if parsing a policy file fails + + :returns: List of services with deferred restarts belonging to application. + :rtype: list + """ + + deferred_restart_events = filter( + lambda e: e['action'] == 'restart', get_deferred_events()) + + deferred_restart_services = set() + for restart_event in deferred_restart_events: + if application: + if ( + restart_event['policy_requestor_type'] != 'charm' or + restart_event['policy_requestor_type'] == 'charm' and + restart_event['policy_requestor_name'] == application + ): + deferred_restart_services.add(restart_event['service']) + else: + deferred_restart_services.add(restart_event['service']) + + return list(deferred_restart_services) + + +def main(): + """Check for services with deferred restarts.""" + parser = argparse.ArgumentParser( + description='Check for services with deferred restarts') + parser.add_argument( + '--application', help='Check services belonging to this application only') + + args = parser.parse_args() + + services = set(get_deferred_restart_services(args.application)) + + if len(services) == 0: + print('OK: No deferred service restarts.') + sys.exit(0) + else: + print( + 'CRITICAL: Restarts are deferred for services: {}.'.format(', '.join(services))) + sys.exit(1) + + +if __name__ == '__main__': + try: + main() + except OSError as e: + print('CRITICAL: A system error occurred: {} ({})'.format(e.errno, e.strerror)) + sys.exit(1) + except yaml.YAMLError as e: + print('CRITICAL: Failed to parse a policy file: {}'.format(str(e))) + sys.exit(1) + except Exception as e: + print('CRITICAL: An unknown error occurred: {}'.format(str(e))) + sys.exit(1) diff --git a/charmhelpers/contrib/openstack/policyd.py b/charmhelpers/contrib/openstack/policyd.py index 6fa06f2..767943c 100644 --- a/charmhelpers/contrib/openstack/policyd.py +++ b/charmhelpers/contrib/openstack/policyd.py @@ -15,7 +15,6 @@ import collections import contextlib import os -import six import shutil import yaml import zipfile @@ -204,12 +203,6 @@ class BadPolicyYamlFile(Exception): return self.log_message -if six.PY2: - BadZipFile = zipfile.BadZipfile -else: - BadZipFile = zipfile.BadZipFile - - def is_policyd_override_valid_on_this_release(openstack_release): """Check that the charm is running on at least Ubuntu Xenial, and at least the queens release. @@ -487,10 +480,10 @@ def read_and_validate_yaml(stream_or_doc, blacklist_keys=None): if blacklisted_keys_present: raise BadPolicyYamlFile("blacklisted keys {} present." .format(", ".join(blacklisted_keys_present))) - if not all(isinstance(k, six.string_types) for k in keys): + if not all(isinstance(k, str) for k in keys): raise BadPolicyYamlFile("keys in yaml aren't all strings?") # check that the dictionary looks like a mapping of str to str - if not all(isinstance(v, six.string_types) for v in doc.values()): + if not all(isinstance(v, str) for v in doc.values()): raise BadPolicyYamlFile("values in yaml aren't all strings?") return doc @@ -530,8 +523,7 @@ def clean_policyd_dir_for(service, keep_paths=None, user=None, group=None): hookenv.log("Cleaning path: {}".format(path), level=hookenv.DEBUG) if not os.path.exists(path): ch_host.mkdir(path, owner=_user, group=_group, perms=0o775) - _scanner = os.scandir if hasattr(os, 'scandir') else _fallback_scandir - for direntry in _scanner(path): + for direntry in os.scandir(path): # see if the path should be kept. if direntry.path in keep_paths: continue @@ -558,36 +550,6 @@ def maybe_create_directory_for(path, user, group): ch_host.mkdir(_dir, owner=user, group=group, perms=0o775) -@contextlib.contextmanager -def _fallback_scandir(path): - """Fallback os.scandir implementation. - - provide a fallback implementation of os.scandir if this module ever gets - used in a py2 or py34 charm. Uses os.listdir() to get the names in the path, - and then mocks the is_dir() function using os.path.isdir() to check for - directory. - - :param path: the path to list the directories for - :type path: str - :returns: Generator that provides _FBDirectory objects - :rtype: ContextManager[_FBDirectory] - """ - for f in os.listdir(path): - yield _FBDirectory(f) - - -class _FBDirectory(object): - """Mock a scandir Directory object with enough to use in - clean_policyd_dir_for - """ - - def __init__(self, path): - self.path = path - - def is_dir(self): - return os.path.isdir(self.path) - - def path_for_policy_file(service, name): """Return the full path for a policy.d file that will be written to the service's policy.d directory. @@ -768,7 +730,7 @@ def process_policy_resource_file(resource_file, _group) # Every thing worked, so we mark up a success. completed = True - except (BadZipFile, BadPolicyZipFile, BadPolicyYamlFile) as e: + except (zipfile.BadZipFile, BadPolicyZipFile, BadPolicyYamlFile) as e: hookenv.log("Processing {} failed: {}".format(resource_file, str(e)), level=POLICYD_LOG_LEVEL_DEFAULT) except IOError as e: diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index 9cc96d6..1fa2814 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -25,7 +25,6 @@ import re import itertools import functools -import six import traceback import uuid import yaml @@ -159,6 +158,7 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2021.1', 'wallaby'), ('2021.2', 'xena'), ('2022.1', 'yoga'), + ('2022.2', 'zed'), ]) # The ugly duckling - must list releases oldest to newest @@ -362,6 +362,8 @@ def get_os_codename_install_source(src): rel = '' if src is None: return rel + if src in OPENSTACK_RELEASES: + return src if src in ['distro', 'distro-proposed', 'proposed']: try: rel = UBUNTU_OPENSTACK_RELEASE[ubuntu_rel] @@ -399,19 +401,23 @@ def get_os_codename_version(vers): error_out(e) -def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES): +def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES, + raise_exception=False): '''Determine OpenStack version number from codename.''' - for k, v in six.iteritems(version_map): + for k, v in version_map.items(): if v == codename: return k e = 'Could not derive OpenStack version for '\ 'codename: %s' % codename + if raise_exception: + raise ValueError(str(e)) error_out(e) def get_os_version_codename_swift(codename): '''Determine OpenStack version number of swift from codename.''' - for k, v in six.iteritems(SWIFT_CODENAMES): + # for k, v in six.iteritems(SWIFT_CODENAMES): + for k, v in SWIFT_CODENAMES.items(): if k == codename: return v[-1] e = 'Could not derive swift version for '\ @@ -421,17 +427,17 @@ def get_os_version_codename_swift(codename): def get_swift_codename(version): '''Determine OpenStack codename that corresponds to swift version.''' - codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v] + codenames = [k for k, v in SWIFT_CODENAMES.items() if version in v] if len(codenames) > 1: # If more than one release codename contains this version we determine # the actual codename based on the highest available install source. for codename in reversed(codenames): releases = UBUNTU_OPENSTACK_RELEASE - release = [k for k, v in six.iteritems(releases) if codename in v] - ret = subprocess.check_output(['apt-cache', 'policy', 'swift']) - if six.PY3: - ret = ret.decode('UTF-8') + release = [k for k, v in releases.items() if codename in v] + ret = (subprocess + .check_output(['apt-cache', 'policy', 'swift']) + .decode('UTF-8')) if codename in ret or release[0] in ret: return codename elif len(codenames) == 1: @@ -441,7 +447,7 @@ def get_swift_codename(version): match = re.match(r'^(\d+)\.(\d+)', version) if match: major_minor_version = match.group(0) - for codename, versions in six.iteritems(SWIFT_CODENAMES): + for codename, versions in SWIFT_CODENAMES.items(): for release_version in versions: if release_version.startswith(major_minor_version): return codename @@ -477,9 +483,7 @@ def get_os_codename_package(package, fatal=True): if snap_install_requested(): cmd = ['snap', 'list', package] try: - out = subprocess.check_output(cmd) - if six.PY3: - out = out.decode('UTF-8') + out = subprocess.check_output(cmd).decode('UTF-8') except subprocess.CalledProcessError: return None lines = out.split('\n') @@ -549,16 +553,14 @@ def get_os_version_package(pkg, fatal=True): if 'swift' in pkg: vers_map = SWIFT_CODENAMES - for cname, version in six.iteritems(vers_map): + for cname, version in vers_map.items(): if cname == codename: return version[-1] else: vers_map = OPENSTACK_CODENAMES - for version, cname in six.iteritems(vers_map): + for version, cname in vers_map.items(): if cname == codename: return version - # e = "Could not determine OpenStack version for package: %s" % pkg - # error_out(e) def get_installed_os_version(): @@ -821,10 +823,10 @@ def save_script_rc(script_path="scripts/scriptrc", **env_vars): if not os.path.exists(os.path.dirname(juju_rc_path)): os.mkdir(os.path.dirname(juju_rc_path)) with open(juju_rc_path, 'wt') as rc_script: - rc_script.write( - "#!/bin/bash\n") - [rc_script.write('export %s=%s\n' % (u, p)) - for u, p in six.iteritems(env_vars) if u != "script_path"] + rc_script.write("#!/bin/bash\n") + for u, p in env_vars.items(): + if u != "script_path": + rc_script.write('export %s=%s\n' % (u, p)) def openstack_upgrade_available(package): @@ -1039,7 +1041,7 @@ def _determine_os_workload_status( state, message, lambda: charm_func(configs)) if state is None: - state, message = _ows_check_services_running(services, ports) + state, message = ows_check_services_running(services, ports) if state is None: state = 'active' @@ -1213,7 +1215,12 @@ def _ows_check_charm_func(state, message, charm_func_with_configs): return state, message +@deprecate("use ows_check_services_running() instead", "2022-05", log=juju_log) def _ows_check_services_running(services, ports): + return ows_check_services_running(services, ports) + + +def ows_check_services_running(services, ports): """Check that the services that should be running are actually running and that any ports specified are being listened to. @@ -1413,45 +1420,75 @@ def incomplete_relation_data(configs, required_interfaces): for i in incomplete_relations} -def do_action_openstack_upgrade(package, upgrade_callback, configs, - force_upgrade=False): +def do_action_openstack_upgrade(package, upgrade_callback, configs): """Perform action-managed OpenStack upgrade. Upgrades packages to the configured openstack-origin version and sets the corresponding action status as a result. - If the charm was installed from source we cannot upgrade it. For backwards compatibility a config flag (action-managed-upgrade) must be set for this code to run, otherwise a full service level upgrade will fire on config-changed. - @param package: package name for determining if upgrade available + @param package: package name for determining if openstack upgrade available @param upgrade_callback: function callback to charm's upgrade function @param configs: templating object derived from OSConfigRenderer class - @param force_upgrade: perform dist-upgrade regardless of new openstack @return: True if upgrade successful; False if upgrade failed or skipped """ ret = False - if openstack_upgrade_available(package) or force_upgrade: + if openstack_upgrade_available(package): if config('action-managed-upgrade'): juju_log('Upgrading OpenStack release') try: upgrade_callback(configs=configs) - action_set({'outcome': 'success, upgrade completed.'}) + action_set({'outcome': 'success, upgrade completed'}) ret = True except Exception: - action_set({'outcome': 'upgrade failed, see traceback.'}) + action_set({'outcome': 'upgrade failed, see traceback'}) action_set({'traceback': traceback.format_exc()}) - action_fail('do_openstack_upgrade resulted in an ' + action_fail('upgrade callback resulted in an ' 'unexpected error') else: action_set({'outcome': 'action-managed-upgrade config is ' - 'False, skipped upgrade.'}) + 'False, skipped upgrade'}) else: - action_set({'outcome': 'no upgrade available.'}) + action_set({'outcome': 'no upgrade available'}) + + return ret + + +def do_action_package_upgrade(package, upgrade_callback, configs): + """Perform package upgrade within the current OpenStack release. + + Upgrades packages only if there is not an openstack upgrade available, + and sets the corresponding action status as a result. + + @param package: package name for determining if openstack upgrade available + @param upgrade_callback: function callback to charm's upgrade function + @param configs: templating object derived from OSConfigRenderer class + + @return: True if upgrade successful; False if upgrade failed or skipped + """ + ret = False + + if not openstack_upgrade_available(package): + juju_log('Upgrading packages') + + try: + upgrade_callback(configs=configs) + action_set({'outcome': 'success, upgrade completed'}) + ret = True + except Exception: + action_set({'outcome': 'upgrade failed, see traceback'}) + action_set({'traceback': traceback.format_exc()}) + action_fail('upgrade callback resulted in an ' + 'unexpected error') + else: + action_set({'outcome': 'upgrade skipped because an openstack upgrade ' + 'is available'}) return ret @@ -1849,21 +1886,20 @@ def pausable_restart_on_change(restart_map, stopstart=False, """ def wrap(f): - # py27 compatible nonlocal variable. When py3 only, replace with - # nonlocal keyword - __restart_map_cache = {'cache': None} + __restart_map_cache = None @functools.wraps(f) def wrapped_f(*args, **kwargs): + nonlocal __restart_map_cache if is_unit_paused_set(): return f(*args, **kwargs) - if __restart_map_cache['cache'] is None: - __restart_map_cache['cache'] = restart_map() \ + if __restart_map_cache is None: + __restart_map_cache = restart_map() \ if callable(restart_map) else restart_map # otherwise, normal restart_on_change functionality return restart_on_change_helper( (lambda: f(*args, **kwargs)), - __restart_map_cache['cache'], + __restart_map_cache, stopstart, restart_functions, can_restart_now_f, @@ -1888,7 +1924,7 @@ def ordered(orderme): raise ValueError('argument must be a dict type') result = OrderedDict() - for k, v in sorted(six.iteritems(orderme), key=lambda x: x[0]): + for k, v in sorted(orderme.items(), key=lambda x: x[0]): if isinstance(v, dict): result[k] = ordered(v) else: diff --git a/charmhelpers/contrib/python.py b/charmhelpers/contrib/python.py index 84cba8c..fcded68 100644 --- a/charmhelpers/contrib/python.py +++ b/charmhelpers/contrib/python.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - # deprecated aliases for backwards compatibility from charmhelpers.fetch.python import debug # noqa from charmhelpers.fetch.python import packages # noqa diff --git a/charmhelpers/contrib/storage/linux/ceph.py b/charmhelpers/contrib/storage/linux/ceph.py index c70aeb2..1b20b8f 100644 --- a/charmhelpers/contrib/storage/linux/ceph.py +++ b/charmhelpers/contrib/storage/linux/ceph.py @@ -23,7 +23,6 @@ import collections import errno import hashlib import math -import six import os import shutil @@ -218,7 +217,7 @@ def validator(value, valid_type, valid_range=None): "was given {} of type {}" .format(valid_range, type(valid_range))) # If we're dealing with strings - if isinstance(value, six.string_types): + if isinstance(value, str): assert value in valid_range, ( "{} is not in the list {}".format(value, valid_range)) # Integer, float should have a min and max @@ -434,9 +433,9 @@ class BasePool(object): :type mode: str """ # Check the input types and values - validator(value=cache_pool, valid_type=six.string_types) + validator(value=cache_pool, valid_type=str) validator( - value=mode, valid_type=six.string_types, + value=mode, valid_type=str, valid_range=["readonly", "writeback"]) check_call([ @@ -615,7 +614,8 @@ class Pool(BasePool): class ReplicatedPool(BasePool): def __init__(self, service, name=None, pg_num=None, replicas=None, - percent_data=None, app_name=None, op=None): + percent_data=None, app_name=None, op=None, + profile_name='replicated_rule'): """Initialize ReplicatedPool object. Pool information is either initialized from individual keyword @@ -632,6 +632,8 @@ class ReplicatedPool(BasePool): to this replicated pool. :type replicas: int :raises: KeyError + :param profile_name: Crush Profile to use + :type profile_name: Optional[str] """ # NOTE: Do not perform initialization steps that require live data from # a running cluster here. The *Pool classes may be used for validation. @@ -646,11 +648,20 @@ class ReplicatedPool(BasePool): # we will fail with KeyError if it is not provided. self.replicas = op['replicas'] self.pg_num = op.get('pg_num') + self.profile_name = op.get('crush-profile') or profile_name else: self.replicas = replicas or 2 self.pg_num = pg_num + self.profile_name = profile_name or 'replicated_rule' def _create(self): + # Validate if crush profile exists + if self.profile_name is None: + msg = ("Failed to discover crush profile named " + "{}".format(self.profile_name)) + log(msg, level=ERROR) + raise PoolCreationError(msg) + # Do extra validation on pg_num with data from live cluster if self.pg_num: # Since the number of placement groups were specified, ensure @@ -668,12 +679,12 @@ class ReplicatedPool(BasePool): '--pg-num-min={}'.format( min(AUTOSCALER_DEFAULT_PGS, self.pg_num) ), - self.name, str(self.pg_num) + self.name, str(self.pg_num), self.profile_name ] else: cmd = [ 'ceph', '--id', self.service, 'osd', 'pool', 'create', - self.name, str(self.pg_num) + self.name, str(self.pg_num), self.profile_name ] check_call(cmd) @@ -692,7 +703,7 @@ class ErasurePool(BasePool): def __init__(self, service, name=None, erasure_code_profile=None, percent_data=None, app_name=None, op=None, allow_ec_overwrites=False): - """Initialize ReplicatedPool object. + """Initialize ErasurePool object. Pool information is either initialized from individual keyword arguments or from a individual CephBrokerRq operation Dict. @@ -778,10 +789,11 @@ def enabled_manager_modules(): :rtype: List[str] """ cmd = ['ceph', 'mgr', 'module', 'ls'] + quincy_or_later = cmp_pkgrevno('ceph-common', '17.1.0') >= 0 + if quincy_or_later: + cmd.append('--format=json') try: - modules = check_output(cmd) - if six.PY3: - modules = modules.decode('UTF-8') + modules = check_output(cmd).decode('utf-8') except CalledProcessError as e: log("Failed to list ceph modules: {}".format(e), WARNING) return [] @@ -814,10 +826,10 @@ def get_mon_map(service): ceph command fails. """ try: - mon_status = check_output(['ceph', '--id', service, - 'mon_status', '--format=json']) - if six.PY3: - mon_status = mon_status.decode('UTF-8') + octopus_or_later = cmp_pkgrevno('ceph-common', '15.0.0') >= 0 + mon_status_cmd = 'quorum_status' if octopus_or_later else 'mon_status' + mon_status = (check_output(['ceph', '--id', service, mon_status_cmd, + '--format=json'])).decode('utf-8') try: return json.loads(mon_status) except ValueError as v: @@ -959,9 +971,7 @@ def get_erasure_profile(service, name): try: out = check_output(['ceph', '--id', service, 'osd', 'erasure-code-profile', 'get', - name, '--format=json']) - if six.PY3: - out = out.decode('UTF-8') + name, '--format=json']).decode('utf-8') return json.loads(out) except (CalledProcessError, OSError, ValueError): return None @@ -1164,8 +1174,7 @@ def create_erasure_profile(service, profile_name, 'nvme' ] - validator(erasure_plugin_name, six.string_types, - list(plugin_techniques.keys())) + validator(erasure_plugin_name, str, list(plugin_techniques.keys())) cmd = [ 'ceph', '--id', service, @@ -1176,7 +1185,7 @@ def create_erasure_profile(service, profile_name, ] if erasure_plugin_technique: - validator(erasure_plugin_technique, six.string_types, + validator(erasure_plugin_technique, str, plugin_techniques[erasure_plugin_name]) cmd.append('technique={}'.format(erasure_plugin_technique)) @@ -1189,7 +1198,7 @@ def create_erasure_profile(service, profile_name, failure_domain = 'rack' if failure_domain: - validator(failure_domain, six.string_types, failure_domains) + validator(failure_domain, str, failure_domains) # failure_domain changed in luminous if luminous_or_later: cmd.append('crush-failure-domain={}'.format(failure_domain)) @@ -1198,7 +1207,7 @@ def create_erasure_profile(service, profile_name, # device class new in luminous if luminous_or_later and device_class: - validator(device_class, six.string_types, device_classes) + validator(device_class, str, device_classes) cmd.append('crush-device-class={}'.format(device_class)) else: log('Skipping device class configuration (ceph < 12.0.0)', @@ -1213,7 +1222,7 @@ def create_erasure_profile(service, profile_name, raise ValueError("locality must be provided for lrc plugin") # LRC optional configuration if crush_locality: - validator(crush_locality, six.string_types, failure_domains) + validator(crush_locality, str, failure_domains) cmd.append('crush-locality={}'.format(crush_locality)) if erasure_plugin_name == 'shec': @@ -1241,8 +1250,8 @@ def rename_pool(service, old_name, new_name): :param new_name: Name to rename pool to. :type new_name: str """ - validator(value=old_name, valid_type=six.string_types) - validator(value=new_name, valid_type=six.string_types) + validator(value=old_name, valid_type=str) + validator(value=new_name, valid_type=str) cmd = [ 'ceph', '--id', service, @@ -1260,7 +1269,7 @@ def erasure_profile_exists(service, name): :returns: True if it exists, False otherwise. :rtype: bool """ - validator(value=name, valid_type=six.string_types) + validator(value=name, valid_type=str) try: check_call(['ceph', '--id', service, 'osd', 'erasure-code-profile', 'get', @@ -1280,12 +1289,10 @@ def get_cache_mode(service, pool_name): :returns: Current cache mode. :rtype: Optional[int] """ - validator(value=service, valid_type=six.string_types) - validator(value=pool_name, valid_type=six.string_types) + validator(value=service, valid_type=str) + validator(value=pool_name, valid_type=str) out = check_output(['ceph', '--id', service, - 'osd', 'dump', '--format=json']) - if six.PY3: - out = out.decode('UTF-8') + 'osd', 'dump', '--format=json']).decode('utf-8') try: osd_json = json.loads(out) for pool in osd_json['pools']: @@ -1299,9 +1306,8 @@ def get_cache_mode(service, pool_name): def pool_exists(service, name): """Check to see if a RADOS pool already exists.""" try: - out = check_output(['rados', '--id', service, 'lspools']) - if six.PY3: - out = out.decode('UTF-8') + out = check_output( + ['rados', '--id', service, 'lspools']).decode('utf-8') except CalledProcessError: return False @@ -1320,13 +1326,11 @@ def get_osds(service, device_class=None): out = check_output(['ceph', '--id', service, 'osd', 'crush', 'class', 'ls-osd', device_class, - '--format=json']) + '--format=json']).decode('utf-8') else: out = check_output(['ceph', '--id', service, 'osd', 'ls', - '--format=json']) - if six.PY3: - out = out.decode('UTF-8') + '--format=json']).decode('utf-8') return json.loads(out) @@ -1343,9 +1347,7 @@ def rbd_exists(service, pool, rbd_img): """Check to see if a RADOS block device exists.""" try: out = check_output(['rbd', 'list', '--id', - service, '--pool', pool]) - if six.PY3: - out = out.decode('UTF-8') + service, '--pool', pool]).decode('utf-8') except CalledProcessError: return False @@ -1371,7 +1373,7 @@ def update_pool(client, pool, settings): :raises: CalledProcessError """ cmd = ['ceph', '--id', client, 'osd', 'pool', 'set', pool] - for k, v in six.iteritems(settings): + for k, v in settings.items(): check_call(cmd + [k, v]) @@ -1509,9 +1511,7 @@ def configure(service, key, auth, use_syslog): def image_mapped(name): """Determine whether a RADOS block device is mapped locally.""" try: - out = check_output(['rbd', 'showmapped']) - if six.PY3: - out = out.decode('UTF-8') + out = check_output(['rbd', 'showmapped']).decode('utf-8') except CalledProcessError: return False @@ -1857,7 +1857,7 @@ class CephBrokerRq(object): } def add_op_create_replicated_pool(self, name, replica_count=3, pg_num=None, - **kwargs): + crush_profile=None, **kwargs): """Adds an operation to create a replicated pool. Refer to docstring for ``_partial_build_common_op_create`` for @@ -1871,6 +1871,10 @@ class CephBrokerRq(object): for pool. :type pg_num: int :raises: AssertionError if provided data is of invalid type/range + :param crush_profile: Name of crush profile to use. If not set the + ceph-mon unit handling the broker request will + set its default value. + :type crush_profile: Optional[str] """ if pg_num and kwargs.get('weight'): raise ValueError('pg_num and weight are mutually exclusive') @@ -1880,6 +1884,7 @@ class CephBrokerRq(object): 'name': name, 'replicas': replica_count, 'pg_num': pg_num, + 'crush-profile': crush_profile } op.update(self._partial_build_common_op_create(**kwargs)) diff --git a/charmhelpers/contrib/storage/linux/loopback.py b/charmhelpers/contrib/storage/linux/loopback.py index 74bab40..04daea2 100644 --- a/charmhelpers/contrib/storage/linux/loopback.py +++ b/charmhelpers/contrib/storage/linux/loopback.py @@ -19,8 +19,6 @@ from subprocess import ( check_output, ) -import six - ################################################## # loopback device helpers. @@ -40,9 +38,7 @@ def loopback_devices(): ''' loopbacks = {} cmd = ['losetup', '-a'] - output = check_output(cmd) - if six.PY3: - output = output.decode('utf-8') + output = check_output(cmd).decode('utf-8') devs = [d.strip().split(' ', 2) for d in output.splitlines() if d != ''] for dev, _, f in devs: loopbacks[dev.replace(':', '')] = re.search(r'\((.+)\)', f).groups()[0] @@ -57,7 +53,7 @@ def create_loopback(file_path): ''' file_path = os.path.abspath(file_path) check_call(['losetup', '--find', file_path]) - for d, f in six.iteritems(loopback_devices()): + for d, f in loopback_devices().items(): if f == file_path: return d @@ -71,7 +67,7 @@ def ensure_loopback_device(path, size): :returns: str: Full path to the ensured loopback device (eg, /dev/loop0) ''' - for d, f in six.iteritems(loopback_devices()): + for d, f in loopback_devices().items(): if f == path: return d diff --git a/charmhelpers/contrib/templating/contexts.py b/charmhelpers/contrib/templating/contexts.py index c1adf94..a491fdb 100644 --- a/charmhelpers/contrib/templating/contexts.py +++ b/charmhelpers/contrib/templating/contexts.py @@ -20,8 +20,6 @@ import os import yaml -import six - import charmhelpers.core.hookenv @@ -93,7 +91,8 @@ def juju_state_to_yaml(yaml_path, namespace_separator=':', By default, hyphens are allowed in keys as this is supported by yaml, but for tools like ansible, hyphens are not valid [1]. - [1] http://www.ansibleworks.com/docs/playbooks_variables.html#what-makes-a-valid-variable-name + [1] http://www.ansibleworks.com/docs/playbooks_variables.html + #what-makes-a-valid-variable-name """ config = charmhelpers.core.hookenv.config() @@ -101,16 +100,17 @@ def juju_state_to_yaml(yaml_path, namespace_separator=':', # file resources etc. config['charm_dir'] = charm_dir config['local_unit'] = charmhelpers.core.hookenv.local_unit() - config['unit_private_address'] = charmhelpers.core.hookenv.unit_private_ip() + config['unit_private_address'] = ( + charmhelpers.core.hookenv.unit_private_ip()) config['unit_public_address'] = charmhelpers.core.hookenv.unit_get( 'public-address' ) # Don't use non-standard tags for unicode which will not # work when salt uses yaml.load_safe. - yaml.add_representer(six.text_type, + yaml.add_representer(str, lambda dumper, value: dumper.represent_scalar( - six.u('tag:yaml.org,2002:str'), value)) + 'tag:yaml.org,2002:str', value)) yaml_dir = os.path.dirname(yaml_path) if not os.path.exists(yaml_dir): diff --git a/charmhelpers/contrib/templating/jinja.py b/charmhelpers/contrib/templating/jinja.py index c6ad9d0..de6a647 100644 --- a/charmhelpers/contrib/templating/jinja.py +++ b/charmhelpers/contrib/templating/jinja.py @@ -13,18 +13,14 @@ # limitations under the License. """ -Templating using the python-jinja2 package. +Templating using the python3-jinja2 package. """ -import six from charmhelpers.fetch import apt_install, apt_update try: import jinja2 except ImportError: apt_update(fatal=True) - if six.PY3: - apt_install(["python3-jinja2"], fatal=True) - else: - apt_install(["python-jinja2"], fatal=True) + apt_install(["python3-jinja2"], fatal=True) import jinja2 diff --git a/charmhelpers/core/hookenv.py b/charmhelpers/core/hookenv.py index e94247a..370c3e8 100644 --- a/charmhelpers/core/hookenv.py +++ b/charmhelpers/core/hookenv.py @@ -17,12 +17,11 @@ # Authors: # Charm Helpers Developers -from __future__ import print_function import copy from distutils.version import LooseVersion from enum import Enum from functools import wraps -from collections import namedtuple +from collections import namedtuple, UserDict import glob import os import json @@ -36,12 +35,6 @@ from subprocess import CalledProcessError from charmhelpers import deprecate -import six -if not six.PY3: - from UserDict import UserDict -else: - from collections import UserDict - CRITICAL = "CRITICAL" ERROR = "ERROR" @@ -112,7 +105,7 @@ def log(message, level=None): command = ['juju-log'] if level: command += ['-l', level] - if not isinstance(message, six.string_types): + if not isinstance(message, str): message = repr(message) command += [message[:SH_MAX_ARG]] # Missing juju-log should not cause failures in unit tests @@ -132,7 +125,7 @@ def log(message, level=None): def function_log(message): """Write a function progress message""" command = ['function-log'] - if not isinstance(message, six.string_types): + if not isinstance(message, str): message = repr(message) command += [message[:SH_MAX_ARG]] # Missing function-log should not cause failures in unit tests @@ -445,12 +438,6 @@ def config(scope=None): """ global _cache_config config_cmd_line = ['config-get', '--all', '--format=json'] - try: - # JSON Decode Exception for Python3.5+ - exc_json = json.decoder.JSONDecodeError - except AttributeError: - # JSON Decode Exception for Python2.7 through Python3.4 - exc_json = ValueError try: if _cache_config is None: config_data = json.loads( @@ -459,7 +446,7 @@ def config(scope=None): if scope is not None: return _cache_config.get(scope) return _cache_config - except (exc_json, UnicodeDecodeError) as e: + except (json.decoder.JSONDecodeError, UnicodeDecodeError) as e: log('Unable to parse output from config-get: config_cmd_line="{}" ' 'message="{}"' .format(config_cmd_line, str(e)), level=ERROR) @@ -491,12 +478,26 @@ def relation_get(attribute=None, unit=None, rid=None, app=None): raise +@cached +def _relation_set_accepts_file(): + """Return True if the juju relation-set command accepts a file. + + Cache the result as it won't change during the execution of a hook, and + thus we can make relation_set() more efficient by only checking for the + first relation_set() call. + + :returns: True if relation_set accepts a file. + :rtype: bool + :raises: subprocess.CalledProcessError if the check fails. + """ + return "--file" in subprocess.check_output( + ["relation-set", "--help"], universal_newlines=True) + + def relation_set(relation_id=None, relation_settings=None, app=False, **kwargs): """Set relation information for the current unit""" relation_settings = relation_settings if relation_settings else {} relation_cmd_line = ['relation-set'] - accepts_file = "--file" in subprocess.check_output( - relation_cmd_line + ["--help"], universal_newlines=True) if app: relation_cmd_line.append('--app') if relation_id is not None: @@ -508,7 +509,7 @@ def relation_set(relation_id=None, relation_settings=None, app=False, **kwargs): # sites pass in things like dicts or numbers. if value is not None: settings[key] = "{}".format(value) - if accepts_file: + if _relation_set_accepts_file(): # --file was introduced in Juju 1.23.2. Use it by default if # available, since otherwise we'll break if the relation data is # too big. Ideally we should tell relation-set to read the data from @@ -1003,14 +1004,8 @@ def cmd_exists(cmd): @cached -@deprecate("moved to function_get()", log=log) def action_get(key=None): - """ - .. deprecated:: 0.20.7 - Alias for :func:`function_get`. - - Gets the value of an action parameter, or all key/value param pairs. - """ + """Gets the value of an action parameter, or all key/value param pairs.""" cmd = ['action-get'] if key is not None: cmd.append(key) @@ -1020,8 +1015,12 @@ def action_get(key=None): @cached +@deprecate("moved to action_get()", log=log) def function_get(key=None): - """Gets the value of an action parameter, or all key/value param pairs""" + """ + .. deprecated:: + Gets the value of an action parameter, or all key/value param pairs. + """ cmd = ['function-get'] # Fallback for older charms. if not cmd_exists('function-get'): @@ -1034,22 +1033,20 @@ def function_get(key=None): return function_data -@deprecate("moved to function_set()", log=log) def action_set(values): - """ - .. deprecated:: 0.20.7 - Alias for :func:`function_set`. - - Sets the values to be returned after the action finishes. - """ + """Sets the values to be returned after the action finishes.""" cmd = ['action-set'] for k, v in list(values.items()): cmd.append('{}={}'.format(k, v)) subprocess.check_call(cmd) +@deprecate("moved to action_set()", log=log) def function_set(values): - """Sets the values to be returned after the function finishes""" + """ + .. deprecated:: + Sets the values to be returned after the function finishes. + """ cmd = ['function-set'] # Fallback for older charms. if not cmd_exists('function-get'): @@ -1060,12 +1057,8 @@ def function_set(values): subprocess.check_call(cmd) -@deprecate("moved to function_fail()", log=log) def action_fail(message): """ - .. deprecated:: 0.20.7 - Alias for :func:`function_fail`. - Sets the action status to failed and sets the error message. The results set by action_set are preserved. @@ -1073,10 +1066,14 @@ def action_fail(message): subprocess.check_call(['action-fail', message]) +@deprecate("moved to action_fail()", log=log) def function_fail(message): - """Sets the function status to failed and sets the error message. + """ + .. deprecated:: + Sets the function status to failed and sets the error message. - The results set by function_set are preserved.""" + The results set by function_set are preserved. + """ cmd = ['function-fail'] # Fallback for older charms. if not cmd_exists('function-fail'): diff --git a/charmhelpers/core/host.py b/charmhelpers/core/host.py index 994ec8a..ef6c8ec 100644 --- a/charmhelpers/core/host.py +++ b/charmhelpers/core/host.py @@ -31,7 +31,6 @@ import subprocess import hashlib import functools import itertools -import six from contextlib import contextmanager from collections import OrderedDict, defaultdict @@ -115,6 +114,33 @@ def service_stop(service_name, **kwargs): return service('stop', service_name, **kwargs) +def service_enable(service_name, **kwargs): + """Enable a system service. + + The specified service name is managed via the system level init system. + Some init systems (e.g. upstart) require that additional arguments be + provided in order to directly control service instances whereas other init + systems allow for addressing instances of a service directly by name (e.g. + systemd). + + The kwargs allow for the additional parameters to be passed to underlying + init systems for those systems which require/allow for them. For example, + the ceph-osd upstart script requires the id parameter to be passed along + in order to identify which running daemon should be restarted. The follow- + ing example restarts the ceph-osd service for instance id=4: + + service_enable('ceph-osd', id=4) + + :param service_name: the name of the service to enable + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for init systems not allowing additional + parameters via the commandline (systemd). + """ + return service('enable', service_name, **kwargs) + + def service_restart(service_name, **kwargs): """Restart a system service. @@ -135,7 +161,7 @@ def service_restart(service_name, **kwargs): :param service_name: the name of the service to restart :param **kwargs: additional parameters to pass to the init system when managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs + parameters to the init system's commandline. kwargs are ignored for init systems not allowing additional parameters via the commandline (systemd). """ @@ -251,7 +277,7 @@ def service_resume(service_name, init_dir="/etc/init", return started -def service(action, service_name, **kwargs): +def service(action, service_name=None, **kwargs): """Control a system service. :param action: the action to take on the service @@ -260,10 +286,12 @@ def service(action, service_name, **kwargs): the form of key=value. """ if init_is_systemd(service_name=service_name): - cmd = ['systemctl', action, service_name] + cmd = ['systemctl', action] + if service_name is not None: + cmd.append(service_name) else: cmd = ['service', service_name, action] - for key, value in six.iteritems(kwargs): + for key, value in kwargs.items(): parameter = '%s=%s' % (key, value) cmd.append(parameter) return subprocess.call(cmd) == 0 @@ -289,7 +317,7 @@ def service_running(service_name, **kwargs): if os.path.exists(_UPSTART_CONF.format(service_name)): try: cmd = ['status', service_name] - for key, value in six.iteritems(kwargs): + for key, value in kwargs.items(): parameter = '%s=%s' % (key, value) cmd.append(parameter) output = subprocess.check_output( @@ -564,7 +592,7 @@ def write_file(path, content, owner='root', group='root', perms=0o444): with open(path, 'wb') as target: os.fchown(target.fileno(), uid, gid) os.fchmod(target.fileno(), perms) - if six.PY3 and isinstance(content, six.string_types): + if isinstance(content, str): content = content.encode('UTF-8') target.write(content) return @@ -967,7 +995,7 @@ def get_bond_master(interface): def list_nics(nic_type=None): """Return a list of nics of given type(s)""" - if isinstance(nic_type, six.string_types): + if isinstance(nic_type, str): int_types = [nic_type] else: int_types = nic_type @@ -1081,8 +1109,7 @@ def chownr(path, owner, group, follow_links=True, chowntopdir=False): try: chown(full, uid, gid) except (IOError, OSError) as e: - # Intended to ignore "file not found". Catching both to be - # compatible with both Python 2.7 and 3.x. + # Intended to ignore "file not found". if e.errno == errno.ENOENT: pass diff --git a/charmhelpers/core/host_factory/ubuntu.py b/charmhelpers/core/host_factory/ubuntu.py index 0906c5c..cc2d89f 100644 --- a/charmhelpers/core/host_factory/ubuntu.py +++ b/charmhelpers/core/host_factory/ubuntu.py @@ -30,6 +30,7 @@ UBUNTU_RELEASES = ( 'hirsute', 'impish', 'jammy', + 'kinetic', ) diff --git a/charmhelpers/core/services/base.py b/charmhelpers/core/services/base.py index 9f88029..8d217b5 100644 --- a/charmhelpers/core/services/base.py +++ b/charmhelpers/core/services/base.py @@ -15,9 +15,8 @@ import os import json import inspect -from collections import Iterable, OrderedDict - -import six +from collections import OrderedDict +from collections.abc import Iterable from charmhelpers.core import host from charmhelpers.core import hookenv @@ -171,10 +170,7 @@ class ServiceManager(object): if not units: continue remote_service = units[0].split('/')[0] - if six.PY2: - argspec = inspect.getargspec(provider.provide_data) - else: - argspec = inspect.getfullargspec(provider.provide_data) + argspec = inspect.getfullargspec(provider.provide_data) if len(argspec.args) > 1: data = provider.provide_data(remote_service, service_ready) else: diff --git a/charmhelpers/core/services/helpers.py b/charmhelpers/core/services/helpers.py index 3e6e30d..5bf62dd 100644 --- a/charmhelpers/core/services/helpers.py +++ b/charmhelpers/core/services/helpers.py @@ -179,7 +179,7 @@ class RequiredConfig(dict): self.required_options = args self['config'] = hookenv.config() with open(os.path.join(hookenv.charm_dir(), 'config.yaml')) as fp: - self.config = yaml.load(fp).get('options', {}) + self.config = yaml.safe_load(fp).get('options', {}) def __bool__(self): for option in self.required_options: @@ -227,7 +227,7 @@ class StoredContext(dict): if not os.path.isabs(file_name): file_name = os.path.join(hookenv.charm_dir(), file_name) with open(file_name, 'r') as file_stream: - data = yaml.load(file_stream) + data = yaml.safe_load(file_stream) if not data: raise OSError("%s is empty" % file_name) return data diff --git a/charmhelpers/core/strutils.py b/charmhelpers/core/strutils.py index 28c6b3f..3136687 100644 --- a/charmhelpers/core/strutils.py +++ b/charmhelpers/core/strutils.py @@ -15,7 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import six import re TRUTHY_STRINGS = {'y', 'yes', 'true', 't', 'on'} @@ -27,8 +26,8 @@ def bool_from_string(value, truthy_strings=TRUTHY_STRINGS, falsey_strings=FALSEY Returns True if value translates to True otherwise False. """ - if isinstance(value, six.string_types): - value = six.text_type(value) + if isinstance(value, str): + value = str(value) else: msg = "Unable to interpret non-string value '%s' as boolean" % (value) raise ValueError(msg) @@ -61,8 +60,8 @@ def bytes_from_string(value): 'P': 5, 'PB': 5, } - if isinstance(value, six.string_types): - value = six.text_type(value) + if isinstance(value, str): + value = str(value) else: msg = "Unable to interpret non-string value '%s' as bytes" % (value) raise ValueError(msg) diff --git a/charmhelpers/core/templating.py b/charmhelpers/core/templating.py index 9014015..cb0213d 100644 --- a/charmhelpers/core/templating.py +++ b/charmhelpers/core/templating.py @@ -13,7 +13,6 @@ # limitations under the License. import os -import sys from charmhelpers.core import host from charmhelpers.core import hookenv @@ -43,9 +42,8 @@ def render(source, target, context, owner='root', group='root', The rendered template will be written to the file as well as being returned as a string. - Note: Using this requires python-jinja2 or python3-jinja2; if it is not - installed, calling this will attempt to use charmhelpers.fetch.apt_install - to install it. + Note: Using this requires python3-jinja2; if it is not installed, calling + this will attempt to use charmhelpers.fetch.apt_install to install it. """ try: from jinja2 import FileSystemLoader, Environment, exceptions @@ -57,10 +55,7 @@ def render(source, target, context, owner='root', group='root', 'charmhelpers.fetch to install it', level=hookenv.ERROR) raise - if sys.version_info.major == 2: - apt_install('python-jinja2', fatal=True) - else: - apt_install('python3-jinja2', fatal=True) + apt_install('python3-jinja2', fatal=True) from jinja2 import FileSystemLoader, Environment, exceptions if template_loader: diff --git a/charmhelpers/fetch/__init__.py b/charmhelpers/fetch/__init__.py index 9497ee0..1283f25 100644 --- a/charmhelpers/fetch/__init__.py +++ b/charmhelpers/fetch/__init__.py @@ -20,11 +20,7 @@ from charmhelpers.core.hookenv import ( log, ) -import six -if six.PY3: - from urllib.parse import urlparse, urlunparse -else: - from urlparse import urlparse, urlunparse +from urllib.parse import urlparse, urlunparse # The order of this list is very important. Handlers should be listed in from @@ -134,14 +130,14 @@ def configure_sources(update=False, sources = safe_load((config(sources_var) or '').strip()) or [] keys = safe_load((config(keys_var) or '').strip()) or None - if isinstance(sources, six.string_types): + if isinstance(sources, str): sources = [sources] if keys is None: for source in sources: add_source(source, None) else: - if isinstance(keys, six.string_types): + if isinstance(keys, str): keys = [keys] if len(sources) != len(keys): diff --git a/charmhelpers/fetch/archiveurl.py b/charmhelpers/fetch/archiveurl.py index d25587a..0e35c90 100644 --- a/charmhelpers/fetch/archiveurl.py +++ b/charmhelpers/fetch/archiveurl.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import contextlib import os import hashlib import re @@ -24,28 +25,21 @@ from charmhelpers.payload.archive import ( get_archive_handler, extract, ) +from charmhelpers.core.hookenv import ( + env_proxy_settings, +) from charmhelpers.core.host import mkdir, check_hash -import six -if six.PY3: - from urllib.request import ( - build_opener, install_opener, urlopen, urlretrieve, - HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, - ) - from urllib.parse import urlparse, urlunparse, parse_qs - from urllib.error import URLError -else: - from urllib import urlretrieve - from urllib2 import ( - build_opener, install_opener, urlopen, - HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, - URLError - ) - from urlparse import urlparse, urlunparse, parse_qs +from urllib.request import ( + build_opener, install_opener, urlopen, urlretrieve, + HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, + ProxyHandler +) +from urllib.parse import urlparse, urlunparse, parse_qs +from urllib.error import URLError def splituser(host): - '''urllib.splituser(), but six's support of this seems broken''' _userprog = re.compile('^(.*)@(.*)$') match = _userprog.match(host) if match: @@ -54,7 +48,6 @@ def splituser(host): def splitpasswd(user): - '''urllib.splitpasswd(), but six's support of this is missing''' _passwdprog = re.compile('^([^:]*):(.*)$', re.S) match = _passwdprog.match(user) if match: @@ -62,6 +55,20 @@ def splitpasswd(user): return user, None +@contextlib.contextmanager +def proxy_env(): + """ + Creates a context which temporarily modifies the proxy settings in os.environ. + """ + restore = {**os.environ} # Copy the current os.environ + juju_proxies = env_proxy_settings() or {} + os.environ.update(**juju_proxies) # Insert or Update the os.environ + yield os.environ + for key in juju_proxies: + del os.environ[key] # remove any keys which were added or updated + os.environ.update(**restore) # restore any original values + + class ArchiveUrlFetchHandler(BaseFetchHandler): """ Handler to download archive files from arbitrary URLs. @@ -92,6 +99,7 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): # propagate all exceptions # URLError, OSError, etc proto, netloc, path, params, query, fragment = urlparse(source) + handlers = [] if proto in ('http', 'https'): auth, barehost = splituser(netloc) if auth is not None: @@ -101,10 +109,13 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): # Realm is set to None in add_password to force the username and password # to be used whatever the realm passman.add_password(None, source, username, password) - authhandler = HTTPBasicAuthHandler(passman) - opener = build_opener(authhandler) - install_opener(opener) - response = urlopen(source) + handlers.append(HTTPBasicAuthHandler(passman)) + + with proxy_env(): + handlers.append(ProxyHandler()) + opener = build_opener(*handlers) + install_opener(opener) + response = urlopen(source) try: with open(dest, 'wb') as dest_file: dest_file.write(response.read()) @@ -150,10 +161,7 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): raise UnhandledSource(e.strerror) options = parse_qs(url_parts.fragment) for key, value in options.items(): - if not six.PY3: - algorithms = hashlib.algorithms - else: - algorithms = hashlib.algorithms_available + algorithms = hashlib.algorithms_available if key in algorithms: if len(value) != 1: raise TypeError( diff --git a/charmhelpers/fetch/centos.py b/charmhelpers/fetch/centos.py index a91dcff..f849201 100644 --- a/charmhelpers/fetch/centos.py +++ b/charmhelpers/fetch/centos.py @@ -15,7 +15,6 @@ import subprocess import os import time -import six import yum from tempfile import NamedTemporaryFile @@ -42,7 +41,7 @@ def install(packages, options=None, fatal=False): if options is not None: cmd.extend(options) cmd.append('install') - if isinstance(packages, six.string_types): + if isinstance(packages, str): cmd.append(packages) else: cmd.extend(packages) @@ -71,7 +70,7 @@ def update(fatal=False): def purge(packages, fatal=False): """Purge one or more packages.""" cmd = ['yum', '--assumeyes', 'remove'] - if isinstance(packages, six.string_types): + if isinstance(packages, str): cmd.append(packages) else: cmd.extend(packages) @@ -83,7 +82,7 @@ def yum_search(packages): """Search for a package.""" output = {} cmd = ['yum', 'search'] - if isinstance(packages, six.string_types): + if isinstance(packages, str): cmd.append(packages) else: cmd.extend(packages) diff --git a/charmhelpers/fetch/python/debug.py b/charmhelpers/fetch/python/debug.py index 757135e..dd5cca8 100644 --- a/charmhelpers/fetch/python/debug.py +++ b/charmhelpers/fetch/python/debug.py @@ -15,8 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import atexit import sys diff --git a/charmhelpers/fetch/python/packages.py b/charmhelpers/fetch/python/packages.py index 6004835..93f1fa3 100644 --- a/charmhelpers/fetch/python/packages.py +++ b/charmhelpers/fetch/python/packages.py @@ -16,7 +16,6 @@ # limitations under the License. import os -import six import subprocess import sys @@ -40,10 +39,7 @@ def pip_execute(*args, **kwargs): from pip import main as _pip_execute except ImportError: apt_update() - if six.PY2: - apt_install('python-pip') - else: - apt_install('python3-pip') + apt_install('python3-pip') from pip import main as _pip_execute _pip_execute(*args, **kwargs) finally: @@ -140,12 +136,8 @@ def pip_list(): def pip_create_virtualenv(path=None): """Create an isolated Python environment.""" - if six.PY2: - apt_install('python-virtualenv') - extra_flags = [] - else: - apt_install(['python3-virtualenv', 'virtualenv']) - extra_flags = ['--python=python3'] + apt_install(['python3-virtualenv', 'virtualenv']) + extra_flags = ['--python=python3'] if path: venv_path = path diff --git a/charmhelpers/fetch/ubuntu.py b/charmhelpers/fetch/ubuntu.py index cf8328f..93b9276 100644 --- a/charmhelpers/fetch/ubuntu.py +++ b/charmhelpers/fetch/ubuntu.py @@ -13,10 +13,8 @@ # limitations under the License. from collections import OrderedDict -import os import platform import re -import six import subprocess import sys import time @@ -224,6 +222,14 @@ CLOUD_ARCHIVE_POCKETS = { 'yoga/proposed': 'focal-proposed/yoga', 'focal-yoga/proposed': 'focal-proposed/yoga', 'focal-proposed/yoga': 'focal-proposed/yoga', + # Zed + 'zed': 'jammy-updates/zed', + 'jammy-zed': 'jammy-updates/zed', + 'jammy-zed/updates': 'jammy-updates/zed', + 'jammy-updates/zed': 'jammy-updates/zed', + 'zed/proposed': 'jammy-proposed/zed', + 'jammy-zed/proposed': 'jammy-proposed/zed', + 'jammy-proposed/zed': 'jammy-proposed/zed', } @@ -250,6 +256,7 @@ OPENSTACK_RELEASES = ( 'wallaby', 'xena', 'yoga', + 'zed', ) @@ -276,6 +283,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('hirsute', 'wallaby'), ('impish', 'xena'), ('jammy', 'yoga'), + ('kinetic', 'zed'), ]) @@ -361,7 +369,7 @@ def apt_install(packages, options=None, fatal=False, quiet=False): cmd = ['apt-get', '--assume-yes'] cmd.extend(options) cmd.append('install') - if isinstance(packages, six.string_types): + if isinstance(packages, str): cmd.append(packages) else: cmd.extend(packages) @@ -413,7 +421,7 @@ def apt_purge(packages, fatal=False): :raises: subprocess.CalledProcessError """ cmd = ['apt-get', '--assume-yes', 'purge'] - if isinstance(packages, six.string_types): + if isinstance(packages, str): cmd.append(packages) else: cmd.extend(packages) @@ -440,7 +448,7 @@ def apt_mark(packages, mark, fatal=False): """Flag one or more packages using apt-mark.""" log("Marking {} as {}".format(packages, mark)) cmd = ['apt-mark', mark] - if isinstance(packages, six.string_types): + if isinstance(packages, str): cmd.append(packages) else: cmd.extend(packages) @@ -485,10 +493,7 @@ def import_key(key): if ('-----BEGIN PGP PUBLIC KEY BLOCK-----' in key and '-----END PGP PUBLIC KEY BLOCK-----' in key): log("Writing provided PGP key in the binary format", level=DEBUG) - if six.PY3: - key_bytes = key.encode('utf-8') - else: - key_bytes = key + key_bytes = key.encode('utf-8') key_name = _get_keyid_by_gpg_key(key_bytes) key_gpg = _dearmor_gpg_key(key_bytes) _write_apt_gpg_keyfile(key_name=key_name, key_material=key_gpg) @@ -528,9 +533,8 @@ def _get_keyid_by_gpg_key(key_material): stderr=subprocess.PIPE, stdin=subprocess.PIPE) out, err = ps.communicate(input=key_material) - if six.PY3: - out = out.decode('utf-8') - err = err.decode('utf-8') + out = out.decode('utf-8') + err = err.decode('utf-8') if 'gpg: no valid OpenPGP data found.' in err: raise GPGKeyError('Invalid GPG key material provided') # from gnupg2 docs: fpr :: Fingerprint (fingerprint is in field 10) @@ -588,8 +592,7 @@ def _dearmor_gpg_key(key_asc): stdin=subprocess.PIPE) out, err = ps.communicate(input=key_asc) # no need to decode output as it is binary (invalid utf-8), only error - if six.PY3: - err = err.decode('utf-8') + err = err.decode('utf-8') if 'gpg: no valid OpenPGP data found.' in err: raise GPGKeyError('Invalid GPG key material. Check your network setup' ' (MTU, routing, DNS) and/or proxy server settings' @@ -693,7 +696,7 @@ def add_source(source, key=None, fail_invalid=False): ]) if source is None: source = '' - for r, fn in six.iteritems(_mapping): + for r, fn in _mapping.items(): m = re.match(r, source) if m: if key: @@ -726,7 +729,7 @@ def _add_proposed(): """ release = get_distrib_codename() arch = platform.machine() - if arch not in six.iterkeys(ARCH_TO_PROPOSED_POCKET): + if arch not in ARCH_TO_PROPOSED_POCKET.keys(): raise SourceConfigError("Arch {} not supported for (distro-)proposed" .format(arch)) with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt: @@ -913,9 +916,8 @@ def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,), kwargs = {} if quiet: - devnull = os.devnull if six.PY2 else subprocess.DEVNULL - kwargs['stdout'] = devnull - kwargs['stderr'] = devnull + kwargs['stdout'] = subprocess.DEVNULL + kwargs['stderr'] = subprocess.DEVNULL if not retry_message: retry_message = "Failed executing '{}'".format(" ".join(cmd)) @@ -957,9 +959,8 @@ def _run_apt_command(cmd, fatal=False, quiet=False): else: kwargs = {} if quiet: - devnull = os.devnull if six.PY2 else subprocess.DEVNULL - kwargs['stdout'] = devnull - kwargs['stderr'] = devnull + kwargs['stdout'] = subprocess.DEVNULL + kwargs['stderr'] = subprocess.DEVNULL subprocess.call(cmd, env=get_apt_dpkg_env(), **kwargs) @@ -989,7 +990,7 @@ def get_installed_version(package): Version object """ cache = apt_cache() - dpkg_result = cache._dpkg_list([package]).get(package, {}) + dpkg_result = cache.dpkg_list([package]).get(package, {}) current_ver = None installed_version = dpkg_result.get('version') diff --git a/charmhelpers/fetch/ubuntu_apt_pkg.py b/charmhelpers/fetch/ubuntu_apt_pkg.py index 436e177..6da355f 100644 --- a/charmhelpers/fetch/ubuntu_apt_pkg.py +++ b/charmhelpers/fetch/ubuntu_apt_pkg.py @@ -40,6 +40,9 @@ import os import subprocess import sys +from charmhelpers import deprecate +from charmhelpers.core.hookenv import log + class _container(dict): """Simple container for attributes.""" @@ -79,7 +82,7 @@ class Cache(object): apt_result = self._apt_cache_show([package])[package] apt_result['name'] = apt_result.pop('package') pkg = Package(apt_result) - dpkg_result = self._dpkg_list([package]).get(package, {}) + dpkg_result = self.dpkg_list([package]).get(package, {}) current_ver = None installed_version = dpkg_result.get('version') if installed_version: @@ -88,9 +91,29 @@ class Cache(object): pkg.architecture = dpkg_result.get('architecture') return pkg + @deprecate("use dpkg_list() instead.", "2022-05", log=log) def _dpkg_list(self, packages): + return self.dpkg_list(packages) + + def dpkg_list(self, packages): """Get data from system dpkg database for package. + Note that this method is also useful for querying package names + containing wildcards, for example + + apt_cache().dpkg_list(['nvidia-vgpu-ubuntu-*']) + + may return + + { + 'nvidia-vgpu-ubuntu-470': { + 'name': 'nvidia-vgpu-ubuntu-470', + 'version': '470.68', + 'architecture': 'amd64', + 'description': 'NVIDIA vGPU driver - version 470.68' + } + } + :param packages: Packages to get data from :type packages: List[str] :returns: Structured data about installed packages, keys like diff --git a/metadata.yaml b/metadata.yaml index d31618f..bb795b7 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -7,7 +7,6 @@ description: | tags: - misc series: -- focal - jammy requires: juju-info: diff --git a/osci.yaml b/osci.yaml index 221eb97..e9641dd 100644 --- a/osci.yaml +++ b/osci.yaml @@ -1,9 +1,9 @@ - project: templates: - - charm-unit-jobs-py38 - charm-unit-jobs-py310 - - charm-yoga-functional-jobs + - charm-zed-functional-jobs vars: needs_charm_build: true charm_build_name: hacluster build_type: charmcraft + charmcraft_channel: 2.0/stable diff --git a/requirements.txt b/requirements.txt index ead6e89..3b1cb7b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,14 +11,19 @@ pbr==5.6.0 simplejson>=2.2.0 netifaces>=0.10.4 +# NOTE: newer versions of cryptography require a Rust compiler to build, +# see +# * https://github.com/openstack-charmers/zaza/issues/421 +# * https://mail.python.org/pipermail/cryptography-dev/2021-January/001003.html +# +cryptography<3.4 + # Strange import error with newer netaddr: netaddr>0.7.16,<0.8.0 Jinja2>=2.6 # BSD License (3 clause) six>=1.9.0 -# dnspython 2.0.0 dropped py3.5 support -dnspython<2.0.0; python_version < '3.6' -dnspython; python_version >= '3.6' +dnspython psutil>=1.1.1,<2.0.0 diff --git a/test-requirements.txt b/test-requirements.txt index 823463d..4ef87dc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,6 @@ # all of its own requirements and if it doesn't, fix it there. # pyparsing<3.0.0 # aodhclient is pinned in zaza and needs pyparsing < 3.0.0, but cffi also needs it, so pin here. -cffi==1.14.6; python_version < '3.6' # cffi 1.15.0 drops support for py35. setuptools<50.0.0 # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85 requests>=2.18.4 @@ -19,26 +18,12 @@ stestr>=2.2.0 # https://github.com/mtreinish/stestr/issues/145 cliff<3.0.0 -# Dependencies of stestr. Newer versions use keywords that didn't exist in -# python 3.5 yet (e.g. "ModuleNotFoundError") -importlib-metadata<3.0.0; python_version < '3.6' -importlib-resources<3.0.0; python_version < '3.6' - -# Some Zuul nodes sometimes pull newer versions of these dependencies which -# dropped support for python 3.5: -osprofiler<2.7.0;python_version<'3.6' -stevedore<1.31.0;python_version<'3.6' -debtcollector<1.22.0;python_version<'3.6' -oslo.utils<=3.41.0;python_version<'3.6' - coverage>=4.5.2 pyudev # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking) git+https://github.com/openstack-charmers/zaza.git#egg=zaza git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack # Needed for charm-glance: -git+https://opendev.org/openstack/tempest.git#egg=tempest;python_version>='3.8' -tempest<31.0.0;python_version<'3.8' -tempest<24.0.0;python_version<'3.6' +git+https://opendev.org/openstack/tempest.git#egg=tempest croniter # needed for charm-rabbitmq-server unit tests diff --git a/tests/bundles/focal-yoga.yaml b/tests/bundles/jammy-zed.yaml similarity index 93% rename from tests/bundles/focal-yoga.yaml rename to tests/bundles/jammy-zed.yaml index 0b270a1..9b1b337 100644 --- a/tests/bundles/focal-yoga.yaml +++ b/tests/bundles/jammy-zed.yaml @@ -1,7 +1,7 @@ variables: - openstack-origin: &openstack-origin cloud:focal-yoga + openstack-origin: &openstack-origin cloud:jammy-zed -series: focal +series: jammy machines: '0': diff --git a/tests/bundles/kinetic-zed.yaml b/tests/bundles/kinetic-zed.yaml new file mode 100644 index 0000000..d74fede --- /dev/null +++ b/tests/bundles/kinetic-zed.yaml @@ -0,0 +1,57 @@ +variables: + openstack-origin: &openstack-origin distro + +series: kinetic + +machines: + '0': + constraints: mem=3072M + '1': + constraints: mem=3072M + '2': + constraints: mem=3072M + '3': + '4': + '5': + +applications: + + keystone-mysql-router: + charm: ch:mysql-router + channel: latest/edge + + mysql-innodb-cluster: + charm: ch:mysql-innodb-cluster + num_units: 3 + options: + source: *openstack-origin + to: + - '0' + - '1' + - '2' + channel: latest/edge + + keystone: + charm: ch:keystone + num_units: 3 + options: + token-expiration: 60 + openstack-origin: *openstack-origin + to: + - '3' + - '4' + - '5' + channel: latest/edge + + hacluster: + charm: ../../hacluster.charm + subordinate-to: + - keystone + +relations: + - - 'keystone:shared-db' + - 'keystone-mysql-router:shared-db' + - - 'hacluster:ha' + - 'keystone:ha' + - - "keystone-mysql-router:db-router" + - "mysql-innodb-cluster:db-router" diff --git a/tests/tests.yaml b/tests/tests.yaml index 49f94be..b11fae9 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -1,13 +1,15 @@ charm_name: hacluster smoke_bundles: - - focal-yoga + - jammy-yoga gate_bundles: - - focal-yoga + - jammy-yoga dev_bundles: - jammy-yoga + - jammy-zed + - kinetic-zed configure: - zaza.openstack.charm_tests.keystone.setup.add_demo_user @@ -21,4 +23,4 @@ tests_options: principle-app-name: keystone hacluster-charm-name: hacluster force_deploy: - - jammy-yoga + - kinetic-zed diff --git a/tox.ini b/tox.ini index acbcb1f..bddbd1f 100644 --- a/tox.ini +++ b/tox.ini @@ -48,34 +48,9 @@ basepython = python3 deps = -r{toxinidir}/build-requirements.txt commands = charmcraft clean - charmcraft -v build + charmcraft -v pack {toxinidir}/rename.sh -[testenv:py35] -basepython = python3.5 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt - -[testenv:py36] -basepython = python3.6 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt - -[testenv:py37] -basepython = python3.7 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt - -[testenv:py38] -basepython = python3.8 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt - -[testenv:py39] -basepython = python3.9 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt - [testenv:py310] basepython = python3.10 deps = -r{toxinidir}/requirements.txt @@ -89,7 +64,7 @@ deps = -r{toxinidir}/requirements.txt [testenv:pep8] basepython = python3 deps = flake8==3.9.2 - charm-tools==2.8.3 + git+https://github.com/juju/charm-tools.git commands = flake8 {posargs} hooks unit_tests tests actions lib files charm-proof