From b4efe9dc4b7b7b723472ea4583fe25ca198ee8ab Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 23 Sep 2021 14:36:57 +0100 Subject: [PATCH] Add xena bundles - add non-voting focal-xena bundle - add non-voting impish-xena bundle - charm-helpers sync for new charm-helpers changes - update tox/pip.sh to ensure setuptools<50.0.0 Change-Id: I1f99e0d8e70ce29b647707f7e3b53a59ba93fe05 --- charmhelpers/contrib/charmsupport/nrpe.py | 5 +- charmhelpers/contrib/charmsupport/volumes.py | 4 +- charmhelpers/contrib/hahelpers/cluster.py | 6 +- charmhelpers/contrib/network/ip.py | 4 +- .../contrib/openstack/deferred_events.py | 4 +- .../openstack/files/policy_rc_d_script.py | 2 +- charmhelpers/contrib/openstack/policyd.py | 6 +- charmhelpers/contrib/openstack/utils.py | 71 ++------ charmhelpers/contrib/storage/linux/ceph.py | 15 +- charmhelpers/contrib/storage/linux/lvm.py | 4 +- charmhelpers/core/hookenv.py | 11 +- charmhelpers/core/host.py | 12 +- charmhelpers/core/host_factory/ubuntu.py | 1 + charmhelpers/core/strutils.py | 9 +- charmhelpers/core/unitdata.py | 6 +- charmhelpers/fetch/__init__.py | 6 +- charmhelpers/fetch/python/packages.py | 10 +- charmhelpers/fetch/snap.py | 4 +- charmhelpers/fetch/ubuntu.py | 166 +++++++++++++++++- charmhelpers/fetch/ubuntu_apt_pkg.py | 6 +- charmhelpers/osplatform.py | 3 + osci.yaml | 6 + pip.sh | 18 ++ tests/bundles/focal-xena.yaml | 54 ++++++ tests/bundles/impish-xena.yaml | 54 ++++++ tests/tests.yaml | 9 +- tox.ini | 13 +- 27 files changed, 380 insertions(+), 129 deletions(-) create mode 100755 pip.sh create mode 100644 tests/bundles/focal-xena.yaml create mode 100644 tests/bundles/impish-xena.yaml diff --git a/charmhelpers/contrib/charmsupport/nrpe.py b/charmhelpers/contrib/charmsupport/nrpe.py index e4cb06b..8d1753c 100644 --- a/charmhelpers/contrib/charmsupport/nrpe.py +++ b/charmhelpers/contrib/charmsupport/nrpe.py @@ -1,4 +1,4 @@ -# Copyright 2014-2015 Canonical Limited. +# Copyright 2012-2021 Canonical Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,6 @@ # limitations under the License. """Compatibility with the nrpe-external-master charm""" -# Copyright 2012 Canonical Ltd. # # Authors: # Matthew Wedgwood @@ -511,7 +510,7 @@ def add_haproxy_checks(nrpe, unit_name): def remove_deprecated_check(nrpe, deprecated_services): """ - Remove checks fro deprecated services in list + Remove checks for deprecated services in list :param nrpe: NRPE object to remove check from :type nrpe: NRPE diff --git a/charmhelpers/contrib/charmsupport/volumes.py b/charmhelpers/contrib/charmsupport/volumes.py index 7ea43f0..f7c6fbd 100644 --- a/charmhelpers/contrib/charmsupport/volumes.py +++ b/charmhelpers/contrib/charmsupport/volumes.py @@ -1,4 +1,4 @@ -# Copyright 2014-2015 Canonical Limited. +# Copyright 2014-2021 Canonical Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ Configuration stanzas:: type: boolean default: true description: > - If false, a volume is mounted as sepecified in "volume-map" + If false, a volume is mounted as specified in "volume-map" If true, ephemeral storage will be used, meaning that log data will only exist as long as the machine. YOU HAVE BEEN WARNED. volume-map: diff --git a/charmhelpers/contrib/hahelpers/cluster.py b/charmhelpers/contrib/hahelpers/cluster.py index ba34fba..f0b629a 100644 --- a/charmhelpers/contrib/hahelpers/cluster.py +++ b/charmhelpers/contrib/hahelpers/cluster.py @@ -1,4 +1,4 @@ -# Copyright 2014-2015 Canonical Limited. +# Copyright 2014-2021 Canonical Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -86,7 +86,7 @@ def is_elected_leader(resource): 2. If the charm is part of a corosync cluster, call corosync to determine leadership. 3. If the charm is not part of a corosync cluster, the leader is - determined as being "the alive unit with the lowest unit numer". In + determined as being "the alive unit with the lowest unit number". In other words, the oldest surviving unit. """ try: @@ -418,7 +418,7 @@ def get_managed_services_and_ports(services, external_ports, Return only the services and corresponding ports that are managed by this charm. This excludes haproxy when there is a relation with hacluster. This - is because this charm passes responsability for stopping and starting + is because this charm passes responsibility for stopping and starting haproxy to hacluster. Similarly, if a relation with hacluster exists then the ports returned by diff --git a/charmhelpers/contrib/network/ip.py b/charmhelpers/contrib/network/ip.py index 63e91cc..b356d64 100644 --- a/charmhelpers/contrib/network/ip.py +++ b/charmhelpers/contrib/network/ip.py @@ -1,4 +1,4 @@ -# Copyright 2014-2015 Canonical Limited. +# Copyright 2014-2021 Canonical Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -578,7 +578,7 @@ def get_relation_ip(interface, cidr_network=None): @returns IPv6 or IPv4 address """ # Select the interface address first - # For possible use as a fallback bellow with get_address_in_network + # For possible use as a fallback below with get_address_in_network try: # Get the interface specific IP address = network_get_primary_address(interface) diff --git a/charmhelpers/contrib/openstack/deferred_events.py b/charmhelpers/contrib/openstack/deferred_events.py index 8765ee3..94eacf6 100644 --- a/charmhelpers/contrib/openstack/deferred_events.py +++ b/charmhelpers/contrib/openstack/deferred_events.py @@ -244,7 +244,7 @@ def get_deferred_restarts(): def clear_deferred_restarts(services): - """Clear deferred restart events targetted at `services`. + """Clear deferred restart events targeted at `services`. :param services: Services with deferred actions to clear. :type services: List[str] @@ -253,7 +253,7 @@ def clear_deferred_restarts(services): def process_svc_restart(service): - """Respond to a service restart having occured. + """Respond to a service restart having occurred. :param service: Services that the action was performed against. :type service: str diff --git a/charmhelpers/contrib/openstack/files/policy_rc_d_script.py b/charmhelpers/contrib/openstack/files/policy_rc_d_script.py index 344a766..431e972 100755 --- a/charmhelpers/contrib/openstack/files/policy_rc_d_script.py +++ b/charmhelpers/contrib/openstack/files/policy_rc_d_script.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -"""This script is an implemenation of policy-rc.d +"""This script is an implementation of policy-rc.d For further information on policy-rc.d see *1 diff --git a/charmhelpers/contrib/openstack/policyd.py b/charmhelpers/contrib/openstack/policyd.py index f2bb21e..6fa06f2 100644 --- a/charmhelpers/contrib/openstack/policyd.py +++ b/charmhelpers/contrib/openstack/policyd.py @@ -1,4 +1,4 @@ -# Copyright 2019 Canonical Ltd +# Copyright 2019-2021 Canonical Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -59,7 +59,7 @@ provided: The functions should be called from the install and upgrade hooks in the charm. The `maybe_do_policyd_overrides_on_config_changed` function is designed to be called on the config-changed hook, in that it does an additional check to -ensure that an already overriden policy.d in an upgrade or install hooks isn't +ensure that an already overridden policy.d in an upgrade or install hooks isn't repeated. In order the *enable* this functionality, the charm's install, config_changed, @@ -334,7 +334,7 @@ def maybe_do_policyd_overrides(openstack_release, restart_handler() -@charmhelpers.deprecate("Use maybe_do_poliyd_overrrides instead") +@charmhelpers.deprecate("Use maybe_do_policyd_overrides instead") def maybe_do_policyd_overrides_on_config_changed(*args, **kwargs): """This function is designed to be called from the config changed hook. diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index 1656bd4..d5d301e 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -1,4 +1,4 @@ -# Copyright 2014-2015 Canonical Limited. +# Copyright 2014-2021 Canonical Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -106,6 +106,8 @@ from charmhelpers.fetch import ( filter_installed_packages, filter_missing_packages, ubuntu_apt_pkg as apt, + OPENSTACK_RELEASES, + UBUNTU_OPENSTACK_RELEASE, ) from charmhelpers.fetch.snap import ( @@ -132,54 +134,9 @@ CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA' DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed ' 'restricted main multiverse universe') -OPENSTACK_RELEASES = ( - 'diablo', - 'essex', - 'folsom', - 'grizzly', - 'havana', - 'icehouse', - 'juno', - 'kilo', - 'liberty', - 'mitaka', - 'newton', - 'ocata', - 'pike', - 'queens', - 'rocky', - 'stein', - 'train', - 'ussuri', - 'victoria', - 'wallaby', -) - -UBUNTU_OPENSTACK_RELEASE = OrderedDict([ - ('oneiric', 'diablo'), - ('precise', 'essex'), - ('quantal', 'folsom'), - ('raring', 'grizzly'), - ('saucy', 'havana'), - ('trusty', 'icehouse'), - ('utopic', 'juno'), - ('vivid', 'kilo'), - ('wily', 'liberty'), - ('xenial', 'mitaka'), - ('yakkety', 'newton'), - ('zesty', 'ocata'), - ('artful', 'pike'), - ('bionic', 'queens'), - ('cosmic', 'rocky'), - ('disco', 'stein'), - ('eoan', 'train'), - ('focal', 'ussuri'), - ('groovy', 'victoria'), - ('hirsute', 'wallaby'), -]) - - OPENSTACK_CODENAMES = OrderedDict([ + # NOTE(lourot): 'yyyy.i' isn't actually mapping with any real version + # number. This just means the i-th version of the year yyyy. ('2011.2', 'diablo'), ('2012.1', 'essex'), ('2012.2', 'folsom'), @@ -200,6 +157,8 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2020.1', 'ussuri'), ('2020.2', 'victoria'), ('2021.1', 'wallaby'), + ('2021.2', 'xena'), + ('2022.1', 'yoga'), ]) # The ugly duckling - must list releases oldest to newest @@ -701,7 +660,7 @@ def import_key(keyid): def get_source_and_pgp_key(source_and_key): """Look for a pgp key ID or ascii-armor key in the given input. - :param source_and_key: Sting, "source_spec|keyid" where '|keyid' is + :param source_and_key: String, "source_spec|keyid" where '|keyid' is optional. :returns (source_spec, key_id OR None) as a tuple. Returns None for key_id if there was no '|' in the source_and_key string. @@ -721,7 +680,7 @@ def configure_installation_source(source_plus_key): The functionality is provided by charmhelpers.fetch.add_source() The difference between the two functions is that add_source() signature requires the key to be passed directly, whereas this function passes an - optional key by appending '|' to the end of the source specificiation + optional key by appending '|' to the end of the source specification 'source'. Another difference from add_source() is that the function calls sys.exit(1) @@ -808,7 +767,7 @@ def get_endpoint_notifications(service_names, rel_name='identity-service'): def endpoint_changed(service_name, rel_name='identity-service'): - """Whether a new notification has been recieved for an endpoint. + """Whether a new notification has been received for an endpoint. :param service_name: Service name eg nova, neutron, placement etc :type service_name: str @@ -834,7 +793,7 @@ def endpoint_changed(service_name, rel_name='identity-service'): def save_endpoint_changed_triggers(service_names, rel_name='identity-service'): - """Save the enpoint triggers in db so it can be tracked if they changed. + """Save the endpoint triggers in db so it can be tracked if they changed. :param service_names: List of service name. :type service_name: List @@ -1502,9 +1461,9 @@ def remote_restart(rel_name, remote_service=None): if remote_service: trigger['remote-service'] = remote_service for rid in relation_ids(rel_name): - # This subordinate can be related to two seperate services using + # This subordinate can be related to two separate services using # different subordinate relations so only issue the restart if - # the principle is conencted down the relation we think it is + # the principle is connected down the relation we think it is if related_units(relid=rid): relation_set(relation_id=rid, relation_settings=trigger, @@ -1621,7 +1580,7 @@ def manage_payload_services(action, services=None, charm_func=None): """Run an action against all services. An optional charm_func() can be called. It should raise an Exception to - indicate that the function failed. If it was succesfull it should return + indicate that the function failed. If it was successful it should return None or an optional message. The signature for charm_func is: @@ -1880,7 +1839,7 @@ def pausable_restart_on_change(restart_map, stopstart=False, :param post_svc_restart_f: A function run after a service has restarted. :type post_svc_restart_f: Callable[[str], None] - :param pre_restarts_wait_f: A function callled before any restarts. + :param pre_restarts_wait_f: A function called before any restarts. :type pre_restarts_wait_f: Callable[None, None] :returns: decorator to use a restart_on_change with pausability :rtype: decorator diff --git a/charmhelpers/contrib/storage/linux/ceph.py b/charmhelpers/contrib/storage/linux/ceph.py index d1c6175..3eb46d7 100644 --- a/charmhelpers/contrib/storage/linux/ceph.py +++ b/charmhelpers/contrib/storage/linux/ceph.py @@ -1,4 +1,4 @@ -# Copyright 2014-2015 Canonical Limited. +# Copyright 2014-2021 Canonical Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# -# Copyright 2012 Canonical Ltd. -# # This file is sourced from lp:openstack-charm-helpers # # Authors: @@ -605,7 +602,7 @@ class BasePool(object): class Pool(BasePool): - """Compability shim for any descendents external to this library.""" + """Compatibility shim for any descendents external to this library.""" @deprecate( 'The ``Pool`` baseclass has been replaced by ``BasePool`` class.') @@ -1535,7 +1532,7 @@ def map_block_storage(service, pool, image): def filesystem_mounted(fs): - """Determine whether a filesytems is already mounted.""" + """Determine whether a filesystem is already mounted.""" return fs in [f for f, m in mounts()] @@ -1904,7 +1901,7 @@ class CephBrokerRq(object): set the ceph-mon unit handling the broker request will set its default value. :type erasure_profile: str - :param allow_ec_overwrites: allow EC pools to be overriden + :param allow_ec_overwrites: allow EC pools to be overridden :type allow_ec_overwrites: bool :raises: AssertionError if provided data is of invalid type/range """ @@ -1949,7 +1946,7 @@ class CephBrokerRq(object): :param lrc_locality: Group the coding and data chunks into sets of size locality (lrc plugin) :type lrc_locality: int - :param durability_estimator: The number of parity chuncks each of which includes + :param durability_estimator: The number of parity chunks each of which includes a data chunk in its calculation range (shec plugin) :type durability_estimator: int :param helper_chunks: The number of helper chunks to use for recovery operations @@ -2327,7 +2324,7 @@ class CephOSDConfContext(CephConfContext): settings are in conf['osd_from_client'] and finally settings which do clash are in conf['osd_from_client_conflict']. Rather than silently drop the conflicting settings they are provided in the context so they can be - rendered commented out to give some visability to the admin. + rendered commented out to give some visibility to the admin. """ def __init__(self, permitted_sections=None): diff --git a/charmhelpers/contrib/storage/linux/lvm.py b/charmhelpers/contrib/storage/linux/lvm.py index c8bde69..d0a5721 100644 --- a/charmhelpers/contrib/storage/linux/lvm.py +++ b/charmhelpers/contrib/storage/linux/lvm.py @@ -1,4 +1,4 @@ -# Copyright 2014-2015 Canonical Limited. +# Copyright 2014-2021 Canonical Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ from subprocess import ( ################################################## def deactivate_lvm_volume_group(block_device): ''' - Deactivate any volume gruop associated with an LVM physical volume. + Deactivate any volume group associated with an LVM physical volume. :param block_device: str: Full path to LVM physical volume ''' diff --git a/charmhelpers/core/hookenv.py b/charmhelpers/core/hookenv.py index 47eebb5..e94247a 100644 --- a/charmhelpers/core/hookenv.py +++ b/charmhelpers/core/hookenv.py @@ -1,4 +1,4 @@ -# Copyright 2014-2015 Canonical Limited. +# Copyright 2013-2021 Canonical Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,6 @@ # limitations under the License. "Interactions with the Juju environment" -# Copyright 2013 Canonical Ltd. # # Authors: # Charm Helpers Developers @@ -610,7 +609,7 @@ def expected_related_units(reltype=None): relation_type())) :param reltype: Relation type to list data for, default is to list data for - the realtion type we are currently executing a hook for. + the relation type we are currently executing a hook for. :type reltype: str :returns: iterator :rtype: types.GeneratorType @@ -627,7 +626,7 @@ def expected_related_units(reltype=None): @cached def relation_for_unit(unit=None, rid=None): - """Get the json represenation of a unit's relation""" + """Get the json representation of a unit's relation""" unit = unit or remote_unit() relation = relation_get(unit=unit, rid=rid) for key in relation: @@ -1614,11 +1613,11 @@ def env_proxy_settings(selected_settings=None): def _contains_range(addresses): """Check for cidr or wildcard domain in a string. - Given a string comprising a comma seperated list of ip addresses + Given a string comprising a comma separated list of ip addresses and domain names, determine whether the string contains IP ranges or wildcard domains. - :param addresses: comma seperated list of domains and ip addresses. + :param addresses: comma separated list of domains and ip addresses. :type addresses: str """ return ( diff --git a/charmhelpers/core/host.py b/charmhelpers/core/host.py index d25e6c5..994ec8a 100644 --- a/charmhelpers/core/host.py +++ b/charmhelpers/core/host.py @@ -1,4 +1,4 @@ -# Copyright 2014-2015 Canonical Limited. +# Copyright 2014-2021 Canonical Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -217,7 +217,7 @@ def service_resume(service_name, init_dir="/etc/init", initd_dir="/etc/init.d", **kwargs): """Resume a system service. - Reenable starting again at boot. Start the service. + Re-enable starting again at boot. Start the service. :param service_name: the name of the service to resume :param init_dir: the path to the init dir @@ -727,7 +727,7 @@ class restart_on_change(object): :param post_svc_restart_f: A function run after a service has restarted. :type post_svc_restart_f: Callable[[str], None] - :param pre_restarts_wait_f: A function callled before any restarts. + :param pre_restarts_wait_f: A function called before any restarts. :type pre_restarts_wait_f: Callable[None, None] """ self.restart_map = restart_map @@ -828,7 +828,7 @@ def restart_on_change_helper(lambda_f, restart_map, stopstart=False, :param post_svc_restart_f: A function run after a service has restarted. :type post_svc_restart_f: Callable[[str], None] - :param pre_restarts_wait_f: A function callled before any restarts. + :param pre_restarts_wait_f: A function called before any restarts. :type pre_restarts_wait_f: Callable[None, None] :returns: result of lambda_f() :rtype: ANY @@ -880,7 +880,7 @@ def _post_restart_on_change_helper(checksums, :param post_svc_restart_f: A function run after a service has restarted. :type post_svc_restart_f: Callable[[str], None] - :param pre_restarts_wait_f: A function callled before any restarts. + :param pre_restarts_wait_f: A function called before any restarts. :type pre_restarts_wait_f: Callable[None, None] """ if restart_functions is None: @@ -914,7 +914,7 @@ def _post_restart_on_change_helper(checksums, def pwgen(length=None): - """Generate a random pasword.""" + """Generate a random password.""" if length is None: # A random length is ok to use a weak PRNG length = random.choice(range(35, 45)) diff --git a/charmhelpers/core/host_factory/ubuntu.py b/charmhelpers/core/host_factory/ubuntu.py index 5aa4196..e710c0e 100644 --- a/charmhelpers/core/host_factory/ubuntu.py +++ b/charmhelpers/core/host_factory/ubuntu.py @@ -28,6 +28,7 @@ UBUNTU_RELEASES = ( 'focal', 'groovy', 'hirsute', + 'impish', ) diff --git a/charmhelpers/core/strutils.py b/charmhelpers/core/strutils.py index e8df045..28c6b3f 100644 --- a/charmhelpers/core/strutils.py +++ b/charmhelpers/core/strutils.py @@ -18,8 +18,11 @@ import six import re +TRUTHY_STRINGS = {'y', 'yes', 'true', 't', 'on'} +FALSEY_STRINGS = {'n', 'no', 'false', 'f', 'off'} -def bool_from_string(value): + +def bool_from_string(value, truthy_strings=TRUTHY_STRINGS, falsey_strings=FALSEY_STRINGS, assume_false=False): """Interpret string value as boolean. Returns True if value translates to True otherwise False. @@ -32,9 +35,9 @@ def bool_from_string(value): value = value.strip().lower() - if value in ['y', 'yes', 'true', 't', 'on']: + if value in truthy_strings: return True - elif value in ['n', 'no', 'false', 'f', 'off']: + elif value in falsey_strings or assume_false: return False msg = "Unable to interpret string value '%s' as boolean" % (value) diff --git a/charmhelpers/core/unitdata.py b/charmhelpers/core/unitdata.py index ab55432..d9b8d0b 100644 --- a/charmhelpers/core/unitdata.py +++ b/charmhelpers/core/unitdata.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# Copyright 2014-2015 Canonical Limited. +# Copyright 2014-2021 Canonical Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -61,7 +61,7 @@ Here's a fully worked integration example using hookenv.Hooks:: 'previous value', prev, 'current value', cur) - # Get some unit specific bookeeping + # Get some unit specific bookkeeping if not db.get('pkg_key'): key = urllib.urlopen('https://example.com/pkg_key').read() db.set('pkg_key', key) @@ -449,7 +449,7 @@ class HookData(object): 'previous value', prev, 'current value', cur) - # Get some unit specific bookeeping + # Get some unit specific bookkeeping if not db.get('pkg_key'): key = urllib.urlopen('https://example.com/pkg_key').read() db.set('pkg_key', key) diff --git a/charmhelpers/fetch/__init__.py b/charmhelpers/fetch/__init__.py index 5b689f5..9497ee0 100644 --- a/charmhelpers/fetch/__init__.py +++ b/charmhelpers/fetch/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2014-2015 Canonical Limited. +# Copyright 2014-2021 Canonical Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -106,6 +106,8 @@ if __platform__ == "ubuntu": apt_pkg = fetch.ubuntu_apt_pkg get_apt_dpkg_env = fetch.get_apt_dpkg_env get_installed_version = fetch.get_installed_version + OPENSTACK_RELEASES = fetch.OPENSTACK_RELEASES + UBUNTU_OPENSTACK_RELEASE = fetch.UBUNTU_OPENSTACK_RELEASE elif __platform__ == "centos": yum_search = fetch.yum_search @@ -203,7 +205,7 @@ def plugins(fetch_handlers=None): classname) plugin_list.append(handler_class()) except NotImplementedError: - # Skip missing plugins so that they can be ommitted from + # Skip missing plugins so that they can be omitted from # installation if desired log("FetchHandler {} not found, skipping plugin".format( handler_name)) diff --git a/charmhelpers/fetch/python/packages.py b/charmhelpers/fetch/python/packages.py index 6e95028..6004835 100644 --- a/charmhelpers/fetch/python/packages.py +++ b/charmhelpers/fetch/python/packages.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # coding: utf-8 -# Copyright 2014-2015 Canonical Limited. +# Copyright 2014-2021 Canonical Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ __author__ = "Jorge Niedbalski " def pip_execute(*args, **kwargs): - """Overriden pip_execute() to stop sys.path being changed. + """Overridden pip_execute() to stop sys.path being changed. The act of importing main from the pip module seems to cause add wheels from the /usr/share/python-wheels which are installed by various tools. @@ -142,8 +142,10 @@ def pip_create_virtualenv(path=None): """Create an isolated Python environment.""" if six.PY2: apt_install('python-virtualenv') + extra_flags = [] else: - apt_install('python3-virtualenv') + apt_install(['python3-virtualenv', 'virtualenv']) + extra_flags = ['--python=python3'] if path: venv_path = path @@ -151,4 +153,4 @@ def pip_create_virtualenv(path=None): venv_path = os.path.join(charm_dir(), 'venv') if not os.path.exists(venv_path): - subprocess.check_call(['virtualenv', venv_path]) + subprocess.check_call(['virtualenv', venv_path] + extra_flags) diff --git a/charmhelpers/fetch/snap.py b/charmhelpers/fetch/snap.py index fc70aa9..36d6bce 100644 --- a/charmhelpers/fetch/snap.py +++ b/charmhelpers/fetch/snap.py @@ -1,4 +1,4 @@ -# Copyright 2014-2017 Canonical Limited. +# Copyright 2014-2021 Canonical Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -65,7 +65,7 @@ def _snap_exec(commands): retry_count += + 1 if retry_count > SNAP_NO_LOCK_RETRY_COUNT: raise CouldNotAcquireLockException( - 'Could not aquire lock after {} attempts' + 'Could not acquire lock after {} attempts' .format(SNAP_NO_LOCK_RETRY_COUNT)) return_code = e.returncode log('Snap failed to acquire lock, trying again in {} seconds.' diff --git a/charmhelpers/fetch/ubuntu.py b/charmhelpers/fetch/ubuntu.py index 812a11a..6c7cf6f 100644 --- a/charmhelpers/fetch/ubuntu.py +++ b/charmhelpers/fetch/ubuntu.py @@ -1,4 +1,4 @@ -# Copyright 2014-2015 Canonical Limited. +# Copyright 2014-2021 Canonical Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -208,12 +208,79 @@ CLOUD_ARCHIVE_POCKETS = { 'wallaby/proposed': 'focal-proposed/wallaby', 'focal-wallaby/proposed': 'focal-proposed/wallaby', 'focal-proposed/wallaby': 'focal-proposed/wallaby', + # Xena + 'xena': 'focal-updates/xena', + 'focal-xena': 'focal-updates/xena', + 'focal-xena/updates': 'focal-updates/xena', + 'focal-updates/xena': 'focal-updates/xena', + 'xena/proposed': 'focal-proposed/xena', + 'focal-xena/proposed': 'focal-proposed/xena', + 'focal-proposed/xena': 'focal-proposed/xena', + # Yoga + 'yoga': 'focal-updates/yoga', + 'focal-yoga': 'focal-updates/yoga', + 'focal-yoga/updates': 'focal-updates/yoga', + 'focal-updates/yoga': 'focal-updates/yoga', + 'yoga/proposed': 'focal-proposed/yoga', + 'focal-yoga/proposed': 'focal-proposed/yoga', + 'focal-proposed/yoga': 'focal-proposed/yoga', } +OPENSTACK_RELEASES = ( + 'diablo', + 'essex', + 'folsom', + 'grizzly', + 'havana', + 'icehouse', + 'juno', + 'kilo', + 'liberty', + 'mitaka', + 'newton', + 'ocata', + 'pike', + 'queens', + 'rocky', + 'stein', + 'train', + 'ussuri', + 'victoria', + 'wallaby', + 'xena', + 'yoga', +) + + +UBUNTU_OPENSTACK_RELEASE = OrderedDict([ + ('oneiric', 'diablo'), + ('precise', 'essex'), + ('quantal', 'folsom'), + ('raring', 'grizzly'), + ('saucy', 'havana'), + ('trusty', 'icehouse'), + ('utopic', 'juno'), + ('vivid', 'kilo'), + ('wily', 'liberty'), + ('xenial', 'mitaka'), + ('yakkety', 'newton'), + ('zesty', 'ocata'), + ('artful', 'pike'), + ('bionic', 'queens'), + ('cosmic', 'rocky'), + ('disco', 'stein'), + ('eoan', 'train'), + ('focal', 'ussuri'), + ('groovy', 'victoria'), + ('hirsute', 'wallaby'), + ('impish', 'xena'), +]) + + APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries. -CMD_RETRY_COUNT = 3 # Retry a failing fatal command X times. +CMD_RETRY_COUNT = 10 # Retry a failing fatal command X times. def filter_installed_packages(packages): @@ -246,9 +313,9 @@ def filter_missing_packages(packages): def apt_cache(*_, **__): """Shim returning an object simulating the apt_pkg Cache. - :param _: Accept arguments for compability, not used. + :param _: Accept arguments for compatibility, not used. :type _: any - :param __: Accept keyword arguments for compability, not used. + :param __: Accept keyword arguments for compatibility, not used. :type __: any :returns:Object used to interrogate the system apt and dpkg databases. :rtype:ubuntu_apt_pkg.Cache @@ -283,7 +350,7 @@ def apt_install(packages, options=None, fatal=False, quiet=False): :param fatal: Whether the command's output should be checked and retried. :type fatal: bool - :param quiet: if True (default), supress log message to stdout/stderr + :param quiet: if True (default), suppress log message to stdout/stderr :type quiet: bool :raises: subprocess.CalledProcessError """ @@ -397,7 +464,7 @@ def import_key(key): A Radix64 format keyid is also supported for backwards compatibility. In this case Ubuntu keyserver will be queried for a key via HTTPS by its keyid. This method - is less preferrable because https proxy servers may + is less preferable because https proxy servers may require traffic decryption which is equivalent to a man-in-the-middle attack (a proxy server impersonates keyserver TLS certificates and has to be explicitly @@ -574,6 +641,10 @@ def add_source(source, key=None, fail_invalid=False): with be used. If staging is NOT used then the cloud archive [3] will be added, and the 'ubuntu-cloud-keyring' package will be added for the current distro. + '': translate to cloud: based on the current + distro version (i.e. for 'ussuri' this will either be 'bionic-ussuri' or + 'distro'. + '/proposed': as above, but for proposed. Otherwise the source is not recognised and this is logged to the juju log. However, no error is raised, unless sys_error_on_exit is True. @@ -592,7 +663,7 @@ def add_source(source, key=None, fail_invalid=False): id may also be used, but be aware that only insecure protocols are available to retrieve the actual public key from a public keyserver placing your Juju environment at risk. ppa and cloud archive keys - are securely added automtically, so sould not be provided. + are securely added automatically, so should not be provided. @param fail_invalid: (boolean) if True, then the function raises a SourceConfigError is there is no matching installation source. @@ -600,6 +671,12 @@ def add_source(source, key=None, fail_invalid=False): @raises SourceConfigError() if for cloud:, the is not a valid pocket in CLOUD_ARCHIVE_POCKETS """ + # extract the OpenStack versions from the CLOUD_ARCHIVE_POCKETS; can't use + # the list in contrib.openstack.utils as it might not be included in + # classic charms and would break everything. Having OpenStack specific + # code in this file is a bit of an antipattern, anyway. + os_versions_regex = "({})".format("|".join(OPENSTACK_RELEASES)) + _mapping = OrderedDict([ (r"^distro$", lambda: None), # This is a NOP (r"^(?:proposed|distro-proposed)$", _add_proposed), @@ -609,6 +686,9 @@ def add_source(source, key=None, fail_invalid=False): (r"^cloud:(.*)-(.*)$", _add_cloud_distro_check), (r"^cloud:(.*)$", _add_cloud_pocket), (r"^snap:.*-(.*)-(.*)$", _add_cloud_distro_check), + (r"^{}\/proposed$".format(os_versions_regex), + _add_bare_openstack_proposed), + (r"^{}$".format(os_versions_regex), _add_bare_openstack), ]) if source is None: source = '' @@ -640,7 +720,7 @@ def _add_proposed(): Uses get_distrib_codename to determine the correct stanza for the deb line. - For intel architecutres PROPOSED_POCKET is used for the release, but for + For Intel architectures PROPOSED_POCKET is used for the release, but for other architectures PROPOSED_PORTS_POCKET is used for the release. """ release = get_distrib_codename() @@ -662,7 +742,8 @@ def _add_apt_repository(spec): series = get_distrib_codename() spec = spec.replace('{series}', series) _run_with_retries(['add-apt-repository', '--yes', spec], - cmd_env=env_proxy_settings(['https', 'http'])) + cmd_env=env_proxy_settings(['https', 'http', 'no_proxy']) + ) def _add_cloud_pocket(pocket): @@ -738,6 +819,73 @@ def _verify_is_ubuntu_rel(release, os_release): 'version ({})'.format(release, os_release, ubuntu_rel)) +def _add_bare_openstack(openstack_release): + """Add cloud or distro based on the release given. + + The spec given is, say, 'ussuri', but this could apply cloud:bionic-ussuri + or 'distro' depending on whether the ubuntu release is bionic or focal. + + :param openstack_release: the OpenStack codename to determine the release + for. + :type openstack_release: str + :raises: SourceConfigError + """ + # TODO(ajkavanagh) - surely this means we should be removing cloud archives + # if they exist? + __add_bare_helper(openstack_release, "{}-{}", lambda: None) + + +def _add_bare_openstack_proposed(openstack_release): + """Add cloud of distro but with proposed. + + The spec given is, say, 'ussuri' but this could apply + cloud:bionic-ussuri/proposed or 'distro/proposed' depending on whether the + ubuntu release is bionic or focal. + + :param openstack_release: the OpenStack codename to determine the release + for. + :type openstack_release: str + :raises: SourceConfigError + """ + __add_bare_helper(openstack_release, "{}-{}/proposed", _add_proposed) + + +def __add_bare_helper(openstack_release, pocket_format, final_function): + """Helper for _add_bare_openstack[_proposed] + + The bulk of the work between the two functions is exactly the same except + for the pocket format and the function that is run if it's the distro + version. + + :param openstack_release: the OpenStack codename. e.g. ussuri + :type openstack_release: str + :param pocket_format: the pocket formatter string to construct a pocket str + from the openstack_release and the current ubuntu version. + :type pocket_format: str + :param final_function: the function to call if it is the distro version. + :type final_function: Callable + :raises SourceConfigError on error + """ + ubuntu_version = get_distrib_codename() + possible_pocket = pocket_format.format(ubuntu_version, openstack_release) + if possible_pocket in CLOUD_ARCHIVE_POCKETS: + _add_cloud_pocket(possible_pocket) + return + # Otherwise it's almost certainly the distro version; verify that it + # exists. + try: + assert UBUNTU_OPENSTACK_RELEASE[ubuntu_version] == openstack_release + except KeyError: + raise SourceConfigError( + "Invalid ubuntu version {} isn't known to this library" + .format(ubuntu_version)) + except AssertionError: + raise SourceConfigError( + 'Invalid OpenStack release specified: {} for Ubuntu version {}' + .format(openstack_release, ubuntu_version)) + final_function() + + def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,), retry_message="", cmd_env=None, quiet=False): """Run a command and retry until success or max_retries is reached. diff --git a/charmhelpers/fetch/ubuntu_apt_pkg.py b/charmhelpers/fetch/ubuntu_apt_pkg.py index a2fbe0e..436e177 100644 --- a/charmhelpers/fetch/ubuntu_apt_pkg.py +++ b/charmhelpers/fetch/ubuntu_apt_pkg.py @@ -1,4 +1,4 @@ -# Copyright 2019 Canonical Ltd +# Copyright 2019-2021 Canonical Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -209,7 +209,7 @@ sys.modules[__name__].config = Config() def init(): - """Compability shim that does nothing.""" + """Compatibility shim that does nothing.""" pass @@ -264,7 +264,7 @@ def version_compare(a, b): else: raise RuntimeError('Unable to compare "{}" and "{}", according to ' 'our logic they are neither greater, equal nor ' - 'less than each other.') + 'less than each other.'.format(a, b)) class PkgVersion(): diff --git a/charmhelpers/osplatform.py b/charmhelpers/osplatform.py index 78c81af..1ace468 100644 --- a/charmhelpers/osplatform.py +++ b/charmhelpers/osplatform.py @@ -28,6 +28,9 @@ def get_platform(): elif "elementary" in current_platform: # ElementaryOS fails to run tests locally without this. return "ubuntu" + elif "Pop!_OS" in current_platform: + # Pop!_OS also fails to run tests locally without this. + return "ubuntu" else: raise RuntimeError("This module is not supported on {}." .format(current_platform)) diff --git a/osci.yaml b/osci.yaml index 6a399fa..4e322d5 100644 --- a/osci.yaml +++ b/osci.yaml @@ -2,3 +2,9 @@ templates: - charm-unit-jobs - charm-functional-jobs + check: + jobs: + - focal-xena: + voting: false + - impish-xena: + voting: false diff --git a/pip.sh b/pip.sh new file mode 100755 index 0000000..9a7e6b0 --- /dev/null +++ b/pip.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# +# This file is managed centrally by release-tools and should not be modified +# within individual charm repos. See the 'global' dir contents for available +# choices of tox.ini for OpenStack Charms: +# https://github.com/openstack-charmers/release-tools +# +# setuptools 58.0 dropped the support for use_2to3=true which is needed to +# install blessings (an indirect dependency of charm-tools). +# +# More details on the beahvior of tox and virtualenv creation can be found at +# https://github.com/tox-dev/tox/issues/448 +# +# This script is wrapper to force the use of the pinned versions early in the +# process when the virtualenv was created and upgraded before installing the +# depedencies declared in the target. +pip install 'pip<20.3' 'setuptools<50.0.0' +pip "$@" diff --git a/tests/bundles/focal-xena.yaml b/tests/bundles/focal-xena.yaml new file mode 100644 index 0000000..6a31bcc --- /dev/null +++ b/tests/bundles/focal-xena.yaml @@ -0,0 +1,54 @@ +variables: + openstack-origin: &openstack-origin cloud:focal-xena + +series: focal + +machines: + '0': + constraints: mem=3072M + '1': + constraints: mem=3072M + '2': + constraints: mem=3072M + '3': + '4': + '5': + +applications: + + keystone-mysql-router: + charm: cs:~openstack-charmers-next/mysql-router + + mysql-innodb-cluster: + charm: cs:~openstack-charmers-next/mysql-innodb-cluster + num_units: 3 + options: + source: *openstack-origin + to: + - '0' + - '1' + - '2' + + keystone: + charm: cs:~openstack-charmers-next/keystone + num_units: 3 + options: + token-expiration: 60 + openstack-origin: *openstack-origin + to: + - '3' + - '4' + - '5' + + hacluster: + charm: ../../../hacluster + subordinate-to: + - keystone + +relations: + - - 'keystone:shared-db' + - 'keystone-mysql-router:shared-db' + - - 'hacluster:ha' + - 'keystone:ha' + - - "keystone-mysql-router:db-router" + - "mysql-innodb-cluster:db-router" diff --git a/tests/bundles/impish-xena.yaml b/tests/bundles/impish-xena.yaml new file mode 100644 index 0000000..23cc10b --- /dev/null +++ b/tests/bundles/impish-xena.yaml @@ -0,0 +1,54 @@ +variables: + openstack-origin: &openstack-origin distro + +series: impish + +machines: + '0': + constraints: mem=3072M + '1': + constraints: mem=3072M + '2': + constraints: mem=3072M + '3': + '4': + '5': + +applications: + + keystone-mysql-router: + charm: cs:~openstack-charmers-next/mysql-router + + mysql-innodb-cluster: + charm: cs:~openstack-charmers-next/mysql-innodb-cluster + num_units: 3 + options: + source: *openstack-origin + to: + - '0' + - '1' + - '2' + + keystone: + charm: cs:~openstack-charmers-next/keystone + num_units: 3 + options: + token-expiration: 60 + openstack-origin: *openstack-origin + to: + - '3' + - '4' + - '5' + + hacluster: + charm: ../../../hacluster + subordinate-to: + - keystone + +relations: + - - 'keystone:shared-db' + - 'keystone-mysql-router:shared-db' + - - 'hacluster:ha' + - 'keystone:ha' + - - "keystone-mysql-router:db-router" + - "mysql-innodb-cluster:db-router" diff --git a/tests/tests.yaml b/tests/tests.yaml index 5483735..fe7f640 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -12,7 +12,7 @@ gate_bundles: - focal-ussuri - focal-victoria - focal-wallaby - - groovy-victoria + - hirsute-wallaby dev_bundles: - trusty-mitaka @@ -20,7 +20,9 @@ dev_bundles: - xenial-pike - xenial-queens - bionic-rocky - - hirsute-wallaby + - groovy-victoria + - focal-xena + - impish-xena configure: - zaza.openstack.charm_tests.keystone.setup.add_demo_user @@ -34,4 +36,5 @@ tests_options: principle-app-name: keystone hacluster-charm-name: hacluster force_deploy: - - hirsute-wallaby + - groovy-victoria + - impish-xena diff --git a/tox.ini b/tox.ini index 9ba3f9f..ba4fd5b 100644 --- a/tox.ini +++ b/tox.ini @@ -22,19 +22,22 @@ skip_missing_interpreters = False # * It is also necessary to pin virtualenv as a newer virtualenv would still # lead to fetching the latest pip in the func* tox targets, see # https://stackoverflow.com/a/38133283 -requires = pip < 20.3 - virtualenv < 20.0 +requires = + pip < 20.3 + virtualenv < 20.0 + setuptools < 50.0.0 + # NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci -minversion = 3.2.0 +minversion = 3.18.0 [testenv] setenv = VIRTUAL_ENV={envdir} PYTHONHASHSEED=0 CHARM_DIR={envdir} install_command = - pip install {opts} {packages} + {toxinidir}/pip.sh install {opts} {packages} commands = stestr run --slowest {posargs} -whitelist_externals = juju +allowlist_externals = juju passenv = HOME TERM CS_* OS_* TEST_* deps = -r{toxinidir}/test-requirements.txt