From 7cee61a1933f5beed4309fcf6d1c8f3a734a2123 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 18 Sep 2018 14:19:39 +0000 Subject: [PATCH] py3: Switch to using Python 3 for rocky or later Switch package install to Python 3 for OpenStack Rocky or later. When upgrading, remove any python-* packages that where explicitly installated and then autoremove --purge any dependencies that are no longer required. Change-Id: I120ed44988f9e86a27f464dc879acdc1b8926dd8 --- .../contrib/openstack/amulet/utils.py | 3 ++ charmhelpers/contrib/openstack/context.py | 4 ++ charmhelpers/contrib/openstack/utils.py | 30 +++++++++++- charmhelpers/core/hookenv.py | 3 +- charmhelpers/core/host_factory/ubuntu.py | 1 + charmhelpers/fetch/__init__.py | 2 + charmhelpers/fetch/bzrurl.py | 4 +- charmhelpers/fetch/giturl.py | 4 +- charmhelpers/fetch/ubuntu.py | 20 ++++++++ hooks/ceilometer_hooks.py | 13 +++-- lib/ceilometer_utils.py | 47 ++++++++++++++++--- .../contrib/openstack/amulet/utils.py | 3 ++ tests/charmhelpers/core/hookenv.py | 3 +- .../charmhelpers/core/host_factory/ubuntu.py | 1 + tox.ini | 2 +- unit_tests/test_ceilometer_hooks.py | 13 ++++- unit_tests/test_ceilometer_utils.py | 26 ++++++++++ 17 files changed, 160 insertions(+), 19 deletions(-) diff --git a/charmhelpers/contrib/openstack/amulet/utils.py b/charmhelpers/contrib/openstack/amulet/utils.py index 6637865..936b403 100644 --- a/charmhelpers/contrib/openstack/amulet/utils.py +++ b/charmhelpers/contrib/openstack/amulet/utils.py @@ -1013,6 +1013,9 @@ class OpenStackAmuletUtils(AmuletUtils): cmd, code, output)) amulet.raise_status(amulet.FAIL, msg=msg) + # For mimic ceph osd lspools output + output = output.replace("\n", ",") + # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance, for pool in str(output).split(','): pool_id_name = pool.split(' ') diff --git a/charmhelpers/contrib/openstack/context.py b/charmhelpers/contrib/openstack/context.py index ca91396..3e4e82a 100644 --- a/charmhelpers/contrib/openstack/context.py +++ b/charmhelpers/contrib/openstack/context.py @@ -1519,6 +1519,10 @@ class NeutronAPIContext(OSContextGenerator): 'rel_key': 'enable-qos', 'default': False, }, + 'enable_nsg_logging': { + 'rel_key': 'enable-nsg-logging', + 'default': False, + }, } ctxt = self.get_neutron_options({}) for rid in relation_ids('neutron-plugin-api'): diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index 24f5b80..ae48d6b 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -186,7 +186,7 @@ SWIFT_CODENAMES = OrderedDict([ ('queens', ['2.16.0', '2.17.0']), ('rocky', - ['2.18.0']), + ['2.18.0', '2.19.0']), ]) # >= Liberty version->codename mapping @@ -1733,3 +1733,31 @@ def is_unit_upgrading_set(): return not(not(kv.get('unit-upgrading'))) except Exception: return False + + +def series_upgrade_prepare(pause_unit_helper=None, configs=None): + """ Run common series upgrade prepare tasks. + + :param pause_unit_helper: function: Function to pause unit + :param configs: OSConfigRenderer object: Configurations + :returns None: + """ + set_unit_upgrading() + if pause_unit_helper and configs: + if not is_unit_paused_set(): + pause_unit_helper(configs) + + +def series_upgrade_complete(resume_unit_helper=None, configs=None): + """ Run common series upgrade complete tasks. + + :param resume_unit_helper: function: Function to resume unit + :param configs: OSConfigRenderer object: Configurations + :returns None: + """ + clear_unit_paused() + clear_unit_upgrading() + if configs: + configs.write_all() + if resume_unit_helper: + resume_unit_helper(configs) diff --git a/charmhelpers/core/hookenv.py b/charmhelpers/core/hookenv.py index 6880007..9abf2a4 100644 --- a/charmhelpers/core/hookenv.py +++ b/charmhelpers/core/hookenv.py @@ -48,6 +48,7 @@ INFO = "INFO" DEBUG = "DEBUG" TRACE = "TRACE" MARKER = object() +SH_MAX_ARG = 131071 cache = {} @@ -98,7 +99,7 @@ def log(message, level=None): command += ['-l', level] if not isinstance(message, six.string_types): message = repr(message) - command += [message] + command += [message[:SH_MAX_ARG]] # Missing juju-log should not cause failures in unit tests # Send log output to stderr try: diff --git a/charmhelpers/core/host_factory/ubuntu.py b/charmhelpers/core/host_factory/ubuntu.py index 99451b5..a6d375a 100644 --- a/charmhelpers/core/host_factory/ubuntu.py +++ b/charmhelpers/core/host_factory/ubuntu.py @@ -21,6 +21,7 @@ UBUNTU_RELEASES = ( 'zesty', 'artful', 'bionic', + 'cosmic', ) diff --git a/charmhelpers/fetch/__init__.py b/charmhelpers/fetch/__init__.py index 480a627..8572d34 100644 --- a/charmhelpers/fetch/__init__.py +++ b/charmhelpers/fetch/__init__.py @@ -84,6 +84,7 @@ module = "charmhelpers.fetch.%s" % __platform__ fetch = importlib.import_module(module) filter_installed_packages = fetch.filter_installed_packages +filter_missing_packages = fetch.filter_missing_packages install = fetch.apt_install upgrade = fetch.apt_upgrade update = _fetch_update = fetch.apt_update @@ -96,6 +97,7 @@ if __platform__ == "ubuntu": apt_update = fetch.apt_update apt_upgrade = fetch.apt_upgrade apt_purge = fetch.apt_purge + apt_autoremove = fetch.apt_autoremove apt_mark = fetch.apt_mark apt_hold = fetch.apt_hold apt_unhold = fetch.apt_unhold diff --git a/charmhelpers/fetch/bzrurl.py b/charmhelpers/fetch/bzrurl.py index 07cd029..c4ab3ff 100644 --- a/charmhelpers/fetch/bzrurl.py +++ b/charmhelpers/fetch/bzrurl.py @@ -13,7 +13,7 @@ # limitations under the License. import os -from subprocess import check_call +from subprocess import STDOUT, check_output from charmhelpers.fetch import ( BaseFetchHandler, UnhandledSource, @@ -55,7 +55,7 @@ class BzrUrlFetchHandler(BaseFetchHandler): cmd = ['bzr', 'branch'] cmd += cmd_opts cmd += [source, dest] - check_call(cmd) + check_output(cmd, stderr=STDOUT) def install(self, source, dest=None, revno=None): url_parts = self.parse_url(source) diff --git a/charmhelpers/fetch/giturl.py b/charmhelpers/fetch/giturl.py index 4cf21bc..070ca9b 100644 --- a/charmhelpers/fetch/giturl.py +++ b/charmhelpers/fetch/giturl.py @@ -13,7 +13,7 @@ # limitations under the License. import os -from subprocess import check_call, CalledProcessError +from subprocess import check_output, CalledProcessError, STDOUT from charmhelpers.fetch import ( BaseFetchHandler, UnhandledSource, @@ -50,7 +50,7 @@ class GitUrlFetchHandler(BaseFetchHandler): cmd = ['git', 'clone', source, dest, '--branch', branch] if depth: cmd.extend(['--depth', depth]) - check_call(cmd) + check_output(cmd, stderr=STDOUT) def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) diff --git a/charmhelpers/fetch/ubuntu.py b/charmhelpers/fetch/ubuntu.py index 19aa6ba..ec08cbc 100644 --- a/charmhelpers/fetch/ubuntu.py +++ b/charmhelpers/fetch/ubuntu.py @@ -189,6 +189,18 @@ def filter_installed_packages(packages): return _pkgs +def filter_missing_packages(packages): + """Return a list of packages that are installed. + + :param packages: list of packages to evaluate. + :returns list: Packages that are installed. + """ + return list( + set(packages) - + set(filter_installed_packages(packages)) + ) + + def apt_cache(in_memory=True, progress=None): """Build and return an apt cache.""" from apt import apt_pkg @@ -248,6 +260,14 @@ def apt_purge(packages, fatal=False): _run_apt_command(cmd, fatal) +def apt_autoremove(purge=True, fatal=False): + """Purge one or more packages.""" + cmd = ['apt-get', '--assume-yes', 'autoremove'] + if purge: + cmd.append('--purge') + _run_apt_command(cmd, fatal) + + def apt_mark(packages, mark, fatal=False): """Flag one or more packages using apt-mark.""" log("Marking {} as {}".format(packages, mark)) diff --git a/hooks/ceilometer_hooks.py b/hooks/ceilometer_hooks.py index 65b5407..7f8ca86 100755 --- a/hooks/ceilometer_hooks.py +++ b/hooks/ceilometer_hooks.py @@ -21,8 +21,9 @@ import sys import os from charmhelpers.fetch import ( - apt_install, filter_installed_packages, - apt_update + apt_install, + apt_update, + filter_installed_packages, ) from charmhelpers.core.hookenv import ( open_port, @@ -140,8 +141,12 @@ def db_joined(): def metric_service_joined(): # NOTE(jamespage): gnocchiclient is required to support # the gnocchi event dispatcher - apt_install(filter_installed_packages(['python-gnocchiclient']), - fatal=True) + release = CompareOpenStackReleases( + get_os_codename_install_source(config('openstack-origin'))) + pkgs = ['python-gnocchiclient'] + if release >= 'rocky': + pkgs = ['python3-gnocchiclient'] + apt_install(filter_installed_packages(pkgs), fatal=True) @hooks.hook("amqp-relation-changed", diff --git a/lib/ceilometer_utils.py b/lib/ceilometer_utils.py index 280cd54..08e98dd 100644 --- a/lib/ceilometer_utils.py +++ b/lib/ceilometer_utils.py @@ -54,7 +54,14 @@ from charmhelpers.core.hookenv import ( DEBUG, relation_ids, ) -from charmhelpers.fetch import apt_update, apt_install, apt_upgrade +from charmhelpers.fetch import ( + apt_update, + apt_install, + apt_upgrade, + apt_purge, + apt_autoremove, + filter_missing_packages, +) from charmhelpers.core.host import init_is_systemd from copy import deepcopy @@ -104,6 +111,10 @@ CEILOMETER_BASE_PACKAGES = [ 'python-pymongo', ] +PY3_PACKAGES = [ + 'python3-ceilometer', +] + ICEHOUSE_PACKAGES = [ 'ceilometer-alarm-notifier', 'ceilometer-alarm-evaluator', @@ -361,6 +372,11 @@ def do_openstack_upgrade(configs): options=dpkg_opts, fatal=True) + installed_packages = filter_missing_packages(determine_purge_packages()) + if installed_packages: + apt_purge(installed_packages, fatal=True) + apt_autoremove(purge=True, fatal=True) + # set CONFIGS to load templates from new release configs.set_release(openstack_release=new_os_rel) @@ -397,14 +413,33 @@ def get_packages(): # NOTE(jamespage): @queens ceilometer has no API service, so # no requirement for token caching. if cmp_codename >= 'queens': - return deepcopy(QUEENS_PACKAGES) - - packages = (deepcopy(CEILOMETER_BASE_PACKAGES) + - ceilometer_release_packages()) - packages.extend(token_cache_pkgs(source=config('openstack-origin'))) + packages = deepcopy(QUEENS_PACKAGES) + if cmp_codename >= 'rocky': + packages.extend(PY3_PACKAGES) + else: + packages = (deepcopy(CEILOMETER_BASE_PACKAGES) + + ceilometer_release_packages()) + packages.extend(token_cache_pkgs(source=config('openstack-origin'))) return packages +def determine_purge_packages(): + ''' + Determine list of packages that where previously installed which are no + longer needed. + + :returns: list of package names + ''' + cmp_codename = CompareOpenStackReleases( + get_os_codename_install_source(config('openstack-origin'))) + if cmp_codename >= 'rocky': + pkgs = [p for p in CEILOMETER_BASE_PACKAGES if p.startswith('python-')] + pkgs.append('python-ceilometer') + pkgs.append('python-memcache') + return pkgs + return [] + + def get_shared_secret(): """ Returns the current shared secret for the ceilometer node. If the shared diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index 6637865..936b403 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -1013,6 +1013,9 @@ class OpenStackAmuletUtils(AmuletUtils): cmd, code, output)) amulet.raise_status(amulet.FAIL, msg=msg) + # For mimic ceph osd lspools output + output = output.replace("\n", ",") + # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance, for pool in str(output).split(','): pool_id_name = pool.split(' ') diff --git a/tests/charmhelpers/core/hookenv.py b/tests/charmhelpers/core/hookenv.py index 6880007..9abf2a4 100644 --- a/tests/charmhelpers/core/hookenv.py +++ b/tests/charmhelpers/core/hookenv.py @@ -48,6 +48,7 @@ INFO = "INFO" DEBUG = "DEBUG" TRACE = "TRACE" MARKER = object() +SH_MAX_ARG = 131071 cache = {} @@ -98,7 +99,7 @@ def log(message, level=None): command += ['-l', level] if not isinstance(message, six.string_types): message = repr(message) - command += [message] + command += [message[:SH_MAX_ARG]] # Missing juju-log should not cause failures in unit tests # Send log output to stderr try: diff --git a/tests/charmhelpers/core/host_factory/ubuntu.py b/tests/charmhelpers/core/host_factory/ubuntu.py index 99451b5..a6d375a 100644 --- a/tests/charmhelpers/core/host_factory/ubuntu.py +++ b/tests/charmhelpers/core/host_factory/ubuntu.py @@ -21,6 +21,7 @@ UBUNTU_RELEASES = ( 'zesty', 'artful', 'bionic', + 'cosmic', ) diff --git a/tox.ini b/tox.ini index 930d526..b6e41e7 100644 --- a/tox.ini +++ b/tox.ini @@ -65,7 +65,7 @@ basepython = python2.7 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = - bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-queens --no-destroy + bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-rocky --no-destroy [testenv:func27-dfs] # Charm Functional Test diff --git a/unit_tests/test_ceilometer_hooks.py b/unit_tests/test_ceilometer_hooks.py index 3f2a5a2..6a31adc 100644 --- a/unit_tests/test_ceilometer_hooks.py +++ b/unit_tests/test_ceilometer_hooks.py @@ -462,11 +462,22 @@ class CeilometerHooksTest(CharmTestCase): hooks.hooks.execute(['hooks/ha-relation-changed']) self.assertEqual(mock_keystone_joined.call_count, 1) - def test_metric_service_joined(self): + def test_metric_service_joined_queens(self): self.filter_installed_packages.return_value = ['python-gnocchiclient'] + self.get_os_codename_install_source.return_value = 'queens' hooks.hooks.execute(['hooks/metric-service-relation-joined']) self.filter_installed_packages.assert_called_with( ['python-gnocchiclient'] ) self.apt_install.assert_called_with(['python-gnocchiclient'], fatal=True) + + def test_metric_service_joined_rocky(self): + self.filter_installed_packages.return_value = ['python3-gnocchiclient'] + self.get_os_codename_install_source.return_value = 'rocky' + hooks.hooks.execute(['hooks/metric-service-relation-joined']) + self.filter_installed_packages.assert_called_with( + ['python3-gnocchiclient'] + ) + self.apt_install.assert_called_with(['python3-gnocchiclient'], + fatal=True) diff --git a/unit_tests/test_ceilometer_utils.py b/unit_tests/test_ceilometer_utils.py index 119653d..2e36dc4 100644 --- a/unit_tests/test_ceilometer_utils.py +++ b/unit_tests/test_ceilometer_utils.py @@ -211,6 +211,19 @@ class CeilometerUtilsTest(CharmTestCase): ) self.reset_os_release.assert_called() + def test_determine_purge_packages(self): + 'Ensure no packages are identified for purge prior to rocky' + self.get_os_codename_install_source.return_value = 'queens' + self.assertEqual(utils.determine_purge_packages(), []) + + def test_determine_purge_packages_rocky(self): + 'Ensure python packages are identified for purge at rocky' + self.get_os_codename_install_source.return_value = 'rocky' + self.assertEqual(utils.determine_purge_packages(), + [p for p in utils.CEILOMETER_BASE_PACKAGES + if p.startswith('python-')] + + ['python-ceilometer', 'python-memcache']) + def test_get_packages_icehouse(self): self.get_os_codename_install_source.return_value = 'icehouse' self.token_cache_pkgs.return_value = [] @@ -226,6 +239,19 @@ class CeilometerUtilsTest(CharmTestCase): utils.MITAKA_PACKAGES + ['memcached']) + def test_get_packages_queens(self): + self.get_os_codename_install_source.return_value = 'queens' + self.token_cache_pkgs.return_value = [] + self.assertEqual(utils.get_packages(), + utils.QUEENS_PACKAGES) + + def test_get_packages_rocky(self): + self.get_os_codename_install_source.return_value = 'rocky' + self.token_cache_pkgs.return_value = [] + self.assertEqual(utils.get_packages(), + utils.QUEENS_PACKAGES + + ['python3-ceilometer']) + def test_assess_status(self): with patch.object(utils, 'assess_status_func') as asf: callee = MagicMock()