diff --git a/hooks/swift_storage_hooks.py b/hooks/swift_storage_hooks.py index 1e0a3d7..788312f 100755 --- a/hooks/swift_storage_hooks.py +++ b/hooks/swift_storage_hooks.py @@ -39,9 +39,9 @@ _add_path(_root) from lib.swift_storage_utils import ( - PACKAGES, RESTART_MAP, SWIFT_SVCS, + determine_packages, do_openstack_upgrade, ensure_swift_directories, fetch_swift_rings, @@ -103,6 +103,7 @@ from charmhelpers.contrib.openstack.utils import ( is_unit_paused_set, set_unit_paused, set_unit_upgrading, + get_os_codename_install_source, ) from charmhelpers.contrib.network.ip import ( get_relation_ip, @@ -200,10 +201,13 @@ def initialize_ufw(): def install(): status_set('maintenance', 'Executing pre-install') execd_preinstall() - configure_installation_source(config('openstack-origin')) + src = config('openstack-origin') + configure_installation_source(src) status_set('maintenance', 'Installing apt packages') apt_update() - apt_install(PACKAGES, fatal=True) + rel = get_os_codename_install_source(src) + pkgs = determine_packages(rel) + apt_install(pkgs, fatal=True) initialize_ufw() ensure_swift_directories() @@ -262,7 +266,9 @@ def install_vaultlocker(): @harden() def upgrade_charm(): initialize_ufw() - apt_install(filter_installed_packages(PACKAGES), fatal=True) + rel = get_os_codename_install_source(config('openstack-origin')) + pkgs = determine_packages(rel) + apt_install(pkgs, fatal=True) update_nrpe_config() ensure_devs_tracked() diff --git a/lib/swift_storage_utils.py b/lib/swift_storage_utils.py index 7e4bae5..7978506 100644 --- a/lib/swift_storage_utils.py +++ b/lib/swift_storage_utils.py @@ -24,7 +24,11 @@ from lib.swift_storage_context import ( from charmhelpers.fetch import ( apt_upgrade, - apt_update + apt_update, + apt_install, + apt_purge, + apt_autoremove, + filter_missing_packages, ) from charmhelpers.core.unitdata import ( @@ -57,6 +61,7 @@ from charmhelpers.core.hookenv import ( ingress_address, storage_list, storage_get, + status_set, ) from charmhelpers.contrib.network import ufw @@ -77,6 +82,9 @@ from charmhelpers.contrib.openstack.utils import ( get_os_codename_install_source, get_os_codename_package, save_script_rc as _save_script_rc, + CompareOpenStackReleases, + reset_os_release, + os_release, ) from charmhelpers.contrib.openstack import ( @@ -93,9 +101,29 @@ import charmhelpers.contrib.openstack.vaultlocker as vaultlocker from charmhelpers.core.unitdata import kv PACKAGES = [ - 'swift', 'swift-account', 'swift-container', 'swift-object', - 'xfsprogs', 'gdisk', 'lvm2', 'python-jinja2', 'python-psutil', + 'gdisk', + 'lvm2', + 'swift', + 'swift-account', + 'swift-container', + 'swift-object', + 'python-jinja2', + 'python-psutil', 'ufw', + 'xfsprogs', +] + +PY3_PACKAGES = [ + 'python3-jinja2', + 'python3-psutil', + 'python3-six', + 'python3-swift', +] + +PURGE_PACKAGES = [ + 'python-jinja2', + 'python-psutil', + 'python-swift', ] VERSION_PACKAGE = 'swift-account' @@ -163,7 +191,7 @@ def ensure_swift_directories(): def register_configs(): - release = get_os_codename_package('python-swift', fatal=False) or 'essex' + release = get_os_codename_package('swift', fatal=False) or 'essex' configs = templating.OSConfigRenderer(templates_dir=TEMPLATES, openstack_release=release) configs.register('/etc/swift/swift.conf', @@ -181,6 +209,43 @@ def register_configs(): return configs +def determine_packages(release): + """Determine what packages are needed for a given OpenStack release.""" + cmp_openstack = CompareOpenStackReleases(release) + pkgs = PACKAGES[:] + if cmp_openstack >= 'train': + pkgs = [p for p in pkgs if not p.startswith('python-')] + pkgs.extend(PY3_PACKAGES) + return pkgs + + +def determine_purge_packages(): + ''' + Determine list of packages that where previously installed which are no + longer needed. + + :returns: list of package names + ''' + cmp_openstack = CompareOpenStackReleases(os_release('swift')) + if cmp_openstack >= 'train': + return PURGE_PACKAGES + return [] + + +def remove_old_packages(): + '''Purge any packages that need to be removed. + + :returns: bool Whether packages were removed. + ''' + installed_packages = filter_missing_packages(determine_purge_packages()) + if installed_packages: + log('Removing apt packages') + status_set('maintenance', 'Removing apt packages') + apt_purge(installed_packages, fatal=True) + apt_autoremove(purge=True, fatal=True) + return bool(installed_packages) + + def swift_init(target, action, fatal=False): ''' Call swift-init on a specific target with given action, potentially @@ -204,6 +269,10 @@ def do_openstack_upgrade(configs): ] apt_update() apt_upgrade(options=dpkg_opts, fatal=True, dist=True) + reset_os_release() + apt_install(packages=determine_packages(new_os_rel), + options=dpkg_opts, fatal=True) + remove_old_packages() configs.set_release(openstack_release=new_os_rel) configs.write_all() if not is_paused(): diff --git a/tox.ini b/tox.ini index ce4546b..0b32f08 100644 --- a/tox.ini +++ b/tox.ini @@ -19,8 +19,7 @@ setenv = VIRTUAL_ENV={envdir} install_command = pip install {opts} {packages} commands = stestr run --slowest {posargs} -whitelist_externals = juju -passenv = HOME TERM AMULET_* CS_* OS_* TEST_* +passenv = HOME TERM AMULET_* CS_API_* OS_* TEST_* [testenv:py3] basepython = python3 diff --git a/unit_tests/test_swift_storage_relations.py b/unit_tests/test_swift_storage_relations.py index ebc591f..5d1528a 100644 --- a/unit_tests/test_swift_storage_relations.py +++ b/unit_tests/test_swift_storage_relations.py @@ -169,11 +169,12 @@ class SwiftStorageRelationsTests(CharmTestCase): @patch.object(hooks, 'add_ufw_gre_rule', lambda *args: None) @patch.object(hooks, 'ensure_devs_tracked') def test_upgrade_charm(self, mock_ensure_devs_tracked): - self.filter_installed_packages.return_value = [ - 'python-psutil'] hooks.upgrade_charm() - self.apt_install.assert_called_with([ - 'python-psutil'], fatal=True) + self.apt_install.assert_called_with( + ['gdisk', 'lvm2', 'swift', 'swift-account', + 'swift-container', 'swift-object', 'python-jinja2', + 'python-psutil', 'ufw', 'xfsprogs'], + fatal=True) self.assertTrue(self.update_nrpe_config.called) self.assertTrue(mock_ensure_devs_tracked.called) diff --git a/unit_tests/test_swift_storage_utils.py b/unit_tests/test_swift_storage_utils.py index 309e8ab..864865c 100644 --- a/unit_tests/test_swift_storage_utils.py +++ b/unit_tests/test_swift_storage_utils.py @@ -25,6 +25,7 @@ import lib.swift_storage_utils as swift_utils TO_PATCH = [ 'apt_update', 'apt_upgrade', + 'apt_install', 'log', 'config', 'configure_installation_source', @@ -52,6 +53,8 @@ TO_PATCH = [ 'relation_ids', 'vaultlocker', 'kv', + 'reset_os_release', + 'CompareOpenStackReleases', ] @@ -477,13 +480,15 @@ class SwiftStorageUtilsTests(CharmTestCase): ] self.assertEqual(ex, configs.register.call_args_list) - def test_do_upgrade(self): + @patch.object(swift_utils, 'remove_old_packages') + def test_do_upgrade_queens(self, mock_remove_old_packages): self.is_paused.return_value = False - self.test_config.set('openstack-origin', 'cloud:precise-grizzly') - self.get_os_codename_install_source.return_value = 'grizzly' + self.test_config.set('openstack-origin', 'cloud:bionic-queens') + self.get_os_codename_install_source.return_value = 'queens' + self.CompareOpenStackReleases.return_value = 'queens' swift_utils.do_openstack_upgrade(MagicMock()) self.configure_installation_source.assert_called_with( - 'cloud:precise-grizzly' + 'cloud:bionic-queens' ) dpkg_opts = [ '--option', 'Dpkg::Options::=--force-confnew', @@ -494,6 +499,49 @@ class SwiftStorageUtilsTests(CharmTestCase): options=dpkg_opts, fatal=True, dist=True ) + self.apt_install.assert_called_with( + options=dpkg_opts, + packages=['gdisk', 'lvm2', 'swift', 'swift-account', + 'swift-container', 'swift-object', 'python-jinja2', + 'python-psutil', 'ufw', 'xfsprogs'], + fatal=True + ) + self.assertTrue(mock_remove_old_packages.called) + self.assertTrue(self.reset_os_release.called) + services = (swift_utils.ACCOUNT_SVCS + swift_utils.CONTAINER_SVCS + + swift_utils.OBJECT_SVCS) + for service in services: + self.assertIn(call(service), self.service_restart.call_args_list) + + @patch.object(swift_utils, 'remove_old_packages') + def test_do_upgrade_train(self, mock_remove_old_packages): + self.is_paused.return_value = False + self.test_config.set('openstack-origin', 'cloud:bionic-train') + self.get_os_codename_install_source.return_value = 'train' + self.CompareOpenStackReleases.return_value = 'train' + swift_utils.do_openstack_upgrade(MagicMock()) + self.configure_installation_source.assert_called_with( + 'cloud:bionic-train' + ) + dpkg_opts = [ + '--option', 'Dpkg::Options::=--force-confnew', + '--option', 'Dpkg::Options::=--force-confdef', + ] + self.assertTrue(self.apt_update.called) + self.apt_upgrade.assert_called_with( + options=dpkg_opts, + fatal=True, dist=True + ) + self.apt_install.assert_called_with( + options=dpkg_opts, + packages=['gdisk', 'lvm2', 'swift', 'swift-account', + 'swift-container', 'swift-object', 'ufw', 'xfsprogs', + 'python3-jinja2', 'python3-psutil', 'python3-six', + 'python3-swift'], + fatal=True + ) + self.assertTrue(mock_remove_old_packages.called) + self.assertTrue(self.reset_os_release.called) services = (swift_utils.ACCOUNT_SVCS + swift_utils.CONTAINER_SVCS + swift_utils.OBJECT_SVCS) for service in services: