From 6001f58a48cdb6097e57c692ab2ff0f8c6cda274 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 7 Sep 2018 15:15:43 +0100 Subject: [PATCH] py3: Switch to using Python 3 for rocky or later Switch package install to Python 3 for OpenStack Rocky or later. When upgrading, remove any python-* packages that where explicitly installed and then autoremove --purge any dependencies that are no longer required. Tidy install on ceph relation joined - python-ceph is a dependency of ceph-common, so no need to explicitly install. Change-Id: I7b66bb3ab9f9130c9054411819d51434480cac97 --- .../contrib/openstack/amulet/utils.py | 32 +++++++++++---- charmhelpers/contrib/openstack/context.py | 8 ++++ charmhelpers/contrib/openstack/utils.py | 14 ++++++- charmhelpers/core/hookenv.py | 3 +- charmhelpers/fetch/__init__.py | 2 + charmhelpers/fetch/bzrurl.py | 4 +- charmhelpers/fetch/giturl.py | 4 +- charmhelpers/fetch/ubuntu.py | 20 ++++++++++ hooks/glance_relations.py | 8 +++- hooks/glance_utils.py | 40 ++++++++++++++++++- .../contrib/openstack/amulet/utils.py | 3 ++ tests/charmhelpers/core/hookenv.py | 3 +- tox.ini | 2 +- unit_tests/test_glance_relations.py | 11 ++++- unit_tests/test_glance_utils.py | 26 ++++++++++++ 15 files changed, 160 insertions(+), 20 deletions(-) diff --git a/charmhelpers/contrib/openstack/amulet/utils.py b/charmhelpers/contrib/openstack/amulet/utils.py index ef4ab54b..936b4036 100644 --- a/charmhelpers/contrib/openstack/amulet/utils.py +++ b/charmhelpers/contrib/openstack/amulet/utils.py @@ -24,7 +24,8 @@ import urlparse import cinderclient.v1.client as cinder_client import cinderclient.v2.client as cinder_clientv2 -import glanceclient.v1.client as glance_client +import glanceclient.v1 as glance_client +import glanceclient.v2 as glance_clientv2 import heatclient.v1.client as heat_client from keystoneclient.v2_0 import client as keystone_client from keystoneauth1.identity import ( @@ -623,7 +624,7 @@ class OpenStackAmuletUtils(AmuletUtils): ep = keystone.service_catalog.url_for(service_type='image', interface='adminURL') if keystone.session: - return glance_client.Client(ep, session=keystone.session) + return glance_clientv2.Client("2", session=keystone.session) else: return glance_client.Client(ep, token=keystone.auth_token) @@ -711,10 +712,19 @@ class OpenStackAmuletUtils(AmuletUtils): f.close() # Create glance image - with open(local_path) as f: - image = glance.images.create(name=image_name, is_public=True, - disk_format='qcow2', - container_format='bare', data=f) + if float(glance.version) < 2.0: + with open(local_path) as fimage: + image = glance.images.create(name=image_name, is_public=True, + disk_format='qcow2', + container_format='bare', + data=fimage) + else: + image = glance.images.create( + name=image_name, + disk_format="qcow2", + visibility="public", + container_format="bare") + glance.images.upload(image.id, open(local_path, 'rb')) # Wait for image to reach active status img_id = image.id @@ -729,9 +739,14 @@ class OpenStackAmuletUtils(AmuletUtils): self.log.debug('Validating image attributes...') val_img_name = glance.images.get(img_id).name val_img_stat = glance.images.get(img_id).status - val_img_pub = glance.images.get(img_id).is_public val_img_cfmt = glance.images.get(img_id).container_format val_img_dfmt = glance.images.get(img_id).disk_format + + if float(glance.version) < 2.0: + val_img_pub = glance.images.get(img_id).is_public + else: + val_img_pub = glance.images.get(img_id).visibility == "public" + msg_attr = ('Image attributes - name:{} public:{} id:{} stat:{} ' 'container fmt:{} disk fmt:{}'.format( val_img_name, val_img_pub, img_id, @@ -998,6 +1013,9 @@ class OpenStackAmuletUtils(AmuletUtils): cmd, code, output)) amulet.raise_status(amulet.FAIL, msg=msg) + # For mimic ceph osd lspools output + output = output.replace("\n", ",") + # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance, for pool in str(output).split(','): pool_id_name = pool.split(' ') diff --git a/charmhelpers/contrib/openstack/context.py b/charmhelpers/contrib/openstack/context.py index ca913961..92cb742e 100644 --- a/charmhelpers/contrib/openstack/context.py +++ b/charmhelpers/contrib/openstack/context.py @@ -1519,6 +1519,14 @@ class NeutronAPIContext(OSContextGenerator): 'rel_key': 'enable-qos', 'default': False, }, + 'enable_nsg_logging': { + 'rel_key': 'enable-nsg-logging', + 'default': False, + }, + 'nsg_log_output_base': { + 'rel_key': 'nsg-log-output-base', + 'default': None, + }, } ctxt = self.get_neutron_options({}) for rid in relation_ids('neutron-plugin-api'): diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index bce5b593..ae48d6b4 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -1736,7 +1736,12 @@ def is_unit_upgrading_set(): def series_upgrade_prepare(pause_unit_helper=None, configs=None): - """ Run common series upgrade prepare tasks.""" + """ Run common series upgrade prepare tasks. + + :param pause_unit_helper: function: Function to pause unit + :param configs: OSConfigRenderer object: Configurations + :returns None: + """ set_unit_upgrading() if pause_unit_helper and configs: if not is_unit_paused_set(): @@ -1744,7 +1749,12 @@ def series_upgrade_prepare(pause_unit_helper=None, configs=None): def series_upgrade_complete(resume_unit_helper=None, configs=None): - """ Run common series upgrade complete tasks.""" + """ Run common series upgrade complete tasks. + + :param resume_unit_helper: function: Function to resume unit + :param configs: OSConfigRenderer object: Configurations + :returns None: + """ clear_unit_paused() clear_unit_upgrading() if configs: diff --git a/charmhelpers/core/hookenv.py b/charmhelpers/core/hookenv.py index 68800074..9abf2a45 100644 --- a/charmhelpers/core/hookenv.py +++ b/charmhelpers/core/hookenv.py @@ -48,6 +48,7 @@ INFO = "INFO" DEBUG = "DEBUG" TRACE = "TRACE" MARKER = object() +SH_MAX_ARG = 131071 cache = {} @@ -98,7 +99,7 @@ def log(message, level=None): command += ['-l', level] if not isinstance(message, six.string_types): message = repr(message) - command += [message] + command += [message[:SH_MAX_ARG]] # Missing juju-log should not cause failures in unit tests # Send log output to stderr try: diff --git a/charmhelpers/fetch/__init__.py b/charmhelpers/fetch/__init__.py index 480a6276..8572d34f 100644 --- a/charmhelpers/fetch/__init__.py +++ b/charmhelpers/fetch/__init__.py @@ -84,6 +84,7 @@ module = "charmhelpers.fetch.%s" % __platform__ fetch = importlib.import_module(module) filter_installed_packages = fetch.filter_installed_packages +filter_missing_packages = fetch.filter_missing_packages install = fetch.apt_install upgrade = fetch.apt_upgrade update = _fetch_update = fetch.apt_update @@ -96,6 +97,7 @@ if __platform__ == "ubuntu": apt_update = fetch.apt_update apt_upgrade = fetch.apt_upgrade apt_purge = fetch.apt_purge + apt_autoremove = fetch.apt_autoremove apt_mark = fetch.apt_mark apt_hold = fetch.apt_hold apt_unhold = fetch.apt_unhold diff --git a/charmhelpers/fetch/bzrurl.py b/charmhelpers/fetch/bzrurl.py index 07cd0293..c4ab3ff1 100644 --- a/charmhelpers/fetch/bzrurl.py +++ b/charmhelpers/fetch/bzrurl.py @@ -13,7 +13,7 @@ # limitations under the License. import os -from subprocess import check_call +from subprocess import STDOUT, check_output from charmhelpers.fetch import ( BaseFetchHandler, UnhandledSource, @@ -55,7 +55,7 @@ class BzrUrlFetchHandler(BaseFetchHandler): cmd = ['bzr', 'branch'] cmd += cmd_opts cmd += [source, dest] - check_call(cmd) + check_output(cmd, stderr=STDOUT) def install(self, source, dest=None, revno=None): url_parts = self.parse_url(source) diff --git a/charmhelpers/fetch/giturl.py b/charmhelpers/fetch/giturl.py index 4cf21bc2..070ca9bb 100644 --- a/charmhelpers/fetch/giturl.py +++ b/charmhelpers/fetch/giturl.py @@ -13,7 +13,7 @@ # limitations under the License. import os -from subprocess import check_call, CalledProcessError +from subprocess import check_output, CalledProcessError, STDOUT from charmhelpers.fetch import ( BaseFetchHandler, UnhandledSource, @@ -50,7 +50,7 @@ class GitUrlFetchHandler(BaseFetchHandler): cmd = ['git', 'clone', source, dest, '--branch', branch] if depth: cmd.extend(['--depth', depth]) - check_call(cmd) + check_output(cmd, stderr=STDOUT) def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) diff --git a/charmhelpers/fetch/ubuntu.py b/charmhelpers/fetch/ubuntu.py index 19aa6baf..ec08cbc2 100644 --- a/charmhelpers/fetch/ubuntu.py +++ b/charmhelpers/fetch/ubuntu.py @@ -189,6 +189,18 @@ def filter_installed_packages(packages): return _pkgs +def filter_missing_packages(packages): + """Return a list of packages that are installed. + + :param packages: list of packages to evaluate. + :returns list: Packages that are installed. + """ + return list( + set(packages) - + set(filter_installed_packages(packages)) + ) + + def apt_cache(in_memory=True, progress=None): """Build and return an apt cache.""" from apt import apt_pkg @@ -248,6 +260,14 @@ def apt_purge(packages, fatal=False): _run_apt_command(cmd, fatal) +def apt_autoremove(purge=True, fatal=False): + """Purge one or more packages.""" + cmd = ['apt-get', '--assume-yes', 'autoremove'] + if purge: + cmd.append('--purge') + _run_apt_command(cmd, fatal) + + def apt_mark(packages, mark, fatal=False): """Flag one or more packages using apt-mark.""" log("Marking {} as {}".format(packages, mark)) diff --git a/hooks/glance_relations.py b/hooks/glance_relations.py index fafe0ce8..a810819b 100755 --- a/hooks/glance_relations.py +++ b/hooks/glance_relations.py @@ -105,6 +105,7 @@ from charmhelpers.contrib.openstack.utils import ( os_requires_version, series_upgrade_prepare, series_upgrade_complete, + CompareOpenStackReleases, ) from charmhelpers.contrib.storage.linux.ceph import ( send_request_if_needed, @@ -263,7 +264,7 @@ def object_store_joined(): @hooks.hook('ceph-relation-joined') def ceph_joined(): - apt_install(['ceph-common', 'python-ceph']) + apt_install(['ceph-common']) def get_ceph_request(): @@ -586,7 +587,10 @@ def install_packages_for_cinder_store(): optional_packages = ["python-cinderclient", "python-os-brick", "python-oslo.rootwrap"] - apt_install(filter_installed_packages(optional_packages), fatal=True) + release = os_release('glance-common') + cmp_release = CompareOpenStackReleases(release) + if cmp_release < 'rocky': + apt_install(filter_installed_packages(optional_packages), fatal=True) @hooks.hook('cinder-volume-service-relation-joined') diff --git a/hooks/glance_utils.py b/hooks/glance_utils.py index 6be9f6fa..ccdec8f5 100644 --- a/hooks/glance_utils.py +++ b/hooks/glance_utils.py @@ -27,7 +27,10 @@ from charmhelpers.fetch import ( apt_upgrade, apt_update, apt_install, - add_source) + add_source, + apt_autoremove, + apt_purge, + filter_missing_packages) from charmhelpers.core.hookenv import ( config, @@ -87,6 +90,16 @@ PACKAGES = [ "apache2", "glance", "python-mysqldb", "python-swiftclient", "python-psycopg2", "python-keystone", "python-six", "uuid", "haproxy", ] +PY3_PACKAGES = [ + "python3-glance", + "python3-rados", + "python3-rbd", + "python3-swiftclient", + "python3-cinderclient", + "python3-os-brick", + "python3-oslo.rootwrap", +] + VERSION_PACKAGE = 'glance-common' SERVICES = [ @@ -247,9 +260,29 @@ def register_configs(): def determine_packages(): packages = set(PACKAGES) packages |= set(token_cache_pkgs(source=config('openstack-origin'))) + if CompareOpenStackReleases(os_release(VERSION_PACKAGE)) >= 'rocky': + packages = [p for p in packages if not p.startswith('python-')] + packages.extend(PY3_PACKAGES) return sorted(packages) +def determine_purge_packages(): + ''' + Determine list of packages that where previously installed which are no + longer needed. + + :returns: list of package names + ''' + if CompareOpenStackReleases(os_release('cinder')) >= 'rocky': + pkgs = [p for p in PACKAGES if p.startswith('python-')] + pkgs.append('python-glance') + pkgs.append('python-memcache') + pkgs.extend(["python-cinderclient", + "python-os-brick", + "python-oslo.rootwrap"]) + return [] + + # NOTE(jamespage): Retry deals with sync issues during one-shot HA deploys. # mysql might be restarting or suchlike. @retry_on_exception(5, base_delay=3, exc_type=subprocess.CalledProcessError) @@ -284,6 +317,11 @@ def do_openstack_upgrade(configs): reset_os_release() apt_install(determine_packages(), fatal=True) + installed_packages = filter_missing_packages(determine_purge_packages()) + if installed_packages: + apt_purge(installed_packages, fatal=True) + apt_autoremove(purge=True, fatal=True) + # set CONFIGS to load templates from new release and regenerate config configs.set_release(openstack_release=new_os_rel) configs.write_all() diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index 6637865d..936b4036 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -1013,6 +1013,9 @@ class OpenStackAmuletUtils(AmuletUtils): cmd, code, output)) amulet.raise_status(amulet.FAIL, msg=msg) + # For mimic ceph osd lspools output + output = output.replace("\n", ",") + # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance, for pool in str(output).split(','): pool_id_name = pool.split(' ') diff --git a/tests/charmhelpers/core/hookenv.py b/tests/charmhelpers/core/hookenv.py index 68800074..9abf2a45 100644 --- a/tests/charmhelpers/core/hookenv.py +++ b/tests/charmhelpers/core/hookenv.py @@ -48,6 +48,7 @@ INFO = "INFO" DEBUG = "DEBUG" TRACE = "TRACE" MARKER = object() +SH_MAX_ARG = 131071 cache = {} @@ -98,7 +99,7 @@ def log(message, level=None): command += ['-l', level] if not isinstance(message, six.string_types): message = repr(message) - command += [message] + command += [message[:SH_MAX_ARG]] # Missing juju-log should not cause failures in unit tests # Send log output to stderr try: diff --git a/tox.ini b/tox.ini index 7201b237..be3f7e91 100644 --- a/tox.ini +++ b/tox.ini @@ -67,7 +67,7 @@ basepython = python2.7 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = - bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-queens --no-destroy + bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-rocky --no-destroy [testenv:func27-dfs] # Charm Functional Test diff --git a/unit_tests/test_glance_relations.py b/unit_tests/test_glance_relations.py index 88971705..e77c48e0 100644 --- a/unit_tests/test_glance_relations.py +++ b/unit_tests/test_glance_relations.py @@ -112,6 +112,7 @@ class GlanceRelationTests(CharmTestCase): super(GlanceRelationTests, self).setUp(relations, TO_PATCH) self.config.side_effect = self.test_config.get self.restart_on_change.return_value = None + self.os_release.return_value = 'icehouse' def test_install_hook(self): repo = 'cloud:precise-grizzly' @@ -297,7 +298,7 @@ class GlanceRelationTests(CharmTestCase): def test_ceph_joined(self): relations.ceph_joined() - self.apt_install.assert_called_with(['ceph-common', 'python-ceph']) + self.apt_install.assert_called_with(['ceph-common']) @patch.object(relations, 'CONFIGS') def test_ceph_changed_missing_relation_data(self, configs): @@ -815,6 +816,14 @@ class GlanceRelationTests(CharmTestCase): "python-oslo.rootwrap"], fatal=True ) + @patch.object(relations, 'CONFIGS') + def test_cinder_volume_joined_rocky(self, configs): + self.filter_installed_packages.side_effect = lambda pkgs: pkgs + self.os_release.return_value = 'rocky' + relations.cinder_volume_service_relation_joined() + self.assertTrue(configs.write_all.called) + self.apt_install.assert_not_called() + @patch.object(relations, 'CONFIGS') def test_storage_backend_changed(self, configs): self.filter_installed_packages.side_effect = lambda pkgs: pkgs diff --git a/unit_tests/test_glance_utils.py b/unit_tests/test_glance_utils.py index 6cc79cca..83f45622 100644 --- a/unit_tests/test_glance_utils.py +++ b/unit_tests/test_glance_utils.py @@ -36,6 +36,9 @@ TO_PATCH = [ 'apt_update', 'apt_upgrade', 'apt_install', + 'apt_purge', + 'apt_autoremove', + 'filter_missing_packages', 'mkdir', 'os_release', 'service_start', @@ -161,6 +164,7 @@ class TestGlanceUtils(CharmTestCase): @patch.object(utils, 'token_cache_pkgs') def test_determine_packages(self, token_cache_pkgs): self.config.side_effect = None + self.os_release.return_value = 'queens' token_cache_pkgs.return_value = [] ex = utils.PACKAGES self.assertEqual(set(ex), set(utils.determine_packages())) @@ -172,6 +176,7 @@ class TestGlanceUtils(CharmTestCase): def test_openstack_upgrade_leader(self, migrate): self.config.side_effect = None self.config.return_value = 'cloud:precise-havana' + self.os_release.return_value = 'havana' self.is_elected_leader.return_value = True self.get_os_codename_install_source.return_value = 'havana' configs = MagicMock() @@ -184,10 +189,31 @@ class TestGlanceUtils(CharmTestCase): configs.set_release.assert_called_with(openstack_release='havana') self.assertTrue(migrate.called) + @patch.object(utils, 'migrate_database') + def test_openstack_upgrade_rocky(self, migrate): + self.config.side_effect = None + self.config.return_value = 'cloud:bionic-rocky' + self.os_release.return_value = 'rocky' + self.is_elected_leader.return_value = True + self.get_os_codename_install_source.return_value = 'rocky' + self.filter_missing_packages.return_value = ['python-glance'] + configs = MagicMock() + utils.do_openstack_upgrade(configs) + self.assertTrue(configs.write_all.called) + self.apt_install.assert_called_with(utils.determine_packages(), + fatal=True) + self.apt_upgrade.assert_called_with(options=DPKG_OPTS, + fatal=True, dist=True) + self.apt_purge.assert_called_with(['python-glance'], fatal=True) + self.apt_autoremove.assert_called_with(purge=True, fatal=True) + configs.set_release.assert_called_with(openstack_release='rocky') + self.assertTrue(migrate.called) + @patch.object(utils, 'migrate_database') def test_openstack_upgrade_not_leader(self, migrate): self.config.side_effect = None self.config.return_value = 'cloud:precise-havana' + self.os_release.return_value = 'havana' self.is_elected_leader.return_value = False self.get_os_codename_install_source.return_value = 'havana' configs = MagicMock()