From 49d377bfea0a0217bea23de4f8871050295d9425 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 18 Sep 2018 15:59:40 +0200 Subject: [PATCH] py3: Switch to Python 3 for OpenStack Rocky Switch neutron installation to use Python 3 for OpenStack Rocky. Purge python- packages on upgrade. Fix duplicate keystone entry in api-paste.ini for Rocky. Change-Id: I9ead4d0b637f3067e0aa9a20604b2738221860df --- hooks/charmhelpers/contrib/openstack/utils.py | 20 ++++- .../charmhelpers/core/host_factory/ubuntu.py | 1 + hooks/charmhelpers/fetch/__init__.py | 1 + hooks/charmhelpers/fetch/bzrurl.py | 4 +- hooks/charmhelpers/fetch/giturl.py | 4 +- hooks/charmhelpers/fetch/ubuntu.py | 12 +++ hooks/neutron_api_utils.py | 87 +++++++++++++------ templates/rocky/api-paste.ini | 1 - .../charmhelpers/core/host_factory/ubuntu.py | 1 + tox.ini | 2 +- unit_tests/test_neutron_api_utils.py | 25 ++++++ 11 files changed, 120 insertions(+), 38 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 61bc48c0..ae48d6b4 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -1736,7 +1736,12 @@ def is_unit_upgrading_set(): def series_upgrade_prepare(pause_unit_helper=None, configs=None): - """ Run common series upgrade prepare tasks.""" + """ Run common series upgrade prepare tasks. + + :param pause_unit_helper: function: Function to pause unit + :param configs: OSConfigRenderer object: Configurations + :returns None: + """ set_unit_upgrading() if pause_unit_helper and configs: if not is_unit_paused_set(): @@ -1744,8 +1749,15 @@ def series_upgrade_prepare(pause_unit_helper=None, configs=None): def series_upgrade_complete(resume_unit_helper=None, configs=None): - """ Run common series upgrade complete tasks.""" + """ Run common series upgrade complete tasks. + + :param resume_unit_helper: function: Function to resume unit + :param configs: OSConfigRenderer object: Configurations + :returns None: + """ clear_unit_paused() clear_unit_upgrading() - if resume_unit_helper and configs: - resume_unit_helper(configs) + if configs: + configs.write_all() + if resume_unit_helper: + resume_unit_helper(configs) diff --git a/hooks/charmhelpers/core/host_factory/ubuntu.py b/hooks/charmhelpers/core/host_factory/ubuntu.py index 99451b59..a6d375af 100644 --- a/hooks/charmhelpers/core/host_factory/ubuntu.py +++ b/hooks/charmhelpers/core/host_factory/ubuntu.py @@ -21,6 +21,7 @@ UBUNTU_RELEASES = ( 'zesty', 'artful', 'bionic', + 'cosmic', ) diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index f26140dc..8572d34f 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -84,6 +84,7 @@ module = "charmhelpers.fetch.%s" % __platform__ fetch = importlib.import_module(module) filter_installed_packages = fetch.filter_installed_packages +filter_missing_packages = fetch.filter_missing_packages install = fetch.apt_install upgrade = fetch.apt_upgrade update = _fetch_update = fetch.apt_update diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py index 07cd0293..c4ab3ff1 100644 --- a/hooks/charmhelpers/fetch/bzrurl.py +++ b/hooks/charmhelpers/fetch/bzrurl.py @@ -13,7 +13,7 @@ # limitations under the License. import os -from subprocess import check_call +from subprocess import STDOUT, check_output from charmhelpers.fetch import ( BaseFetchHandler, UnhandledSource, @@ -55,7 +55,7 @@ class BzrUrlFetchHandler(BaseFetchHandler): cmd = ['bzr', 'branch'] cmd += cmd_opts cmd += [source, dest] - check_call(cmd) + check_output(cmd, stderr=STDOUT) def install(self, source, dest=None, revno=None): url_parts = self.parse_url(source) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 4cf21bc2..070ca9bb 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -13,7 +13,7 @@ # limitations under the License. import os -from subprocess import check_call, CalledProcessError +from subprocess import check_output, CalledProcessError, STDOUT from charmhelpers.fetch import ( BaseFetchHandler, UnhandledSource, @@ -50,7 +50,7 @@ class GitUrlFetchHandler(BaseFetchHandler): cmd = ['git', 'clone', source, dest, '--branch', branch] if depth: cmd.extend(['--depth', depth]) - check_call(cmd) + check_output(cmd, stderr=STDOUT) def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py index dfce214c..ec08cbc2 100644 --- a/hooks/charmhelpers/fetch/ubuntu.py +++ b/hooks/charmhelpers/fetch/ubuntu.py @@ -189,6 +189,18 @@ def filter_installed_packages(packages): return _pkgs +def filter_missing_packages(packages): + """Return a list of packages that are installed. + + :param packages: list of packages to evaluate. + :returns list: Packages that are installed. + """ + return list( + set(packages) - + set(filter_installed_packages(packages)) + ) + + def apt_cache(in_memory=True, progress=None): """Build and return an apt cache.""" from apt import apt_pkg diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index fb70401e..b76c064c 100755 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -59,7 +59,10 @@ from charmhelpers.fetch import ( apt_update, apt_install, apt_upgrade, - add_source + add_source, + filter_missing_packages, + apt_purge, + apt_autoremove, ) from charmhelpers.core.host import ( @@ -99,23 +102,31 @@ KILO_PACKAGES = [ 'python-neutron-vpnaas', ] -VERSION_PACKAGE = 'neutron-common' - -BASE_GIT_PACKAGES = [ - 'libffi-dev', - 'libmysqlclient-dev', - 'libssl-dev', - 'libxml2-dev', - 'libxslt1-dev', - 'libyaml-dev', - 'openstack-pkg-tools', - 'python-dev', - 'python-neutronclient', # required for get_neutron_client() import - 'python-pip', - 'python-setuptools', - 'zlib1g-dev', +PY3_PACKAGES = [ + 'python3-neutron', + 'python3-neutron-lbaas', + 'python3-neutron-fwaas', + 'python3-neutron-dynamic-routing', + 'python3-networking-hyperv', + 'python3-memcache', ] +PURGE_PACKAGES = [ + 'python-neutron', + 'python-neutron-lbaas', + 'python-neutron-fwaas', + 'python-neutron-vpnaas', + 'python-neutron-dynamic-routing', + 'python-networking-hyperv', + 'python-memcache', + 'python-keystoneclient', + 'python-mysqldb', + 'python-psycopg2', + 'python-six', +] + +VERSION_PACKAGE = 'neutron-common' + BASE_SERVICES = [ 'neutron-server' ] @@ -372,7 +383,11 @@ def manage_plugin(): def determine_packages(source=None): # currently all packages match service names - packages = [] + BASE_PACKAGES + release = get_os_codename_install_source(source) + cmp_release = CompareOpenStackReleases(release) + packages = deepcopy(BASE_PACKAGES) + if cmp_release >= 'rocky': + packages.extend(PY3_PACKAGES) for v in resource_map().values(): packages.extend(v['services']) @@ -382,26 +397,37 @@ def determine_packages(source=None): 'neutron') packages.extend(pkgs) - release = get_os_codename_install_source(source) + packages.extend(token_cache_pkgs(release=release)) - if CompareOpenStackReleases(release) >= 'kilo': - packages.extend(KILO_PACKAGES) - if CompareOpenStackReleases(release) >= 'ocata': - packages.append('python-neutron-dynamic-routing') - if CompareOpenStackReleases(release) >= 'pike': - packages.remove('python-neutron-vpnaas') + if cmp_release < 'rocky': + if cmp_release >= 'kilo': + packages.extend(KILO_PACKAGES) + if cmp_release >= 'ocata': + packages.append('python-neutron-dynamic-routing') + if cmp_release >= 'pike': + packages.remove('python-neutron-vpnaas') - if release == 'kilo' or CompareOpenStackReleases(release) >= 'mitaka': - packages.append('python-networking-hyperv') + if release == 'kilo' or cmp_release >= 'mitaka': + packages.append('python-networking-hyperv') if config('neutron-plugin') == 'vsp': nuage_pkgs = config('nuage-packages').split() - packages += nuage_pkgs + packages.extend(nuage_pkgs) + + if cmp_release >= 'rocky': + packages = [p for p in packages if not p.startswith('python-')] - packages.extend(token_cache_pkgs(release=release)) return list(set(packages)) +def determine_purge_packages(): + '''Return a list of packages to purge for the current OS release''' + cmp_os_source = CompareOpenStackReleases(os_release('neutron-common')) + if cmp_os_source >= 'rocky': + return PURGE_PACKAGES + return [] + + def determine_ports(): '''Assemble a list of API ports for services we are managing''' ports = [] @@ -524,6 +550,11 @@ def do_openstack_upgrade(configs): options=dpkg_opts, fatal=True) + installed_packages = filter_missing_packages(determine_purge_packages()) + if installed_packages: + apt_purge(installed_packages, fatal=True) + apt_autoremove(purge=True, fatal=True) + # set CONFIGS to load templates from new release configs.set_release(openstack_release=new_os_rel) # Before kilo it's nova-cloud-controllers job diff --git a/templates/rocky/api-paste.ini b/templates/rocky/api-paste.ini index 5ff3da0d..e45e7a9e 100644 --- a/templates/rocky/api-paste.ini +++ b/templates/rocky/api-paste.ini @@ -6,7 +6,6 @@ use = egg:Paste#urlmap [composite:neutronapi_v2_0] use = call:neutron.auth:pipeline_factory noauth = cors http_proxy_to_wsgi request_id catch_errors extensions neutronapiapp_v2_0 -keystone = cors http_proxy_to_wsgi request_id catch_errors authtoken keystonecontext extensions neutronapiapp_v2_0 keystone = {% for m in extra_middleware %}{{ m.name }} {% endfor %}cors http_proxy_to_wsgi request_id catch_errors authtoken keystonecontext extensions neutronapiapp_v2_0 {% for m in extra_middleware -%} diff --git a/tests/charmhelpers/core/host_factory/ubuntu.py b/tests/charmhelpers/core/host_factory/ubuntu.py index 99451b59..a6d375af 100644 --- a/tests/charmhelpers/core/host_factory/ubuntu.py +++ b/tests/charmhelpers/core/host_factory/ubuntu.py @@ -21,6 +21,7 @@ UBUNTU_RELEASES = ( 'zesty', 'artful', 'bionic', + 'cosmic', ) diff --git a/tox.ini b/tox.ini index c5765bae..b7a79be6 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,7 @@ basepython = python2.7 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = - bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-queens --no-destroy + bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-rocky --no-destroy [testenv:func27-dfs] # Charm Functional Test diff --git a/unit_tests/test_neutron_api_utils.py b/unit_tests/test_neutron_api_utils.py index c0cb7a6f..0263abe0 100644 --- a/unit_tests/test_neutron_api_utils.py +++ b/unit_tests/test_neutron_api_utils.py @@ -37,6 +37,9 @@ TO_PATCH = [ 'apt_install', 'apt_update', 'apt_upgrade', + 'apt_purge', + 'apt_autoremove', + 'filter_missing_packages', 'add_source', 'b64encode', 'config', @@ -312,6 +315,28 @@ class TestNeutronAPIUtils(CharmTestCase): nutils.do_openstack_upgrade(configs) self.assertFalse(stamp_neutron_db.called) + @patch.object(charmhelpers.contrib.openstack.utils, + 'get_os_codename_install_source') + @patch.object(nutils, 'migrate_neutron_database') + @patch.object(nutils, 'stamp_neutron_database') + def test_do_openstack_upgrade_rocky(self, + stamp_neutron_db, + migrate_neutron_db, + gsrc): + self.is_elected_leader.return_value = True + self.os_release.return_value = 'rocky' + self.config.side_effect = self.test_config.get + self.test_config.set('openstack-origin', 'cloud:bionic-rocky') + gsrc.return_value = 'rocky' + self.get_os_codename_install_source.return_value = 'rocky' + self.filter_missing_packages.return_value = ['python-neutron'] + configs = MagicMock() + nutils.do_openstack_upgrade(configs) + self.apt_purge.assert_called_with(['python-neutron'], fatal=True) + self.apt_autoremove.assert_called_with(purge=True, fatal=True) + self.filter_missing_packages.assert_called_with(nutils.PURGE_PACKAGES) + self.assertFalse(stamp_neutron_db.called) + @patch.object(charmhelpers.contrib.openstack.utils, 'get_os_codename_install_source') @patch.object(nutils, 'migrate_neutron_database')