py3: Switch to Python 3 for OpenStack Rocky

Switch neutron installation to use Python 3 for OpenStack Rocky.

Purge python- packages on upgrade.

Fix duplicate keystone entry in api-paste.ini for Rocky.

Change-Id: I9ead4d0b637f3067e0aa9a20604b2738221860df

hooks/charmhelpers/contrib/openstack/utils.py

@@ -1736,7 +1736,12 @@ def is_unit_upgrading_set():
 
 
 def series_upgrade_prepare(pause_unit_helper=None, configs=None):
-    """ Run common series upgrade prepare tasks."""
+    """ Run common series upgrade prepare tasks.
+
+    :param pause_unit_helper: function: Function to pause unit
+    :param configs: OSConfigRenderer object: Configurations
+    :returns None:
+    """
     set_unit_upgrading()
     if pause_unit_helper and configs:
         if not is_unit_paused_set():
@@ -1744,8 +1749,15 @@ def series_upgrade_prepare(pause_unit_helper=None, configs=None):
 
 
 def series_upgrade_complete(resume_unit_helper=None, configs=None):
-    """ Run common series upgrade complete tasks."""
+    """ Run common series upgrade complete tasks.
+
+    :param resume_unit_helper: function: Function to resume unit
+    :param configs: OSConfigRenderer object: Configurations
+    :returns None:
+    """
     clear_unit_paused()
     clear_unit_upgrading()
-    if resume_unit_helper and configs:
-        resume_unit_helper(configs)
+    if configs:
+        configs.write_all()
+        if resume_unit_helper:
+            resume_unit_helper(configs)

hooks/charmhelpers/core/host_factory/ubuntu.py

@@ -21,6 +21,7 @@ UBUNTU_RELEASES = (
     'zesty',
     'artful',
     'bionic',
+    'cosmic',
 )
 
 

hooks/charmhelpers/fetch/__init__.py

@@ -84,6 +84,7 @@ module = "charmhelpers.fetch.%s" % __platform__
 fetch = importlib.import_module(module)
 
 filter_installed_packages = fetch.filter_installed_packages
+filter_missing_packages = fetch.filter_missing_packages
 install = fetch.apt_install
 upgrade = fetch.apt_upgrade
 update = _fetch_update = fetch.apt_update

hooks/charmhelpers/fetch/bzrurl.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import os
-from subprocess import check_call
+from subprocess import STDOUT, check_output
 from charmhelpers.fetch import (
     BaseFetchHandler,
     UnhandledSource,
@@ -55,7 +55,7 @@ class BzrUrlFetchHandler(BaseFetchHandler):
             cmd = ['bzr', 'branch']
             cmd += cmd_opts
             cmd += [source, dest]
-        check_call(cmd)
+        check_output(cmd, stderr=STDOUT)
 
     def install(self, source, dest=None, revno=None):
         url_parts = self.parse_url(source)

hooks/charmhelpers/fetch/giturl.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import os
-from subprocess import check_call, CalledProcessError
+from subprocess import check_output, CalledProcessError, STDOUT
 from charmhelpers.fetch import (
     BaseFetchHandler,
     UnhandledSource,
@@ -50,7 +50,7 @@ class GitUrlFetchHandler(BaseFetchHandler):
             cmd = ['git', 'clone', source, dest, '--branch', branch]
             if depth:
                 cmd.extend(['--depth', depth])
-        check_call(cmd)
+        check_output(cmd, stderr=STDOUT)
 
     def install(self, source, branch="master", dest=None, depth=None):
         url_parts = self.parse_url(source)

hooks/charmhelpers/fetch/ubuntu.py

@@ -189,6 +189,18 @@ def filter_installed_packages(packages):
     return _pkgs
 
 
+def filter_missing_packages(packages):
+    """Return a list of packages that are installed.
+
+    :param packages: list of packages to evaluate.
+    :returns list: Packages that are installed.
+    """
+    return list(
+        set(packages) -
+        set(filter_installed_packages(packages))
+    )
+
+
 def apt_cache(in_memory=True, progress=None):
     """Build and return an apt cache."""
     from apt import apt_pkg

hooks/neutron_api_utils.py

@@ -59,7 +59,10 @@ from charmhelpers.fetch import (
     apt_update,
     apt_install,
     apt_upgrade,
-    add_source
+    add_source,
+    filter_missing_packages,
+    apt_purge,
+    apt_autoremove,
 )
 
 from charmhelpers.core.host import (
@@ -99,23 +102,31 @@ KILO_PACKAGES = [
     'python-neutron-vpnaas',
 ]
 
-VERSION_PACKAGE = 'neutron-common'
-
-BASE_GIT_PACKAGES = [
-    'libffi-dev',
-    'libmysqlclient-dev',
-    'libssl-dev',
-    'libxml2-dev',
-    'libxslt1-dev',
-    'libyaml-dev',
-    'openstack-pkg-tools',
-    'python-dev',
-    'python-neutronclient',  # required for get_neutron_client() import
-    'python-pip',
-    'python-setuptools',
-    'zlib1g-dev',
+PY3_PACKAGES = [
+    'python3-neutron',
+    'python3-neutron-lbaas',
+    'python3-neutron-fwaas',
+    'python3-neutron-dynamic-routing',
+    'python3-networking-hyperv',
+    'python3-memcache',
 ]
 
+PURGE_PACKAGES = [
+    'python-neutron',
+    'python-neutron-lbaas',
+    'python-neutron-fwaas',
+    'python-neutron-vpnaas',
+    'python-neutron-dynamic-routing',
+    'python-networking-hyperv',
+    'python-memcache',
+    'python-keystoneclient',
+    'python-mysqldb',
+    'python-psycopg2',
+    'python-six',
+]
+
+VERSION_PACKAGE = 'neutron-common'
+
 BASE_SERVICES = [
     'neutron-server'
 ]
@@ -372,7 +383,11 @@ def manage_plugin():
 
 def determine_packages(source=None):
     # currently all packages match service names
-    packages = [] + BASE_PACKAGES
+    release = get_os_codename_install_source(source)
+    cmp_release = CompareOpenStackReleases(release)
+    packages = deepcopy(BASE_PACKAGES)
+    if cmp_release >= 'rocky':
+        packages.extend(PY3_PACKAGES)
 
     for v in resource_map().values():
         packages.extend(v['services'])
@@ -382,26 +397,37 @@ def determine_packages(source=None):
                                             'neutron')
             packages.extend(pkgs)
 
-    release = get_os_codename_install_source(source)
+    packages.extend(token_cache_pkgs(release=release))
 
-    if CompareOpenStackReleases(release) >= 'kilo':
-        packages.extend(KILO_PACKAGES)
-    if CompareOpenStackReleases(release) >= 'ocata':
-        packages.append('python-neutron-dynamic-routing')
-    if CompareOpenStackReleases(release) >= 'pike':
-        packages.remove('python-neutron-vpnaas')
+    if cmp_release < 'rocky':
+        if cmp_release >= 'kilo':
+            packages.extend(KILO_PACKAGES)
+        if cmp_release >= 'ocata':
+            packages.append('python-neutron-dynamic-routing')
+        if cmp_release >= 'pike':
+            packages.remove('python-neutron-vpnaas')
 
-    if release == 'kilo' or CompareOpenStackReleases(release) >= 'mitaka':
-        packages.append('python-networking-hyperv')
+        if release == 'kilo' or cmp_release >= 'mitaka':
+            packages.append('python-networking-hyperv')
 
     if config('neutron-plugin') == 'vsp':
         nuage_pkgs = config('nuage-packages').split()
-        packages += nuage_pkgs
+        packages.extend(nuage_pkgs)
+
+    if cmp_release >= 'rocky':
+        packages = [p for p in packages if not p.startswith('python-')]
 
-    packages.extend(token_cache_pkgs(release=release))
     return list(set(packages))
 
 
+def determine_purge_packages():
+    '''Return a list of packages to purge for the current OS release'''
+    cmp_os_source = CompareOpenStackReleases(os_release('neutron-common'))
+    if cmp_os_source >= 'rocky':
+        return PURGE_PACKAGES
+    return []
+
+
 def determine_ports():
     '''Assemble a list of API ports for services we are managing'''
     ports = []
@@ -524,6 +550,11 @@ def do_openstack_upgrade(configs):
                 options=dpkg_opts,
                 fatal=True)
 
+    installed_packages = filter_missing_packages(determine_purge_packages())
+    if installed_packages:
+        apt_purge(installed_packages, fatal=True)
+        apt_autoremove(purge=True, fatal=True)
+
     # set CONFIGS to load templates from new release
     configs.set_release(openstack_release=new_os_rel)
     # Before kilo it's nova-cloud-controllers job

templates/rocky/api-paste.ini

@@ -6,7 +6,6 @@ use = egg:Paste#urlmap
 [composite:neutronapi_v2_0]
 use = call:neutron.auth:pipeline_factory
 noauth = cors http_proxy_to_wsgi request_id catch_errors extensions neutronapiapp_v2_0
-keystone = cors http_proxy_to_wsgi request_id catch_errors authtoken keystonecontext extensions neutronapiapp_v2_0
 keystone = {% for m in extra_middleware %}{{ m.name }} {% endfor %}cors http_proxy_to_wsgi request_id catch_errors authtoken keystonecontext extensions neutronapiapp_v2_0
 
 {% for m in extra_middleware -%}

tests/charmhelpers/core/host_factory/ubuntu.py

@@ -21,6 +21,7 @@ UBUNTU_RELEASES = (
     'zesty',
     'artful',
     'bionic',
+    'cosmic',
 )
 
 

tox.ini

@@ -66,7 +66,7 @@ basepython = python2.7
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 commands =
-    bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-queens --no-destroy
+    bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-rocky --no-destroy
 
 [testenv:func27-dfs]
 # Charm Functional Test

unit_tests/test_neutron_api_utils.py

@@ -37,6 +37,9 @@ TO_PATCH = [
     'apt_install',
     'apt_update',
     'apt_upgrade',
+    'apt_purge',
+    'apt_autoremove',
+    'filter_missing_packages',
     'add_source',
     'b64encode',
     'config',
@@ -312,6 +315,28 @@ class TestNeutronAPIUtils(CharmTestCase):
         nutils.do_openstack_upgrade(configs)
         self.assertFalse(stamp_neutron_db.called)
 
+    @patch.object(charmhelpers.contrib.openstack.utils,
+                  'get_os_codename_install_source')
+    @patch.object(nutils, 'migrate_neutron_database')
+    @patch.object(nutils, 'stamp_neutron_database')
+    def test_do_openstack_upgrade_rocky(self,
+                                        stamp_neutron_db,
+                                        migrate_neutron_db,
+                                        gsrc):
+        self.is_elected_leader.return_value = True
+        self.os_release.return_value = 'rocky'
+        self.config.side_effect = self.test_config.get
+        self.test_config.set('openstack-origin', 'cloud:bionic-rocky')
+        gsrc.return_value = 'rocky'
+        self.get_os_codename_install_source.return_value = 'rocky'
+        self.filter_missing_packages.return_value = ['python-neutron']
+        configs = MagicMock()
+        nutils.do_openstack_upgrade(configs)
+        self.apt_purge.assert_called_with(['python-neutron'], fatal=True)
+        self.apt_autoremove.assert_called_with(purge=True, fatal=True)
+        self.filter_missing_packages.assert_called_with(nutils.PURGE_PACKAGES)
+        self.assertFalse(stamp_neutron_db.called)
+
     @patch.object(charmhelpers.contrib.openstack.utils,
                   'get_os_codename_install_source')
     @patch.object(nutils, 'migrate_neutron_database')