py3: Switch to using Python 3 for rocky or later

Switch package install to Python 3 for OpenStack Rocky or later.

When upgrading, remove any python-* packages that where explicitly
installated and then autoremove --purge any dependencies that are
no longer required.

Change-Id: I120ed44988f9e86a27f464dc879acdc1b8926dd8

charmhelpers/contrib/openstack/amulet/utils.py

@@ -1013,6 +1013,9 @@ class OpenStackAmuletUtils(AmuletUtils):
                                cmd, code, output))
             amulet.raise_status(amulet.FAIL, msg=msg)
 
+        # For mimic ceph osd lspools output
+        output = output.replace("\n", ",")
+
         # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance,
         for pool in str(output).split(','):
             pool_id_name = pool.split(' ')

charmhelpers/contrib/openstack/context.py

@@ -1519,6 +1519,10 @@ class NeutronAPIContext(OSContextGenerator):
                 'rel_key': 'enable-qos',
                 'default': False,
             },
+            'enable_nsg_logging': {
+                'rel_key': 'enable-nsg-logging',
+                'default': False,
+            },
         }
         ctxt = self.get_neutron_options({})
         for rid in relation_ids('neutron-plugin-api'):

charmhelpers/contrib/openstack/utils.py

@@ -186,7 +186,7 @@ SWIFT_CODENAMES = OrderedDict([
     ('queens',
         ['2.16.0', '2.17.0']),
     ('rocky',
-        ['2.18.0']),
+        ['2.18.0', '2.19.0']),
 ])
 
 # >= Liberty version->codename mapping
@@ -1733,3 +1733,31 @@ def is_unit_upgrading_set():
             return not(not(kv.get('unit-upgrading')))
     except Exception:
         return False
+
+
+def series_upgrade_prepare(pause_unit_helper=None, configs=None):
+    """ Run common series upgrade prepare tasks.
+
+    :param pause_unit_helper: function: Function to pause unit
+    :param configs: OSConfigRenderer object: Configurations
+    :returns None:
+    """
+    set_unit_upgrading()
+    if pause_unit_helper and configs:
+        if not is_unit_paused_set():
+            pause_unit_helper(configs)
+
+
+def series_upgrade_complete(resume_unit_helper=None, configs=None):
+    """ Run common series upgrade complete tasks.
+
+    :param resume_unit_helper: function: Function to resume unit
+    :param configs: OSConfigRenderer object: Configurations
+    :returns None:
+    """
+    clear_unit_paused()
+    clear_unit_upgrading()
+    if configs:
+        configs.write_all()
+        if resume_unit_helper:
+            resume_unit_helper(configs)

charmhelpers/core/hookenv.py

@@ -48,6 +48,7 @@ INFO = "INFO"
 DEBUG = "DEBUG"
 TRACE = "TRACE"
 MARKER = object()
+SH_MAX_ARG = 131071
 
 cache = {}
 
@@ -98,7 +99,7 @@ def log(message, level=None):
         command += ['-l', level]
     if not isinstance(message, six.string_types):
         message = repr(message)
-    command += [message]
+    command += [message[:SH_MAX_ARG]]
     # Missing juju-log should not cause failures in unit tests
     # Send log output to stderr
     try:

charmhelpers/core/host_factory/ubuntu.py

@@ -21,6 +21,7 @@ UBUNTU_RELEASES = (
     'zesty',
     'artful',
     'bionic',
+    'cosmic',
 )
 
 

charmhelpers/fetch/__init__.py

@@ -84,6 +84,7 @@ module = "charmhelpers.fetch.%s" % __platform__
 fetch = importlib.import_module(module)
 
 filter_installed_packages = fetch.filter_installed_packages
+filter_missing_packages = fetch.filter_missing_packages
 install = fetch.apt_install
 upgrade = fetch.apt_upgrade
 update = _fetch_update = fetch.apt_update
@@ -96,6 +97,7 @@ if __platform__ == "ubuntu":
     apt_update = fetch.apt_update
     apt_upgrade = fetch.apt_upgrade
     apt_purge = fetch.apt_purge
+    apt_autoremove = fetch.apt_autoremove
     apt_mark = fetch.apt_mark
     apt_hold = fetch.apt_hold
     apt_unhold = fetch.apt_unhold

charmhelpers/fetch/bzrurl.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import os
-from subprocess import check_call
+from subprocess import STDOUT, check_output
 from charmhelpers.fetch import (
     BaseFetchHandler,
     UnhandledSource,
@@ -55,7 +55,7 @@ class BzrUrlFetchHandler(BaseFetchHandler):
             cmd = ['bzr', 'branch']
             cmd += cmd_opts
             cmd += [source, dest]
-        check_call(cmd)
+        check_output(cmd, stderr=STDOUT)
 
     def install(self, source, dest=None, revno=None):
         url_parts = self.parse_url(source)

charmhelpers/fetch/giturl.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import os
-from subprocess import check_call, CalledProcessError
+from subprocess import check_output, CalledProcessError, STDOUT
 from charmhelpers.fetch import (
     BaseFetchHandler,
     UnhandledSource,
@@ -50,7 +50,7 @@ class GitUrlFetchHandler(BaseFetchHandler):
             cmd = ['git', 'clone', source, dest, '--branch', branch]
             if depth:
                 cmd.extend(['--depth', depth])
-        check_call(cmd)
+        check_output(cmd, stderr=STDOUT)
 
     def install(self, source, branch="master", dest=None, depth=None):
         url_parts = self.parse_url(source)

charmhelpers/fetch/ubuntu.py

@@ -189,6 +189,18 @@ def filter_installed_packages(packages):
     return _pkgs
 
 
+def filter_missing_packages(packages):
+    """Return a list of packages that are installed.
+
+    :param packages: list of packages to evaluate.
+    :returns list: Packages that are installed.
+    """
+    return list(
+        set(packages) -
+        set(filter_installed_packages(packages))
+    )
+
+
 def apt_cache(in_memory=True, progress=None):
     """Build and return an apt cache."""
     from apt import apt_pkg
@@ -248,6 +260,14 @@ def apt_purge(packages, fatal=False):
     _run_apt_command(cmd, fatal)
 
 
+def apt_autoremove(purge=True, fatal=False):
+    """Purge one or more packages."""
+    cmd = ['apt-get', '--assume-yes', 'autoremove']
+    if purge:
+        cmd.append('--purge')
+    _run_apt_command(cmd, fatal)
+
+
 def apt_mark(packages, mark, fatal=False):
     """Flag one or more packages using apt-mark."""
     log("Marking {} as {}".format(packages, mark))

hooks/ceilometer_hooks.py

@@ -21,8 +21,9 @@ import sys
 import os
 
 from charmhelpers.fetch import (
-    apt_install, filter_installed_packages,
+    apt_install,
-    apt_update
+    apt_update,
+    filter_installed_packages,
 )
 from charmhelpers.core.hookenv import (
     open_port,
@@ -140,8 +141,12 @@ def db_joined():
 def metric_service_joined():
     # NOTE(jamespage): gnocchiclient is required to support
     #                  the gnocchi event dispatcher
-    apt_install(filter_installed_packages(['python-gnocchiclient']),
+    release = CompareOpenStackReleases(
-                fatal=True)
+        get_os_codename_install_source(config('openstack-origin')))
+    pkgs = ['python-gnocchiclient']
+    if release >= 'rocky':
+        pkgs = ['python3-gnocchiclient']
+    apt_install(filter_installed_packages(pkgs), fatal=True)
 
 
 @hooks.hook("amqp-relation-changed",

lib/ceilometer_utils.py

@@ -54,7 +54,14 @@ from charmhelpers.core.hookenv import (
     DEBUG,
     relation_ids,
 )
-from charmhelpers.fetch import apt_update, apt_install, apt_upgrade
+from charmhelpers.fetch import (
+    apt_update,
+    apt_install,
+    apt_upgrade,
+    apt_purge,
+    apt_autoremove,
+    filter_missing_packages,
+)
 from charmhelpers.core.host import init_is_systemd
 from copy import deepcopy
 
@@ -104,6 +111,10 @@ CEILOMETER_BASE_PACKAGES = [
     'python-pymongo',
 ]
 
+PY3_PACKAGES = [
+    'python3-ceilometer',
+]
+
 ICEHOUSE_PACKAGES = [
     'ceilometer-alarm-notifier',
     'ceilometer-alarm-evaluator',
@@ -361,6 +372,11 @@ def do_openstack_upgrade(configs):
                 options=dpkg_opts,
                 fatal=True)
 
+    installed_packages = filter_missing_packages(determine_purge_packages())
+    if installed_packages:
+        apt_purge(installed_packages, fatal=True)
+        apt_autoremove(purge=True, fatal=True)
+
     # set CONFIGS to load templates from new release
     configs.set_release(openstack_release=new_os_rel)
 
@@ -397,14 +413,33 @@ def get_packages():
     # NOTE(jamespage): @queens ceilometer has no API service, so
     #                  no requirement for token caching.
     if cmp_codename >= 'queens':
-        return deepcopy(QUEENS_PACKAGES)
+        packages = deepcopy(QUEENS_PACKAGES)
-
+        if cmp_codename >= 'rocky':
-    packages = (deepcopy(CEILOMETER_BASE_PACKAGES) +
+            packages.extend(PY3_PACKAGES)
-                ceilometer_release_packages())
+    else:
-    packages.extend(token_cache_pkgs(source=config('openstack-origin')))
+        packages = (deepcopy(CEILOMETER_BASE_PACKAGES) +
+                    ceilometer_release_packages())
+        packages.extend(token_cache_pkgs(source=config('openstack-origin')))
     return packages
 
 
+def determine_purge_packages():
+    '''
+    Determine list of packages that where previously installed which are no
+    longer needed.
+
+    :returns: list of package names
+    '''
+    cmp_codename = CompareOpenStackReleases(
+        get_os_codename_install_source(config('openstack-origin')))
+    if cmp_codename >= 'rocky':
+        pkgs = [p for p in CEILOMETER_BASE_PACKAGES if p.startswith('python-')]
+        pkgs.append('python-ceilometer')
+        pkgs.append('python-memcache')
+        return pkgs
+    return []
+
+
 def get_shared_secret():
     """
     Returns the current shared secret for the ceilometer node. If the shared

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -1013,6 +1013,9 @@ class OpenStackAmuletUtils(AmuletUtils):
                                cmd, code, output))
             amulet.raise_status(amulet.FAIL, msg=msg)
 
+        # For mimic ceph osd lspools output
+        output = output.replace("\n", ",")
+
         # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance,
         for pool in str(output).split(','):
             pool_id_name = pool.split(' ')

tests/charmhelpers/core/hookenv.py

@@ -48,6 +48,7 @@ INFO = "INFO"
 DEBUG = "DEBUG"
 TRACE = "TRACE"
 MARKER = object()
+SH_MAX_ARG = 131071
 
 cache = {}
 
@@ -98,7 +99,7 @@ def log(message, level=None):
         command += ['-l', level]
     if not isinstance(message, six.string_types):
         message = repr(message)
-    command += [message]
+    command += [message[:SH_MAX_ARG]]
     # Missing juju-log should not cause failures in unit tests
     # Send log output to stderr
     try:

tests/charmhelpers/core/host_factory/ubuntu.py

@@ -21,6 +21,7 @@ UBUNTU_RELEASES = (
     'zesty',
     'artful',
     'bionic',
+    'cosmic',
 )
 
 

tox.ini

@@ -65,7 +65,7 @@ basepython = python2.7
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 commands =
-    bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-queens --no-destroy
+    bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-rocky --no-destroy
 
 [testenv:func27-dfs]
 # Charm Functional Test

unit_tests/test_ceilometer_hooks.py

@@ -462,11 +462,22 @@ class CeilometerHooksTest(CharmTestCase):
         hooks.hooks.execute(['hooks/ha-relation-changed'])
         self.assertEqual(mock_keystone_joined.call_count, 1)
 
-    def test_metric_service_joined(self):
+    def test_metric_service_joined_queens(self):
         self.filter_installed_packages.return_value = ['python-gnocchiclient']
+        self.get_os_codename_install_source.return_value = 'queens'
         hooks.hooks.execute(['hooks/metric-service-relation-joined'])
         self.filter_installed_packages.assert_called_with(
             ['python-gnocchiclient']
         )
         self.apt_install.assert_called_with(['python-gnocchiclient'],
                                             fatal=True)
+
+    def test_metric_service_joined_rocky(self):
+        self.filter_installed_packages.return_value = ['python3-gnocchiclient']
+        self.get_os_codename_install_source.return_value = 'rocky'
+        hooks.hooks.execute(['hooks/metric-service-relation-joined'])
+        self.filter_installed_packages.assert_called_with(
+            ['python3-gnocchiclient']
+        )
+        self.apt_install.assert_called_with(['python3-gnocchiclient'],
+                                            fatal=True)

unit_tests/test_ceilometer_utils.py

@@ -211,6 +211,19 @@ class CeilometerUtilsTest(CharmTestCase):
         )
         self.reset_os_release.assert_called()
 
+    def test_determine_purge_packages(self):
+        'Ensure no packages are identified for purge prior to rocky'
+        self.get_os_codename_install_source.return_value = 'queens'
+        self.assertEqual(utils.determine_purge_packages(), [])
+
+    def test_determine_purge_packages_rocky(self):
+        'Ensure python packages are identified for purge at rocky'
+        self.get_os_codename_install_source.return_value = 'rocky'
+        self.assertEqual(utils.determine_purge_packages(),
+                         [p for p in utils.CEILOMETER_BASE_PACKAGES
+                          if p.startswith('python-')] +
+                         ['python-ceilometer', 'python-memcache'])
+
     def test_get_packages_icehouse(self):
         self.get_os_codename_install_source.return_value = 'icehouse'
         self.token_cache_pkgs.return_value = []
@@ -226,6 +239,19 @@ class CeilometerUtilsTest(CharmTestCase):
                          utils.MITAKA_PACKAGES +
                          ['memcached'])
 
+    def test_get_packages_queens(self):
+        self.get_os_codename_install_source.return_value = 'queens'
+        self.token_cache_pkgs.return_value = []
+        self.assertEqual(utils.get_packages(),
+                         utils.QUEENS_PACKAGES)
+
+    def test_get_packages_rocky(self):
+        self.get_os_codename_install_source.return_value = 'rocky'
+        self.token_cache_pkgs.return_value = []
+        self.assertEqual(utils.get_packages(),
+                         utils.QUEENS_PACKAGES +
+                         ['python3-ceilometer'])
+
     def test_assess_status(self):
         with patch.object(utils, 'assess_status_func') as asf:
             callee = MagicMock()