py3: Switch packages to Python 3 for train and later

Switch package install to Python 3 for OpenStack Train and later.

When upgrading, remove any python-* packages that were explicitly
installated and then autoremove --purge any dependencies that are
no longer required.

This patch also includes the following related changes:
  * Use the common files package, swift, rather than python-swift
    as the package name when determining releases.
  * Drop the python2 shebang from manager.py in favor of specifying
    the interpreter on the subprocess call. The python interpreter
    version must match the python version of the OpenStack payload
    due to the swift library imports.
  * Enable the cPickle import in manager.py for Python 3
    (C-optimized module name is _pickle), and fix 'result'
    variable that is a set type but should be a dict type.

Change-Id: Ia3fdbf2020137bcf39039478ee3606717d3d6e20
Closes-Bug: #1841184

actions/openstack_upgrade.py

@@ -50,7 +50,7 @@ def openstack_upgrade():
     code to run, otherwise a full service level upgrade will fire
     on config-changed."""
 
-    if (do_action_openstack_upgrade('python-swift',
+    if (do_action_openstack_upgrade('swift',
                                     do_openstack_upgrade,
                                     CONFIGS)):
         config_changed()

hooks/swift_hooks.py

@@ -179,7 +179,7 @@ def config_changed():
 
     # Determine whether or not we should do an upgrade.
     if not config('action-managed-upgrade') and \
-            openstack.openstack_upgrade_available('python-swift'):
+            openstack.openstack_upgrade_available('swift'):
         do_openstack_upgrade(CONFIGS)
         status_set('maintenance', 'Running openstack upgrade')
 

lib/swift_utils.py

@@ -35,6 +35,7 @@ from charmhelpers.contrib.openstack.utils import (
     make_assess_status_func,
     os_application_version_set,
     CompareOpenStackReleases,
+    reset_os_release,
 )
 from charmhelpers.contrib.hahelpers.cluster import (
     is_elected_leader,
@@ -57,12 +58,16 @@ from charmhelpers.core.hookenv import (
     is_leader,
     leader_set,
     leader_get,
+    status_set,
 )
 from charmhelpers.fetch import (
     apt_update,
     apt_upgrade,
     apt_install,
-    add_source
+    apt_purge,
+    apt_autoremove,
+    add_source,
+    filter_missing_packages,
 )
 from charmhelpers.core.host import (
     lsb_release,
@@ -134,6 +139,19 @@ MITAKA_PACKAGES = [
     'python-ceilometermiddleware',
     'python-keystonemiddleware',
 ]
+PY3_PACKAGES = [
+    'python3-ceilometermiddleware',
+    'python3-keystonemiddleware',
+    'python3-six',
+    'python3-swift',
+    'python3-swiftclient',
+]
+PURGE_PACKAGES = [
+    'python-ceilometermiddleware',
+    'python-keystonemiddleware',
+    'python-swift',
+    'python-swiftclient',
+]
 
 SWIFT_HA_RES = 'grp_swift_vips'
 TEMPLATES = 'templates/'
@@ -474,9 +492,39 @@ def determine_packages(release):
         pkgs.remove('python-keystone')
     if cmp_openstack >= 'rocky':
         pkgs.remove('swift-plugin-s3')
+    if cmp_openstack >= 'train':
+        pkgs = [p for p in pkgs if not p.startswith('python-')]
+        pkgs.extend(PY3_PACKAGES)
     return pkgs
 
 
+def determine_purge_packages():
+    '''
+    Determine list of packages that where previously installed which are no
+    longer needed.
+
+    :returns: list of package names
+    '''
+    cmp_openstack = CompareOpenStackReleases(os_release('swift'))
+    if cmp_openstack >= 'train':
+        return PURGE_PACKAGES
+    return []
+
+
+def remove_old_packages():
+    '''Purge any packages that need to be removed.
+
+    :returns: bool Whether packages were removed.
+    '''
+    installed_packages = filter_missing_packages(determine_purge_packages())
+    if installed_packages:
+        log('Removing apt packages')
+        status_set('maintenance', 'Removing apt packages')
+        apt_purge(installed_packages, fatal=True)
+        apt_autoremove(purge=True, fatal=True)
+    return bool(installed_packages)
+
+
 def initialize_ring(path, part_power, replicas, min_hours):
     get_manager().initialize_ring(path, part_power, replicas, min_hours)
 
@@ -645,6 +693,10 @@ def do_openstack_upgrade(configs):
     ]
     apt_update()
     apt_upgrade(options=dpkg_opts, fatal=True, dist=True)
+    reset_os_release()
+    apt_install(packages=determine_packages(new_os_rel),
+                options=dpkg_opts, fatal=True)
+    remove_old_packages()
     configs.set_release(openstack_release=new_os_rel)
     configs.write_all()
 
@@ -1189,6 +1241,11 @@ def _proxy_manager_call(path, args, kwargs):
                    args=args,
                    kwargs=kwargs)
     serialized = json.dumps(package, **JSON_ENCODE_OPTIONS)
+    cmp_openstack = CompareOpenStackReleases(os_release('swift'))
+    if cmp_openstack >= 'train':
+        python = 'python3'
+    else:
+        python = 'python2'
     script = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                           '..',
                                           'swift_manager',
@@ -1198,14 +1255,14 @@ def _proxy_manager_call(path, args, kwargs):
         if sys.version_info < (3, 5):
             # remove this after trusty support is removed.  No subprocess.run
             # in Python 3.4
-            process = subprocess.Popen([script, serialized],
+            process = subprocess.Popen([python, script, serialized],
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE,
                                        env=env)
             out, err = process.communicate()
             result = json.loads(out.decode('UTF-8'))
         else:
-            completed = subprocess.run([script, serialized],
+            completed = subprocess.run([python, script, serialized],
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE,
                                        env=env)

swift_manager/manager.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python2
-#
 # Copyright 2016 Canonical Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,12 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# NOTE(tinwood): This file needs to remain Python2 as it uses keystoneclient
-# from the payload software to do it's work.
-
 from __future__ import print_function
 
-import cPickle as pickle
+try:
+    import cPickle as pickle
+except ModuleNotFoundError:
+    import _pickle as pickle
 import json
 import os
 import sys
@@ -259,7 +257,7 @@ if __name__ == '__main__':
         print(str(e), file=sys.stderr)
         import traceback
         print(traceback.format_exc(), file=sys.stderr)
-        result = {'error', str(e)}
+        result = {'error': str(e)}
     except Exception as e:
         print("{}: something went wrong: {}".format(__file__, str(e)),
               file=sys.stderr)

unit_tests/test_swift_utils.py

@@ -618,6 +618,32 @@ class SwiftUtilsTestCase(unittest.TestCase):
             swift_utils.determine_packages('rocky')
         )
 
+        self.assertEqual(
+            ['swift',
+             'swift-proxy',
+             'memcached',
+             'apache2',
+             'python-swiftclient',
+             'swauth',
+             'python-ceilometermiddleware',
+             'python-keystonemiddleware'],
+            swift_utils.determine_packages('stein')
+        )
+
+        self.assertEqual(
+            ['swift',
+             'swift-proxy',
+             'memcached',
+             'apache2',
+             'swauth',
+             'python3-ceilometermiddleware',
+             'python3-keystonemiddleware',
+             'python3-six',
+             'python3-swift',
+             'python3-swiftclient'],
+            swift_utils.determine_packages('train')
+        )
+
     @mock.patch('lib.swift_utils.config')
     def test_determine_replicas_account_set(self, config):
         config.side_effect = lambda key: {