Enable focal and ussuri as part of the gate tests

Add bionic-ussuri and focal-ussuri (with mysql8 support) bundles.

Also, for focal, switch the install and upgrade-charm hooks to install
python3 modules rather than python2 modules.

Change-Id: I75dbf72eb4c2d7d2986730e0ada9a6c359f825d3

charmhelpers/contrib/storage/linux/loopback.py

@@ -32,6 +32,10 @@ def loopback_devices():
 
         /dev/loop0: [0807]:961814 (/tmp/my.img)
 
+    or:
+
+        /dev/loop0: [0807]:961814 (/tmp/my.img (deleted))
+
     :returns: dict: a dict mapping {loopback_dev: backing_file}
     '''
     loopbacks = {}
@@ -39,9 +43,9 @@ def loopback_devices():
     output = check_output(cmd)
     if six.PY3:
         output = output.decode('utf-8')
-    devs = [d.strip().split(' ') for d in output.splitlines() if d != '']
+    devs = [d.strip().split(' ', 2) for d in output.splitlines() if d != '']
     for dev, _, f in devs:
-        loopbacks[dev.replace(':', '')] = re.search(r'\((\S+)\)', f).groups()[0]
+        loopbacks[dev.replace(':', '')] = re.search(r'\((.+)\)', f).groups()[0]
     return loopbacks
 
 

charmhelpers/core/hookenv.py

@@ -1093,7 +1093,7 @@ def status_set(workload_state, message):
 
     Use status-set to set the workload state with a message which is visible
     to the user via juju status. If the status-set command is not found then
-    assume this is juju < 1.23 and juju-log the message unstead.
+    assume this is juju < 1.23 and juju-log the message instead.
 
     workload_state -- valid juju workload state.
     message        -- status update message
@@ -1526,13 +1526,13 @@ def env_proxy_settings(selected_settings=None):
     """Get proxy settings from process environment variables.
 
     Get charm proxy settings from environment variables that correspond to
-    juju-http-proxy, juju-https-proxy and juju-no-proxy (available as of 2.4.2,
-    see lp:1782236) in a format suitable for passing to an application that
-    reacts to proxy settings passed as environment variables. Some applications
-    support lowercase or uppercase notation (e.g. curl), some support only
-    lowercase (e.g. wget), there are also subjectively rare cases of only
-    uppercase notation support. no_proxy CIDR and wildcard support also varies
-    between runtimes and applications as there is no enforced standard.
+    juju-http-proxy, juju-https-proxy juju-no-proxy (available as of 2.4.2, see
+    lp:1782236) and juju-ftp-proxy in a format suitable for passing to an
+    application that reacts to proxy settings passed as environment variables.
+    Some applications support lowercase or uppercase notation (e.g. curl), some
+    support only lowercase (e.g. wget), there are also subjectively rare cases
+    of only uppercase notation support. no_proxy CIDR and wildcard support also
+    varies between runtimes and applications as there is no enforced standard.
 
     Some applications may connect to multiple destinations and expose config
     options that would affect only proxy settings for a specific destination

hooks/horizon_hooks.py

@@ -157,7 +157,7 @@ def install():
         apt_install(packages, fatal=True)
 
 
-@hooks.hook('upgrade-charm')
+@hooks.hook('upgrade-charm.real')
 @restart_on_change(restart_map(), stopstart=True, sleep=3)
 @harden()
 def upgrade_charm():

hooks/horizon_utils.py

@@ -259,7 +259,7 @@ class HorizonOSConfigRenderer(templating.OSConfigRenderer):
         the LOCAL_SETTINGS file until after processing the policyd stuff.
         """
         _hook = hook_name()
-        if _hook not in ('upgrade-charm', 'config-changed'):
+        if _hook not in ('upgrade-charm.real', 'config-changed'):
             return super(HorizonOSConfigRenderer, self).write_all()
         # Otherwise, first do all the other templates
         for k in self.templates.keys():

hooks/install

@@ -1,6 +1,4 @@
 #!/bin/bash -e
-# Wrapper to deal with newer Ubuntu versions that don't have py2 installed
-# by default.
 
 declare -a DEPS=('apt' 'netaddr' 'netifaces' 'pip' 'yaml' 'dnspython')
 
@@ -11,7 +9,7 @@ check_and_install() {
     fi
 }
 
-PYTHON="python"
+PYTHON="python3"
 
 for dep in ${DEPS[@]}; do
     check_and_install ${PYTHON} ${dep}

hooks/upgrade-charm

@@ -1 +0,0 @@
-horizon_hooks.py

hooks/upgrade-charm

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+
+declare -a DEPS=('apt' 'netaddr' 'netifaces' 'pip' 'yaml' 'dnspython')
+
+check_and_install() {
+    pkg="${1}-${2}"
+    if ! dpkg -s ${pkg} 2>&1 > /dev/null; then
+        apt-get -y install ${pkg}
+    fi
+}
+
+PYTHON="python3"
+
+for dep in ${DEPS[@]}; do
+    check_and_install ${PYTHON} ${dep}
+done
+
+exec ./hooks/upgrade-charm.real

hooks/upgrade-charm.real

@@ -0,0 +1 @@
+horizon_hooks.py

metadata.yaml

@@ -13,6 +13,7 @@ series:
   - xenial
   - bionic
   - eoan
+  - focal
   - trusty
 provides:
   nrpe-external-master:

tests/bundles/bionic-ussuri.yaml

@@ -0,0 +1,39 @@
+series: bionic
+
+comment:
+  - 'machines section to decide order of deployment. database sooner = faster'
+  - 'virt-type=kvm is workaround while awaiting new release of python-libjuju'
+
+machines:
+  '0':
+    constraints: virt-type=kvm mem=3072M
+  '1':
+    constraints: virt-type=kvm
+  '2':
+    constraints: virt-type=kvm mem=3072M
+
+relations:
+  - ["keystone:shared-db", "mysql:shared-db"]
+  - ["openstack-dashboard:shared-db", "mysql:shared-db"]
+  - ["openstack-dashboard:identity-service", "keystone:identity-service"]
+
+applications:
+  mysql:
+    charm: cs:~openstack-charmers-next/percona-cluster
+    num_units: 1
+    to:
+      - '0'
+  keystone:
+    charm: cs:~openstack-charmers-next/keystone
+    num_units: 1
+    options:
+      openstack-origin: cloud:bionic-ussuri
+    to:
+      - '1'
+  openstack-dashboard:
+    charm: ../../../openstack-dashboard
+    num_units: 1
+    options:
+      openstack-origin: cloud:bionic-ussuri
+    to:
+      - '2'

tests/bundles/focal-ussuri.yaml

@@ -0,0 +1,69 @@
+variables:
+  openstack-origin: &openstack-origin distro
+
+series: focal
+
+comment:
+  - 'machines section to decide order of deployment. database sooner = faster'
+  - 'virt-type=kvm is workaround while awaiting new release of python-libjuju'
+
+machines:
+  '0':
+    constraints: virt-type=kvm mem=3072M
+  '1':
+    constraints: virt-type=kvm mem=3072M
+  '2':
+    constraints: virt-type=kvm mem=3072M
+  '3':
+    constraints: virt-type=kvm
+  '4':
+    constraints: virt-type=kvm mem=3072M
+
+applications:
+
+  keystone-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  openstack-dashboard-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+
+  mysql-innodb-cluster:
+    charm: cs:~openstack-charmers-next/mysql-innodb-cluster
+    num_units: 3
+    options:
+      source: *openstack-origin
+    to:
+      - '0'
+      - '1'
+      - '2'
+
+  keystone:
+    charm: cs:~openstack-charmers-next/keystone
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '3'
+
+  openstack-dashboard:
+    charm: ../../../openstack-dashboard
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '4'
+
+relations:
+
+  - - 'keystone:shared-db'
+    - 'keystone-mysql-router:shared-db'
+  - - 'keystone-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'openstack-dashboard:shared-db'
+    - 'openstack-dashboard-mysql-router:shared-db'
+  - - 'openstack-dashboard-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+
+  - - 'openstack-dashboard:identity-service'
+    - 'keystone:identity-service'

tests/tests.yaml

@@ -15,6 +15,8 @@ gate_bundles:
   - bionic-rocky
   - bionic-stein
   - bionic-train
+  - bionic-ussuri
+  - focal-ussuri
 dev_bundles:
   - eoan-train
 
@@ -27,3 +29,6 @@ tests:
 tests_options:
   policyd:
     service: openstack-dashboard
+  force_deploy:
+    - focal-ussuri
+

unit_tests/test_horizon_hooks.py

@@ -171,7 +171,7 @@ class TestHorizonHooks(CharmTestCase):
         [side_effects.append('bar') for f in RESTART_MAP.keys()]
         _hash.side_effect = side_effects
         self.filter_installed_packages.return_value = ['foo']
-        self._call_hook('upgrade-charm')
+        self._call_hook('upgrade-charm.real')
         self.apt_install.assert_called_with(['foo'], fatal=True)
         self.assertTrue(self.register_configs().write_all.called)
         ex = [
@@ -205,7 +205,7 @@ class TestHorizonHooks(CharmTestCase):
         [side_effects.append('bar') for f in RESTART_MAP.keys()]
         _hash.side_effect = side_effects
         self.filter_installed_packages.return_value = ['foo']
-        self._call_hook('upgrade-charm')
+        self._call_hook('upgrade-charm.real')
         self.remove_old_packages.assert_called_once_with()
         self.service_restart.assert_called_once_with('apache2')