Fix haproxy not running and race in zone creation

This change cherry-picks 3 bugfixes at the same time as they are all needed in order to get the gate green: - Fix haproxy not running on fresh deployments - Sync release-tools - Fix race condition in default zone creation Sync release-tools boilerplate containing these changes in order to pin pip < 20.3: https://github.com/openstack-charmers/release-tools/pull/125 https://github.com/openstack-charmers/release-tools/pull/126 https://github.com/openstack-charmers/release-tools/pull/127 https://github.com/openstack-charmers/release-tools/pull/129 Also add Groovy to the test gate. Change-Id: I04ca79487085d9088811bc0ffd5f0981db50c42e Closes-Bug: #1904411 Closes-Bug: #1905985 (cherry picked from commit a43af5c7e7) (cherry picked from commit 330539066a) (cherry picked from commit f35f3e0392)
2020-11-05 12:44:31 +01:00 · 2020-11-05 12:44:31 +01:00 · 975024ea53
parent 5944d47c31
commit 975024ea53
9 changed files with 117 additions and 23 deletions
--- a/hooks/hooks.py
+++ b/hooks/hooks.py
@ -156,14 +156,21 @@ def install_packages():
    )
    if pkgs:
        status_set('maintenance', 'Installing radosgw packages')
+        if ('apache2' in pkgs):
+            # NOTE(lourot): Apache's default config makes it listen on port 80,
+            # which will prevent HAProxy from listening on that same port. We
+            # use Apache in this setup however for SSL (different port). We
+            # need to let Apache free port 80 before we can install HAProxy
+            # otherwise HAProxy will crash. See lp:1904411
+            log('Installing Apache')
+            apt_install(['apache2'], fatal=True)
+            disable_unused_apache_sites()
        apt_install(pkgs, fatal=True)

    pkgs = filter_missing_packages(APACHE_PACKAGES)
    if pkgs:
        apt_purge(pkgs)

-    disable_unused_apache_sites()
-

@hooks.hook('install.real')
@harden()
@ -315,7 +322,7 @@ def mon_relation(rid=None, unit=None):

            if multisite_deployment():
                process_multisite_relations()
-            elif is_leader():
+            elif is_leader() and 'mon' in CONFIGS.complete_contexts():
                # In a non multi-site deployment create the
                # zone using the default zonegroup and restart the service
                internal_url = '{}:{}'.format(
@ -325,11 +332,32 @@ def mon_relation(rid=None, unit=None):
                endpoints = [internal_url]
                zonegroup = 'default'
                zone = config('zone')
-                if zone not in multisite.list_zones():
-                    multisite.create_zone(zone,
-                                          endpoints=endpoints,
-                                          default=True, master=True,
-                                          zonegroup=zonegroup)
+                existing_zones = multisite.list_zones()
+                log('Existing zones {}'.format(existing_zones), level=DEBUG)
+                if zone not in existing_zones:
+                    log("Zone '{}' doesn't exist, creating".format(zone))
+                    try:
+                        multisite.create_zone(zone,
+                                              endpoints=endpoints,
+                                              default=True, master=True,
+                                              zonegroup=zonegroup)
+                    except subprocess.CalledProcessError as e:
+                        if 'File exists' in e.stderr.decode('UTF-8'):
+                            # NOTE(lourot): may have been created in the
+                            # background by the Rados Gateway daemon, see
+                            # lp:1856106
+                            log("Zone '{}' existed already after all".format(
+                                zone))
+                        else:
+                            raise
+
+                    existing_zones = multisite.list_zones()
+                    log('Existing zones {}'.format(existing_zones),
+                        level=DEBUG)
+                    if zone not in existing_zones:
+                        raise RuntimeError("Could not create zone '{}'".format(
+                            zone))
+
                    service_restart(service_name())
        else:
            send_request_if_needed(rq, relation='mon')
--- a/hooks/multisite.py
+++ b/hooks/multisite.py
@ -29,7 +29,7 @@ RGW_ADMIN = 'radosgw-admin'
 def _check_output(cmd):
    """Logging wrapper for subprocess.check_ouput"""
    hookenv.log("Executing: {}".format(' '.join(cmd)), level=hookenv.DEBUG)
-    return subprocess.check_output(cmd).decode('UTF-8')
+    return subprocess.check_output(cmd, stderr=subprocess.PIPE).decode('UTF-8')


@decorators.retry_on_exception(num_retries=5, base_delay=3,
--- a/hooks/utils.py
+++ b/hooks/utils.py
@ -28,6 +28,7 @@ from charmhelpers.core.hookenv import (
    application_version_set,
    config,
    leader_get,
+    log,
 )
 from charmhelpers.contrib.openstack import (
    context,
@ -47,6 +48,8 @@ from charmhelpers.core.host import (
    lsb_release,
    CompareHostReleases,
    init_is_systemd,
+    service,
+    service_running,
 )
 from charmhelpers.fetch import (
    apt_cache,
@ -349,6 +352,7 @@ def disable_unused_apache_sites():
    """Ensure that unused apache configurations are disabled to prevent them
    from conflicting with the charm-provided version.
    """
+    log('Disabling unused Apache sites')
    for apache_site_file in UNUSED_APACHE_SITE_FILES:
        apache_site = apache_site_file.split('/')[-1].split('.')[0]
        if os.path.exists(apache_site_file):
@ -362,6 +366,10 @@ def disable_unused_apache_sites():
    with open(APACHE_PORTS_FILE, 'w') as ports:
        ports.write("")

+    if service_running('apache2'):
+        log('Restarting Apache')
+        service('restart', 'apache2')
+

 def systemd_based_radosgw():
    """Determine if install should use systemd based radosgw instances"""
--- a/requirements.txt
+++ b/requirements.txt
@ -7,11 +7,13 @@
 #       requirements.  They are intertwined.  Also, Zaza itself should specify
 #       all of its own requirements and if it doesn't, fix it there.
 #
-setuptools<50.0.0  # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85
 pbr>=1.8.0,<1.9.0
 simplejson>=2.2.0
 netifaces>=0.10.4
-netaddr>=0.7.12,!=0.7.16
+
+# Strange import error with newer netaddr:
+netaddr>0.7.16,<0.8.0
+
 Jinja2>=2.6  # BSD License (3 clause)
 six>=1.9.0

--- a/test-requirements.txt
+++ b/test-requirements.txt
@ -9,11 +9,44 @@
 #
 setuptools<50.0.0  # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85
 charm-tools>=2.4.4
+
+# Workaround until https://github.com/juju/charm-tools/pull/589 gets
+# published
+keyring<21
+
 requests>=2.18.4
-mock>=1.2
+
+# Newer mock seems to have some syntax which is newer than python3.5 (e.g.
+# f'{something}'
+mock>=1.2,<4.0.0; python_version < '3.6'
+mock>=1.2; python_version >= '3.6'
+
 flake8>=2.2.4
 stestr>=2.2.0
+
+# Dependency of stestr. Workaround for
+# https://github.com/mtreinish/stestr/issues/145
+cliff<3.0.0
+
+# Dependencies of stestr. Newer versions use keywords that didn't exist in
+# python 3.5 yet (e.g. "ModuleNotFoundError")
+importlib-metadata<3.0.0; python_version < '3.6'
+importlib-resources<3.0.0; python_version < '3.6'
+
+# Some Zuul nodes sometimes pull newer versions of these dependencies which
+# dropped support for python 3.5:
+osprofiler<2.7.0;python_version<'3.6'
+stevedore<1.31.0;python_version<'3.6'
+debtcollector<1.22.0;python_version<'3.6'
+oslo.utils<=3.41.0;python_version<'3.6'
+
 coverage>=4.5.2
 pyudev              # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking)
 git+https://github.com/openstack-charmers/zaza.git#egg=zaza;python_version>='3.0'
 git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack
+
+# Needed for charm-glance:
+git+https://opendev.org/openstack/tempest.git#egg=tempest;python_version>='3.6'
+tempest<24.0.0;python_version<'3.6'
+
+croniter            # needed for charm-rabbitmq-server unit tests
--- a/tests/tests.yaml
+++ b/tests/tests.yaml
@ -1,5 +1,7 @@
 charm_name: ceph-radosgw
 gate_bundles:
+  - vault: groovy-victoria
+  - vault: groovy-victoria-namespaced
  - vault: focal-victoria
  - vault: focal-victoria-namespaced
  - vault: focal-ussuri-ec
@ -24,8 +26,6 @@ gate_bundles:
 smoke_bundles:
  - vault: bionic-ussuri
 dev_bundles:
-  - vault: groovy-victoria
-  - vault: groovy-victoria-namespaced
  - bionic-queens-multisite
  - bionic-rocky-multisite
 target_deploy_status:
--- a/tox.ini
+++ b/tox.ini
@ -14,6 +14,18 @@ skipsdist = True
 sitepackages = False
 # NOTE: Avoid false positives by not skipping missing interpreters.
 skip_missing_interpreters = False
+# NOTES:
+# * We avoid the new dependency resolver by pinning pip < 20.3, see
+#   https://github.com/pypa/pip/issues/9187
+# * Pinning dependencies requires tox >= 3.2.0, see
+#   https://tox.readthedocs.io/en/latest/config.html#conf-requires
+# * It is also necessary to pin virtualenv as a newer virtualenv would still
+#   lead to fetching the latest pip in the func* tox targets, see
+#   https://stackoverflow.com/a/38133283
+requires = pip < 20.3
+           virtualenv < 20.0
+# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
+minversion = 3.2.0

 [testenv]
 setenv = VIRTUAL_ENV={envdir}
--- a/unit_tests/test_hooks.py
+++ b/unit_tests/test_hooks.py
@ -198,7 +198,10 @@ class CephRadosGWTests(CharmTestCase):
        _ceph.import_radosgw_key.return_value = True
        is_leader.return_value = True
        self.relation_get.return_value = 'seckey'
-        self.multisite.list_zones.return_value = []
+        self.multisite.list_zones.side_effect = [
+            [],           # at first the default zone doesn't exist, then...
+            ['default'],  # ... it got created
+        ]
        self.socket.gethostname.return_value = 'testinghostname'
        ceph_hooks.mon_relation()
        self.relation_set.assert_not_called()
@ -218,6 +221,10 @@ class CephRadosGWTests(CharmTestCase):
        _ceph.import_radosgw_key.return_value = True
        is_leader.return_value = True
        self.relation_get.return_value = 'seckey'
+        self.multisite.list_zones.side_effect = [
+            [],           # at first the default zone doesn't exist, then...
+            ['default'],  # ... it got created
+        ]
        self.socket.gethostname.return_value = 'testinghostname'
        self.request_per_unit_key.return_value = True
        ceph_hooks.mon_relation()
@ -241,6 +248,10 @@ class CephRadosGWTests(CharmTestCase):
        _ceph.import_radosgw_key.return_value = False
        self.relation_get.return_value = None
        is_leader.return_value = True
+        self.multisite.list_zones.side_effect = [
+            [],           # at first the default zone doesn't exist, then...
+            ['default'],  # ... it got created
+        ]
        ceph_hooks.mon_relation()
        self.assertFalse(_ceph.import_radosgw_key.called)
        self.service_resume.assert_not_called()
--- a/unit_tests/test_multisite.py
+++ b/unit_tests/test_multisite.py
@ -51,7 +51,7 @@ class TestMultisiteHelpers(CharmTestCase):
                'radosgw-admin', '--id=rgw.testhost',
                'realm', 'create',
                '--rgw-realm=beedata', '--default'
-            ])
+            ], stderr=mock.ANY)

    def test_list_realms(self):
        with open(self._testdata(whoami()), 'rb') as f:
@ -86,7 +86,7 @@ class TestMultisiteHelpers(CharmTestCase):
                '--rgw-realm=beedata',
                '--default',
                '--master'
-            ])
+            ], stderr=mock.ANY)

    def test_list_zonegroups(self):
        with open(self._testdata(whoami()), 'rb') as f:
@ -117,7 +117,7 @@ class TestMultisiteHelpers(CharmTestCase):
                '--access-key=mykey',
                '--secret=mypassword',
                '--read-only=0',
-            ])
+            ], stderr=mock.ANY)

    def test_modify_zone(self):
        multisite.modify_zone(
@ -134,7 +134,7 @@ class TestMultisiteHelpers(CharmTestCase):
            '--endpoints=http://localhost:80,https://localhost:443',
            '--access-key=mykey', '--secret=secret',
            '--read-only=1',
-        ])
+        ], stderr=mock.ANY)

    def test_modify_zone_promote_master(self):
        multisite.modify_zone(
@ -149,7 +149,7 @@ class TestMultisiteHelpers(CharmTestCase):
            '--master',
            '--default',
            '--read-only=0',
-        ])
+        ], stderr=mock.ANY)

    def test_modify_zone_partial_credentials(self):
        multisite.modify_zone(
@ -163,7 +163,7 @@ class TestMultisiteHelpers(CharmTestCase):
            '--rgw-zone=brundall-east',
            '--endpoints=http://localhost:80,https://localhost:443',
            '--read-only=0',
-        ])
+        ], stderr=mock.ANY)

    def test_list_zones(self):
        with open(self._testdata(whoami()), 'rb') as f:
@ -223,7 +223,7 @@ class TestMultisiteHelpers(CharmTestCase):
            'realm', 'pull',
            '--url=http://master:80',
            '--access-key=testkey', '--secret=testsecret',
-        ])
+        ], stderr=mock.ANY)

    def test_pull_period(self):
        multisite.pull_period(url='http://master:80',
@ -234,4 +234,4 @@ class TestMultisiteHelpers(CharmTestCase):
            'period', 'pull',
            '--url=http://master:80',
            '--access-key=testkey', '--secret=testsecret',
-        ])
+        ], stderr=mock.ANY)