Updates for caracal testing support

These updates, on the master branch, are to support testing the caracal
packages and support of the charms for caracal.  They do NOT lock the charms
down, and don't change the testing branches to stable branches.

Change-Id: I121370811bec483cb026559cbd4b759d54f10c23

charmcraft.yaml

@@ -30,9 +30,3 @@ bases:
       - name: ubuntu
         channel: "22.04"
         architectures: [amd64, s390x, ppc64el, arm64]
-      - name: ubuntu
-        channel: "23.04"
-        architectures: [amd64, s390x, ppc64el, arm64]
-      - name: ubuntu
-        channel: "23.10"
-        architectures: [amd64, s390x, ppc64el, arm64]

hooks/charmhelpers/contrib/openstack/cert_utils.py

@@ -414,18 +414,27 @@ def get_requests_for_local_unit(relation_name=None):
         is_legacy_request = set(sent).intersection(legacy_keys)
         for unit in related_units(rid):
             data = relation_get(rid=rid, unit=unit)
-            if data.get(raw_certs_key):
+            # Note: Bug#2028683 - data may not be available if the certificates
-                bundles.append({
+            # relation hasn't been populated by the providing charm. If no 'ca'
-                    'ca': data['ca'],
+            # in the data then don't attempt the bundle at all.
-                    'chain': data.get('chain'),
+            if data.get('ca'):
-                    'certs': json.loads(data[raw_certs_key])})
+                if data.get(raw_certs_key):
-            elif is_legacy_request:
+                    bundles.append({
-                bundles.append({
+                        'ca': data['ca'],
-                    'ca': data['ca'],
+                        'chain': data.get('chain'),
-                    'chain': data.get('chain'),
+                        'certs': json.loads(data[raw_certs_key])
-                    'certs': {sent['common_name']:
+                    })
-                              {'cert': data.get(local_name + '.server.cert'),
+                elif is_legacy_request:
-                               'key': data.get(local_name + '.server.key')}}})
+                    bundles.append({
+                        'ca': data['ca'],
+                        'chain': data.get('chain'),
+                        'certs': {
+                            sent['common_name']: {
+                                'cert': data.get(local_name + '.server.cert'),
+                                'key': data.get(local_name + '.server.key')
+                            }
+                        }
+                    })
 
     return bundles
 

hooks/charmhelpers/contrib/openstack/context.py

@@ -545,7 +545,7 @@ class IdentityServiceContext(OSContextGenerator):
                         'internal_auth_url': internal_auth_url,
                     })
 
-                # we keep all veriables in ctxt for compatibility and
+                # we keep all variables in ctxt for compatibility and
                 # add nested dictionary for keystone_authtoken generic
                 # templating
                 if keystonemiddleware_os_release:
@@ -557,6 +557,7 @@ class IdentityServiceContext(OSContextGenerator):
                     # NOTE(jamespage) this is required for >= icehouse
                     # so a missing value just indicates keystone needs
                     # upgrading
+                    ctxt['admin_user_id'] = _resolve('service_user_id')
                     ctxt['admin_tenant_id'] = _resolve('service_tenant_id')
                     ctxt['admin_domain_id'] = _resolve('service_domain_id')
                     return ctxt
@@ -1748,9 +1749,9 @@ class WSGIWorkerConfigContext(WorkerConfigContext):
 
     def __call__(self):
         total_processes = _calculate_workers()
-        enable_wsgi_rotation = config('wsgi-rotation')
+        enable_wsgi_socket_rotation = config('wsgi-socket-rotation')
-        if enable_wsgi_rotation is None:
+        if enable_wsgi_socket_rotation is None:
-            enable_wsgi_rotation = True
+            enable_wsgi_socket_rotation = True
         ctxt = {
             "service_name": self.service_name,
             "user": self.user,
@@ -1764,7 +1765,7 @@ class WSGIWorkerConfigContext(WorkerConfigContext):
             "public_processes": int(math.ceil(self.public_process_weight *
                                               total_processes)),
             "threads": 1,
-            "wsgi_rotation": enable_wsgi_rotation,
+            "wsgi_socket_rotation": enable_wsgi_socket_rotation,
         }
         return ctxt
 

hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf

@@ -12,7 +12,7 @@ Listen {{ admin_port }}
 Listen {{ public_port }}
 {% endif -%}
 
-{% if wsgi_rotation -%}
+{% if wsgi_socket_rotation -%}
 WSGISocketRotation On
 {% else -%}
 WSGISocketRotation Off

hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-metadata.conf

@@ -12,7 +12,7 @@ Listen {{ admin_port }}
 Listen {{ public_port }}
 {% endif -%}
 
-{% if wsgi_rotation -%}
+{% if wsgi_socket_rotation -%}
 WSGISocketRotation On
 {% else -%}
 WSGISocketRotation Off

hooks/charmhelpers/contrib/openstack/utils.py

@@ -161,6 +161,7 @@ OPENSTACK_CODENAMES = OrderedDict([
     ('2022.2', 'zed'),
     ('2023.1', 'antelope'),
     ('2023.2', 'bobcat'),
+    ('2024.1', 'caracal'),
 ])
 
 # The ugly duckling - must list releases oldest to newest
@@ -416,17 +417,6 @@ def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES,
     error_out(e)
 
 
-def get_os_version_codename_swift(codename):
-    '''Determine OpenStack version number of swift from codename.'''
-    # for k, v in six.iteritems(SWIFT_CODENAMES):
-    for k, v in SWIFT_CODENAMES.items():
-        if k == codename:
-            return v[-1]
-    e = 'Could not derive swift version for '\
-        'codename: %s' % codename
-    error_out(e)
-
-
 def get_swift_codename(version):
     '''Determine OpenStack codename that corresponds to swift version.'''
     codenames = [k for k, v in SWIFT_CODENAMES.items() if version in v]
@@ -585,7 +575,6 @@ def get_installed_os_version():
     return openstack_release().get('OPENSTACK_CODENAME')
 
 
-@cached
 def openstack_release():
     """Return /etc/os-release in a dict."""
     d = {}
@@ -847,14 +836,10 @@ def openstack_upgrade_available(package):
     if not cur_vers:
         # The package has not been installed yet do not attempt upgrade
         return False
-    if "swift" in package:
+    try:
-        codename = get_os_codename_install_source(src)
+        avail_vers = get_os_version_install_source(src)
-        avail_vers = get_os_version_codename_swift(codename)
+    except Exception:
-    else:
+        avail_vers = cur_vers
-        try:
-            avail_vers = get_os_version_install_source(src)
-        except Exception:
-            avail_vers = cur_vers
     apt.init()
     return apt.version_compare(avail_vers, cur_vers) >= 1
 

hooks/charmhelpers/contrib/storage/linux/ceph.py

@@ -158,15 +158,19 @@ def get_osd_settings(relation_name):
     return _order_dict_by_key(osd_settings)
 
 
-def send_application_name(relid=None):
+def send_application_name(relid=None, app_name=None):
     """Send the application name down the relation.
 
     :param relid: Relation id to set application name in.
     :type relid: str
+    :param app_name: Application name to send in the relation.
+    :type app_name: str
     """
+    if app_name is None:
+        app_name = application_name()
     relation_set(
         relation_id=relid,
-        relation_settings={'application-name': application_name()})
+        relation_settings={'application-name': app_name})
 
 
 def send_osd_settings():

hooks/charmhelpers/core/unitdata.py

@@ -151,6 +151,7 @@ import contextlib
 import datetime
 import itertools
 import json
+import logging
 import os
 import pprint
 import sqlite3
@@ -521,6 +522,42 @@ _KV = None
 
 def kv():
     global _KV
+
+    # If we are running unit tests, it is useful to go into memory-backed KV store to
+    # avoid concurrency issues when running multiple tests. This is not a
+    # problem when juju is running normally.
+
+    env_var = os.environ.get("CHARM_HELPERS_TESTMODE", "auto").lower()
+    if env_var not in ["auto", "no", "yes"]:
+        logging.warning("Unknown value for CHARM_HELPERS_TESTMODE '%s'"
+                        ", assuming 'no'", env_var)
+        env_var = "no"
+
+    if env_var == "no":
+        in_memory_db = False
+    elif env_var == "yes":
+        in_memory_db = True
+    elif env_var == "auto":
+        # If UNIT_STATE_DB is set, respect this request
+        if "UNIT_STATE_DB" in os.environ:
+            in_memory_db = False
+        # Autodetect normal juju execution by looking for juju variables
+        elif "JUJU_CHARM_DIR" in os.environ or "JUJU_UNIT_NAME" in os.environ:
+            in_memory_db = False
+        else:
+            # We are probably running in unit test mode
+            logging.warning("Auto-detected unit test environment for KV store.")
+            in_memory_db = True
+    else:
+        # Help the linter realise that in_memory_db is always set
+        raise Exception("Cannot reach this line")
+
     if _KV is None:
-        _KV = Storage()
+        if in_memory_db:
+            _KV = Storage(":memory:")
+        else:
+            _KV = Storage()
+    else:
+        if in_memory_db and _KV.db_path != ":memory:":
+            logging.warning("Running with in_memory_db and KV is not set to :memory:")
     return _KV

hooks/charmhelpers/fetch/snap.py

@@ -52,7 +52,7 @@ def _snap_exec(commands):
     :param commands: List commands
     :return: Integer exit code
     """
-    assert type(commands) == list
+    assert isinstance(commands, list)
 
     retry_count = 0
     return_code = None

hooks/charmhelpers/fetch/ubuntu.py

@@ -246,6 +246,14 @@ CLOUD_ARCHIVE_POCKETS = {
     'bobcat/proposed': 'jammy-proposed/bobcat',
     'jammy-bobcat/proposed': 'jammy-proposed/bobcat',
     'jammy-proposed/bobcat': 'jammy-proposed/bobcat',
+    # caracal
+    'caracal': 'jammy-updates/caracal',
+    'jammy-caracal': 'jammy-updates/caracal',
+    'jammy-caracal/updates': 'jammy-updates/caracal',
+    'jammy-updates/caracal': 'jammy-updates/caracal',
+    'caracal/proposed': 'jammy-proposed/caracal',
+    'jammy-caracal/proposed': 'jammy-proposed/caracal',
+    'jammy-proposed/caracal': 'jammy-proposed/caracal',
 
     # OVN
     'focal-ovn-22.03': 'focal-updates/ovn-22.03',
@@ -279,6 +287,7 @@ OPENSTACK_RELEASES = (
     'zed',
     'antelope',
     'bobcat',
+    'caracal',
 )
 
 
@@ -308,6 +317,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('kinetic', 'zed'),
     ('lunar', 'antelope'),
     ('mantic', 'bobcat'),
+    ('noble', 'caracal'),
 ])
 
 

metadata.yaml

@@ -15,11 +15,9 @@ description: |
   of a Neutron based OpenStack deployment
 docs: https://discourse.charmhub.io/t/neutron-gateway-docs-index/11000
 tags:
-- openstack
+  - openstack
 series:
-- jammy
+  - jammy
-- lunar
-- mantic
 extra-bindings:
   data:
 provides:

tests/bundles/jammy-antelope.yaml

@@ -74,7 +74,7 @@ applications:
       enable-qos: true
     to:
       - '4'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   keystone:
     charm: ch:keystone
@@ -83,7 +83,7 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '5'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   glance:
     charm: ch:glance
@@ -92,11 +92,11 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '6'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   neutron-openvswitch:
     charm: ch:neutron-openvswitch
-    channel: latest/edge
+    channel: 2024.1/edge
   neutron-gateway:
     charm: ../../neutron-gateway.charm
     num_units: 1
@@ -126,7 +126,7 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '8'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   nova-compute:
     charm: ch:nova-compute
@@ -136,7 +136,7 @@ applications:
     to:
       - '9'
       - '10'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   placement:
     charm: ch:placement
@@ -145,7 +145,7 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '11'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   vault-mysql-router:
     charm: ch:mysql-router
@@ -166,7 +166,7 @@ applications:
     channel: latest/edge
   neutron-api-plugin-ovn:
     charm: ch:neutron-api-plugin-ovn
-    channel: latest/edge
+    channel: 2024.1/edge
   ovn-chassis:
     charm: ch:ovn-chassis
     options:

tests/bundles/jammy-bobcat.yaml

@@ -74,7 +74,7 @@ applications:
       enable-qos: true
     to:
       - '4'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   keystone:
     charm: ch:keystone
@@ -83,7 +83,7 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '5'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   glance:
     charm: ch:glance
@@ -92,11 +92,11 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '6'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   neutron-openvswitch:
     charm: ch:neutron-openvswitch
-    channel: latest/edge
+    channel: 2024.1/edge
   neutron-gateway:
     charm: ../../neutron-gateway.charm
     num_units: 1
@@ -126,7 +126,7 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '8'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   nova-compute:
     charm: ch:nova-compute
@@ -136,7 +136,7 @@ applications:
     to:
       - '9'
       - '10'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   placement:
     charm: ch:placement
@@ -145,7 +145,7 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '11'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   vault-mysql-router:
     charm: ch:mysql-router
@@ -166,7 +166,7 @@ applications:
     channel: latest/edge
   neutron-api-plugin-ovn:
     charm: ch:neutron-api-plugin-ovn
-    channel: latest/edge
+    channel: 2024.1/edge
   ovn-chassis:
     charm: ch:ovn-chassis
     options:

tests/bundles/jammy-caracal.yaml

@@ -1,7 +1,7 @@
 variables:
-  openstack-origin: &openstack-origin distro
+  openstack-origin: &openstack-origin cloud:jammy-caracal
 
-series: &series mantic
+series: &series jammy
 
 machines:
   0:
@@ -74,7 +74,7 @@ applications:
       enable-qos: true
     to:
       - '4'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   keystone:
     charm: ch:keystone
@@ -83,7 +83,7 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '5'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   glance:
     charm: ch:glance
@@ -92,11 +92,11 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '6'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   neutron-openvswitch:
     charm: ch:neutron-openvswitch
-    channel: latest/edge
+    channel: 2024.1/edge
   neutron-gateway:
     charm: ../../neutron-gateway.charm
     num_units: 1
@@ -126,7 +126,7 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '8'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   nova-compute:
     charm: ch:nova-compute
@@ -136,7 +136,7 @@ applications:
     to:
       - '9'
       - '10'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   placement:
     charm: ch:placement
@@ -145,7 +145,7 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '11'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   vault-mysql-router:
     charm: ch:mysql-router
@@ -166,7 +166,7 @@ applications:
     channel: latest/edge
   neutron-api-plugin-ovn:
     charm: ch:neutron-api-plugin-ovn
-    channel: latest/edge
+    channel: 2024.1/edge
   ovn-chassis:
     charm: ch:ovn-chassis
     options:

tests/bundles/noble-caracal.yaml

@@ -1,7 +1,7 @@
 variables:
   openstack-origin: &openstack-origin distro
 
-series: &series lunar
+series: &series noble
 
 machines:
   0:
@@ -74,7 +74,7 @@ applications:
       enable-qos: true
     to:
       - '4'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   keystone:
     charm: ch:keystone
@@ -83,7 +83,7 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '5'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   glance:
     charm: ch:glance
@@ -92,11 +92,11 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '6'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   neutron-openvswitch:
     charm: ch:neutron-openvswitch
-    channel: latest/edge
+    channel: 2024.1/edge
   neutron-gateway:
     charm: ../../neutron-gateway.charm
     num_units: 1
@@ -126,7 +126,7 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '8'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   nova-compute:
     charm: ch:nova-compute
@@ -136,7 +136,7 @@ applications:
     to:
       - '9'
       - '10'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   placement:
     charm: ch:placement
@@ -145,7 +145,7 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '11'
-    channel: latest/edge
+    channel: 2024.1/edge
 
   vault-mysql-router:
     charm: ch:mysql-router
@@ -166,7 +166,7 @@ applications:
     channel: latest/edge
   neutron-api-plugin-ovn:
     charm: ch:neutron-api-plugin-ovn
-    channel: latest/edge
+    channel: 2024.1/edge
   ovn-chassis:
     charm: ch:ovn-chassis
     options:

tests/tests.yaml

@@ -6,16 +6,13 @@ charm_name: neutron-gateway
 # OVS to OVN.
 #
 smoke_bundles:
-  - migrate-ovn: jammy-antelope
+  - jammy-caracal
-
 gate_bundles:
-  - migrate-ovn: jammy-antelope
+  - jammy-antelope
-
+  - jammy-bobcat
+  - jammy-caracal
 dev_bundles:
-  - migrate-ovn: jammy-bobcat
+  - noble-caracal
-  - migrate-ovn: lunar-antelope
-  - migrate-ovn: mantic-bobcat
-
 target_deploy_status:
   neutron-api-plugin-ovn:
     # NOTE(lourot): at first we deploy a bundle in which this subordinate charm
@@ -43,13 +40,13 @@ configure:
   - zaza.openstack.charm_tests.nova.setup.manage_ssh_key
   - zaza.openstack.charm_tests.keystone.setup.add_demo_user
   - migrate-ovn:
-    - zaza.openstack.charm_tests.vault.setup.auto_initialize_no_validation
+      - zaza.openstack.charm_tests.vault.setup.auto_initialize_no_validation
-    - zaza.openstack.charm_tests.glance.setup.add_lts_image
+      - zaza.openstack.charm_tests.glance.setup.add_lts_image
-    - zaza.openstack.charm_tests.neutron.setup.basic_overcloud_network
+      - zaza.openstack.charm_tests.neutron.setup.basic_overcloud_network
-    - zaza.openstack.charm_tests.nova.setup.create_flavors
+      - zaza.openstack.charm_tests.nova.setup.create_flavors
-    - zaza.openstack.charm_tests.nova.setup.manage_ssh_key
+      - zaza.openstack.charm_tests.nova.setup.manage_ssh_key
-    - zaza.openstack.charm_tests.keystone.setup.add_demo_user
+      - zaza.openstack.charm_tests.keystone.setup.add_demo_user
-    - zaza.openstack.charm_tests.ovn.setup.pre_migration_configuration
+      - zaza.openstack.charm_tests.ovn.setup.pre_migration_configuration
 
 configure_options:
   configure_gateway_ext_port_use_juju_wait: false
@@ -63,16 +60,16 @@ tests:
   - zaza.openstack.charm_tests.neutron.tests.NeutronOvsVsctlTest
   - zaza.openstack.charm_tests.neutron.tests.NeutronBridgePortMappingTest
   - migrate-ovn:
-    - zaza.openstack.charm_tests.neutron.tests.NeutronGatewayDeferredRestartTest
+      - zaza.openstack.charm_tests.neutron.tests.NeutronGatewayDeferredRestartTest
-    - zaza.openstack.charm_tests.neutron.tests.NeutronGatewayTest
+      - zaza.openstack.charm_tests.neutron.tests.NeutronGatewayTest
-    - zaza.openstack.charm_tests.neutron.tests.NeutronGatewayShowActionsTest
+      - zaza.openstack.charm_tests.neutron.tests.NeutronGatewayShowActionsTest
-    - zaza.openstack.charm_tests.neutron.tests.SecurityTest
+      - zaza.openstack.charm_tests.neutron.tests.SecurityTest
-    - zaza.openstack.charm_tests.neutron.tests.NeutronOvsVsctlTest
+      - zaza.openstack.charm_tests.neutron.tests.NeutronOvsVsctlTest
-    - zaza.openstack.charm_tests.neutron.tests.NeutronBridgePortMappingTest
+      - zaza.openstack.charm_tests.neutron.tests.NeutronBridgePortMappingTest
-    - zaza.openstack.charm_tests.neutron.tests.NeutronNetworkingTest
+      - zaza.openstack.charm_tests.neutron.tests.NeutronNetworkingTest
     # Note(wolsen) - temporarily disabled due to LP #1968647
     # - zaza.openstack.charm_tests.ovn.tests.OVSOVNMigrationTest
-    - zaza.openstack.charm_tests.neutron.tests.NeutronNetworkingTest
+      - zaza.openstack.charm_tests.neutron.tests.NeutronNetworkingTest
 
 tests_options:
   # NOTE: This allows us to run the NeutronNetworkingTest multiple times while
@@ -80,5 +77,4 @@ tests_options:
   # allows verifying instances survive a SDN migration.
   zaza.openstack.charm_tests.neutron.tests.NeutronNetworkingTest.test_instances_have_networking.run_resource_cleanup: false
   force_deploy:
-    - lunar-antelope
+    - noble-caracal
-    - mantic-bobcat