Merge "Add 2023.2 Bobcat support"

charmcraft.yaml

@@ -30,9 +30,9 @@ bases:
       - name: ubuntu
         channel: "22.04"
         architectures: [amd64, s390x, ppc64el, arm64]
-      - name: ubuntu
-        channel: "22.10"
-        architectures: [amd64, s390x, ppc64el, arm64]
       - name: ubuntu
         channel: "23.04"
         architectures: [amd64, s390x, ppc64el, arm64]
+      - name: ubuntu
+        channel: "23.10"
+        architectures: [amd64, s390x, ppc64el, arm64]

hooks/charmhelpers/contrib/hahelpers/cluster.py

@@ -221,6 +221,13 @@ def https():
         return True
     if config_get('ssl_cert') and config_get('ssl_key'):
         return True
+    # Local import to avoid ciruclar dependency.
+    import charmhelpers.contrib.openstack.cert_utils as cert_utils
+    if (
+        cert_utils.get_certificate_request() and not
+        cert_utils.get_requests_for_local_unit("certificates")
+    ):
+        return False
     for r_id in relation_ids('certificates'):
         for unit in relation_list(r_id):
             ca = relation_get('ca', rid=r_id, unit=unit)

hooks/charmhelpers/contrib/openstack/cert_utils.py

@@ -409,6 +409,9 @@ def get_requests_for_local_unit(relation_name=None):
     relation_name = relation_name or 'certificates'
     bundles = []
     for rid in relation_ids(relation_name):
+        sent = relation_get(rid=rid, unit=local_unit())
+        legacy_keys = ['certificate_name', 'common_name']
+        is_legacy_request = set(sent).intersection(legacy_keys)
         for unit in related_units(rid):
             data = relation_get(rid=rid, unit=unit)
             if data.get(raw_certs_key):
@@ -416,6 +419,14 @@ def get_requests_for_local_unit(relation_name=None):
                     'ca': data['ca'],
                     'chain': data.get('chain'),
                     'certs': json.loads(data[raw_certs_key])})
+            elif is_legacy_request:
+                bundles.append({
+                    'ca': data['ca'],
+                    'chain': data.get('chain'),
+                    'certs': {sent['common_name']:
+                              {'cert': data.get(local_name + '.server.cert'),
+                               'key': data.get(local_name + '.server.key')}}})
+
     return bundles
 
 

hooks/charmhelpers/contrib/openstack/context.py

@@ -1748,6 +1748,9 @@ class WSGIWorkerConfigContext(WorkerConfigContext):
 
     def __call__(self):
         total_processes = _calculate_workers()
+        enable_wsgi_rotation = config('wsgi-rotation')
+        if enable_wsgi_rotation is None:
+            enable_wsgi_rotation = True
         ctxt = {
             "service_name": self.service_name,
             "user": self.user,
@@ -1761,6 +1764,7 @@ class WSGIWorkerConfigContext(WorkerConfigContext):
             "public_processes": int(math.ceil(self.public_process_weight *
                                               total_processes)),
             "threads": 1,
+            "wsgi_rotation": enable_wsgi_rotation,
         }
         return ctxt
 

hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken

@@ -12,6 +12,8 @@ signing_dir = {{ signing_dir }}
 {% if service_type -%}
 service_type = {{ service_type }}
 {% endif -%}
+{% if admin_role -%}
 service_token_roles = {{ admin_role }}
 service_token_roles_required = True
 {% endif -%}
+{% endif -%}

hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka

@@ -22,6 +22,8 @@ signing_dir = {{ signing_dir }}
 {% if use_memcache == true %}
 memcached_servers = {{ memcache_url }}
 {% endif -%}
+{% if admin_role -%}
 service_token_roles = {{ admin_role }}
 service_token_roles_required = True
 {% endif -%}
+{% endif -%}

hooks/charmhelpers/contrib/openstack/templates/section-service-user

@@ -3,8 +3,8 @@
 send_service_user_token = true
 auth_type = password
 auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
-project_domain_id = default
+project_domain_name = service_domain
-user_domain_id = default
+user_domain_name = service_domain
 project_name = {{ admin_tenant_name }}
 username = {{ admin_user }}
 password = {{ admin_password }}

hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf

@@ -12,6 +12,12 @@ Listen {{ admin_port }}
 Listen {{ public_port }}
 {% endif -%}
 
+{% if wsgi_rotation -%}
+WSGISocketRotation On
+{% else -%}
+WSGISocketRotation Off
+{% endif -%}
+
 {% if port -%}
 <VirtualHost *:{{ port }}>
     WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ user }} group={{ group }} \

hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-metadata.conf

@@ -12,6 +12,12 @@ Listen {{ admin_port }}
 Listen {{ public_port }}
 {% endif -%}
 
+{% if wsgi_rotation -%}
+WSGISocketRotation On
+{% else -%}
+WSGISocketRotation Off
+{% endif -%}
+
 {% if port -%}
 <VirtualHost *:{{ port }}>
     WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ user }} group={{ group }} \

hooks/charmhelpers/contrib/openstack/utils.py

@@ -160,6 +160,7 @@ OPENSTACK_CODENAMES = OrderedDict([
     ('2022.1', 'yoga'),
     ('2022.2', 'zed'),
     ('2023.1', 'antelope'),
+    ('2023.2', 'bobcat'),
 ])
 
 # The ugly duckling - must list releases oldest to newest
@@ -957,7 +958,7 @@ def os_requires_version(ostack_release, pkg):
     def wrap(f):
         @wraps(f)
         def wrapped_f(*args):
-            if os_release(pkg) < ostack_release:
+            if CompareOpenStackReleases(os_release(pkg)) < ostack_release:
                 raise Exception("This hook is not supported on releases"
                                 " before %s" % ostack_release)
             f(*args)

hooks/charmhelpers/contrib/storage/linux/ceph.py

@@ -28,7 +28,6 @@ import os
 import shutil
 import json
 import time
-import uuid
 
 from subprocess import (
     check_call,
@@ -1677,6 +1676,10 @@ class CephBrokerRq(object):
     The API is versioned and defaults to version 1.
     """
 
+    # The below hash is the result of running
+    # `hashlib.sha1('[]'.encode()).hexdigest()`
+    EMPTY_LIST_SHA = '97d170e1550eee4afc0af065b78cda302a97674c'
+
     def __init__(self, api_version=1, request_id=None, raw_request_data=None):
         """Initialize CephBrokerRq object.
 
@@ -1685,8 +1688,12 @@ class CephBrokerRq(object):
 
         :param api_version: API version for request (default: 1).
         :type api_version: Optional[int]
-        :param request_id: Unique identifier for request.
+        :param request_id: Unique identifier for request. The identifier will
-                           (default: string representation of generated UUID)
+                           be updated as ops are added or removed from the
+                           broker request. This ensures that Ceph will
+                           correctly process requests where operations are
+                           added after the initial request is processed.
+                           (default: sha1 of operations)
         :type request_id: Optional[str]
         :param raw_request_data: JSON-encoded string to build request from.
         :type raw_request_data: Optional[str]
@@ -1695,16 +1702,20 @@ class CephBrokerRq(object):
         if raw_request_data:
             request_data = json.loads(raw_request_data)
             self.api_version = request_data['api-version']
-            self.request_id = request_data['request-id']
             self.set_ops(request_data['ops'])
+            self.request_id = request_data['request-id']
         else:
             self.api_version = api_version
             if request_id:
                 self.request_id = request_id
             else:
-                self.request_id = str(uuid.uuid1())
+                self.request_id = CephBrokerRq.EMPTY_LIST_SHA
             self.ops = []
 
+    def _hash_ops(self):
+        """Return the sha1 of the requested Broker ops."""
+        return hashlib.sha1(json.dumps(self.ops, sort_keys=True).encode()).hexdigest()
+
     def add_op(self, op):
         """Add an op if it is not already in the list.
 
@@ -1713,6 +1724,7 @@ class CephBrokerRq(object):
         """
         if op not in self.ops:
             self.ops.append(op)
+            self.request_id = self._hash_ops()
 
     def add_op_request_access_to_group(self, name, namespace=None,
                                        permission=None, key_name=None,
@@ -1991,6 +2003,7 @@ class CephBrokerRq(object):
         to allow comparisons to ensure validity.
         """
         self.ops = ops
+        self.request_id = self._hash_ops()
 
     @property
     def request(self):

hooks/charmhelpers/core/host_factory/ubuntu.py

@@ -32,6 +32,7 @@ UBUNTU_RELEASES = (
     'jammy',
     'kinetic',
     'lunar',
+    'mantic',
 )
 
 

hooks/charmhelpers/fetch/ubuntu.py

@@ -238,6 +238,14 @@ CLOUD_ARCHIVE_POCKETS = {
     'antelope/proposed': 'jammy-proposed/antelope',
     'jammy-antelope/proposed': 'jammy-proposed/antelope',
     'jammy-proposed/antelope': 'jammy-proposed/antelope',
+    # bobcat
+    'bobcat': 'jammy-updates/bobcat',
+    'jammy-bobcat': 'jammy-updates/bobcat',
+    'jammy-bobcat/updates': 'jammy-updates/bobcat',
+    'jammy-updates/bobcat': 'jammy-updates/bobcat',
+    'bobcat/proposed': 'jammy-proposed/bobcat',
+    'jammy-bobcat/proposed': 'jammy-proposed/bobcat',
+    'jammy-proposed/bobcat': 'jammy-proposed/bobcat',
 
     # OVN
     'focal-ovn-22.03': 'focal-updates/ovn-22.03',
@@ -270,6 +278,7 @@ OPENSTACK_RELEASES = (
     'yoga',
     'zed',
     'antelope',
+    'bobcat',
 )
 
 
@@ -298,6 +307,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('jammy', 'yoga'),
     ('kinetic', 'zed'),
     ('lunar', 'antelope'),
+    ('mantic', 'bobcat'),
 ])
 
 
@@ -591,7 +601,7 @@ def _get_key_by_keyid(keyid):
     curl_cmd = ['curl', keyserver_url.format(keyid)]
     # use proxy server settings in order to retrieve the key
     return subprocess.check_output(curl_cmd,
-                                   env=env_proxy_settings(['https']))
+                                   env=env_proxy_settings(['https', 'no_proxy']))
 
 
 def _dearmor_gpg_key(key_asc):

hooks/charmhelpers/fetch/ubuntu_apt_pkg.py

@@ -122,13 +122,12 @@ class Cache(object):
         :raises: subprocess.CalledProcessError
         """
         pkgs = {}
-        cmd = ['dpkg-query', '--list']
+        cmd = [
+            'dpkg-query', '--show',
+            '--showformat',
+            r'${db:Status-Abbrev}\t${Package}\t${Version}\t${Architecture}\t${binary:Summary}\n'
+        ]
         cmd.extend(packages)
-        if locale.getlocale() == (None, None):
-            # subprocess calls out to locale.getpreferredencoding(False) to
-            # determine encoding.  Workaround for Trusty where the
-            # environment appears to not be set up correctly.
-            locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
         try:
             output = subprocess.check_output(cmd,
                                              stderr=subprocess.STDOUT,
@@ -140,24 +139,17 @@ class Cache(object):
             if cp.returncode != 1:
                 raise
             output = cp.output
-        headings = []
         for line in output.splitlines():
-            if line.startswith('||/'):
+            # only process lines for successfully installed packages
-                headings = line.split()
+            if not (line.startswith('ii ') or line.startswith('hi ')):
-                headings.pop(0)
                 continue
-            elif (line.startswith('|') or line.startswith('+') or
+            status, name, version, arch, desc = line.split('\t', 4)
-                  line.startswith('dpkg-query:')):
+            pkgs[name] = {
-                continue
+                'name': name,
-            else:
+                'version': version,
-                data = line.split(None, 4)
+                'architecture': arch,
-                status = data.pop(0)
+                'description': desc,
-                if status not in ('ii', 'hi'):
+            }
-                    continue
-                pkg = {}
-                pkg.update({k.lower(): v for k, v in zip(headings, data)})
-                if 'name' in pkg:
-                    pkgs.update({pkg['name']: pkg})
         return pkgs
 
     def _apt_cache_show(self, packages):

metadata.yaml

@@ -19,8 +19,8 @@ tags:
 - openstack
 series:
 - jammy
-- kinetic
 - lunar
+- mantic
 extra-bindings:
   data:
 provides:

osci.yaml

@@ -6,4 +6,4 @@
       needs_charm_build: true
       charm_build_name: neutron-openvswitch
       build_type: charmcraft
-      charmcraft_channel: 2.1/stable
+      charmcraft_channel: 2.x/stable

tests/bundles/jammy-bobcat.yaml

@@ -1,5 +1,5 @@
 variables:
-  openstack-origin: &openstack-origin cloud:jammy-zed
+  openstack-origin: &openstack-origin cloud:jammy-bobcat
 
 series: &series jammy
 

tests/bundles/mantic-bobcat.yaml

@@ -0,0 +1,222 @@
+variables:
+  openstack-origin: &openstack-origin distro
+
+series: &series mantic
+
+machines:
+  0:
+    constraints: "mem=3072M"
+  1:
+    constraints: "mem=3072M"
+  2:
+    constraints: "mem=3072M"
+  3: {}
+  4: {}
+  5: {}
+  6: {}
+  7: {}
+  8:
+    constraints: "root-disk=20G mem=4G"
+  9:
+    constraints: "root-disk=20G mem=4G"
+  10: {}
+  11: {}
+  12: {}
+
+# We specify machine placements for these to improve iteration
+# time, given that machine "0" comes up way before machine "7"
+applications:
+  keystone-mysql-router:
+    charm: ch:mysql-router
+    channel: latest/edge
+  nova-mysql-router:
+    charm: ch:mysql-router
+    channel: latest/edge
+  glance-mysql-router:
+    charm: ch:mysql-router
+    channel: latest/edge
+  neutron-mysql-router:
+    charm: ch:mysql-router
+    channel: latest/edge
+  placement-mysql-router:
+    charm: ch:mysql-router
+    channel: latest/edge
+  vault-mysql-router:
+    charm: ch:mysql-router
+    channel: latest/edge
+  mysql-innodb-cluster:
+    charm: ch:mysql-innodb-cluster
+    num_units: 3
+    options:
+      source: *openstack-origin
+    to:
+      - '0'
+      - '1'
+      - '2'
+    channel: latest/edge
+  rabbitmq-server:
+    charm: ch:rabbitmq-server
+    num_units: 1
+    to:
+      - '3'
+    channel: latest/edge
+  neutron-api:
+    charm: ch:neutron-api
+    series: *series
+    num_units: 1
+    options:
+      manage-neutron-plugin-legacy-mode: true
+      overlay-network-type: 'vxlan'
+      l2-population: True
+      flat-network-providers: physnet1
+      neutron-security-groups: true
+      openstack-origin: *openstack-origin
+      enable-dvr: True
+      enable-qos: True
+    to:
+      - '4'
+    channel: latest/edge
+  keystone:
+    charm: ch:keystone
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '5'
+    channel: latest/edge
+  glance:
+    charm: ch:glance
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '6'
+    channel: latest/edge
+  neutron-openvswitch:
+    charm: ../../neutron-openvswitch.charm
+    options:
+      use-dvr-snat: True
+      bridge-mappings: physnet1:br-ex
+      enable-local-dhcp-and-metadata: True
+  nova-cloud-controller:
+    charm: ch:nova-cloud-controller
+    num_units: 1
+    options:
+      network-manager: Neutron
+      openstack-origin: *openstack-origin
+    to:
+      - '7'
+    channel: latest/edge
+  nova-compute:
+    charm: ch:nova-compute
+    num_units: 2
+    options:
+      config-flags: default_ephemeral_format=ext4
+      enable-live-migration: true
+      enable-resize: true
+      migration-auth-type: ssh
+      openstack-origin: *openstack-origin
+    to:
+      - '8'
+      - '9'
+    channel: latest/edge
+  placement:
+    charm: ch:placement
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '10'
+    channel: latest/edge
+  vault-mysql-router:
+    charm: ch:mysql-router
+    channel: latest/edge
+  vault:
+    charm: ch:vault
+    num_units: 1
+    to:
+      - '11'
+    channel: latest/edge
+  ovn-central:
+    charm: ch:ovn-central
+    num_units: 3
+    options:
+      source: *openstack-origin
+    to:
+      - '12'
+    channel: latest/edge
+  neutron-api-plugin-ovn:
+    charm: ch:neutron-api-plugin-ovn
+    channel: latest/edge
+  ovn-chassis:
+    charm: ch:ovn-chassis
+    options:
+      # start new units paused to allow unit by unit OVS to OVN migration
+      new-units-paused: true
+    channel: latest/edge
+relations:
+  - - 'neutron-api:amqp'
+    - 'rabbitmq-server:amqp'
+  - - 'neutron-api:neutron-api'
+    - 'nova-cloud-controller:neutron-api'
+  - - 'neutron-api:identity-service'
+    - 'keystone:identity-service'
+  - - 'nova-compute:neutron-plugin'
+    - 'neutron-openvswitch:neutron-plugin'
+  - - 'neutron-api:neutron-plugin-api'
+    - 'neutron-openvswitch:neutron-plugin-api'
+  - - 'nova-cloud-controller:amqp'
+    - 'rabbitmq-server:amqp'
+  - - 'nova-compute:amqp'
+    - 'rabbitmq-server:amqp'
+  - - 'neutron-openvswitch:amqp'
+    - 'rabbitmq-server:amqp'
+  - - 'nova-cloud-controller:identity-service'
+    - 'keystone:identity-service'
+  - - 'nova-cloud-controller:cloud-compute'
+    - 'nova-compute:cloud-compute'
+  - - 'glance:identity-service'
+    - 'keystone:identity-service'
+  - - 'glance:amqp'
+    - 'rabbitmq-server:amqp'
+  - - 'nova-compute:image-service'
+    - 'glance:image-service'
+  - - 'nova-cloud-controller:image-service'
+    - 'glance:image-service'
+  - - 'placement:identity-service'
+    - 'keystone:identity-service'
+  - - 'placement:placement'
+    - 'nova-cloud-controller:placement'
+  - ["keystone:shared-db", "keystone-mysql-router:shared-db"]
+  - ["glance:shared-db", "glance-mysql-router:shared-db"]
+  - ["nova-cloud-controller:shared-db", "nova-mysql-router:shared-db"]
+  - ["neutron-api:shared-db", "neutron-mysql-router:shared-db"]
+  - ["placement:shared-db", "placement-mysql-router:shared-db"]
+  - ["keystone-mysql-router:db-router", "mysql-innodb-cluster:db-router"]
+  - ["nova-mysql-router:db-router", "mysql-innodb-cluster:db-router"]
+  - ["glance-mysql-router:db-router", "mysql-innodb-cluster:db-router"]
+  - ["neutron-mysql-router:db-router", "mysql-innodb-cluster:db-router"]
+  - ["placement-mysql-router:db-router", "mysql-innodb-cluster:db-router"]
+  # We need to defer the addition of the neutron-api-plugin-ovn subordinate
+  # relation to the functional test as the test will first validate the legacy
+  # Neutron ML2+OVS topology, migrate it to OVN and then confirm connectivity
+  # post migration.
+  #
+  # - - neutron-api-plugin-ovn:neutron-plugin
+  #   - neutron-api:neutron-plugin-api-subordinate
+  - - ovn-central:certificates
+    - vault:certificates
+  - - ovn-central:ovsdb-cms
+    - neutron-api-plugin-ovn:ovsdb-cms
+  - - ovn-chassis:nova-compute
+    - nova-compute:neutron-plugin
+  - - ovn-chassis:certificates
+    - vault:certificates
+  - - ovn-chassis:ovsdb
+    - ovn-central:ovsdb
+  - - vault:certificates
+    - neutron-api-plugin-ovn:certificates
+  - - vault:shared-db
+    - vault-mysql-router:shared-db
+  - - vault-mysql-router:db-router
+    - mysql-innodb-cluster:db-router

tests/tests.yaml

@@ -6,14 +6,15 @@ charm_name: neutron-openvswitch
 # OVS to OVN.
 
 smoke_bundles:
-  - jammy-zed
+  - jammy-antelope
 
 gate_bundles:
-  - jammy-zed
+  - jammy-antelope
 
 dev_bundles:
-  - jammy-antelope
+  - jammy-bobcat
   - lunar-antelope
+  - mantic-bobcat
 
 configure:
   - zaza.openstack.charm_tests.vault.setup.auto_initialize_no_validation
@@ -57,3 +58,4 @@ tests_options:
   zaza.openstack.charm_tests.neutron.tests.NeutronNetworkingTest.test_instances_have_networking.run_resource_cleanup: false
   force_deploy:
     - lunar-antelope-dvr-snat
+    - mantic-bobcat-dvr-snat