[ussuri] Ensure get_requests_for_local_unit doesn't fail on incomplete relation

This is a rebuild/make sync for charms to pickup the fix in charmhelpers to fix
any inadvertant accesses of ['ca'] in the relation data before it is available
from vault in the certificates relation.  Fix in charmhelpers is in [1].

[1] https://github.com/juju/charm-helpers/pull/831
Closes-Bug: #2028683

Change-Id: I28dd638060afe4f4aacbc44f12f573672eaaa6b8

charmhelpers/contrib/hahelpers/cluster.py

@@ -224,6 +224,13 @@ def https():
         return True
     if config_get('ssl_cert') and config_get('ssl_key'):
         return True
+    # Local import to avoid ciruclar dependency.
+    import charmhelpers.contrib.openstack.cert_utils as cert_utils
+    if (
+        cert_utils.get_certificate_request() and not
+        cert_utils.get_requests_for_local_unit("certificates")
+    ):
+        return False
     for r_id in relation_ids('certificates'):
         for unit in relation_list(r_id):
             ca = relation_get('ca', rid=r_id, unit=unit)
@@ -327,7 +334,7 @@ def valid_hacluster_config():
     '''
     vip = config_get('vip')
     dns = config_get('dns-ha')
-    if not(bool(vip) ^ bool(dns)):
+    if not (bool(vip) ^ bool(dns)):
         msg = ('HA: Either vip or dns-ha must be set but not both in order to '
                'use high availability')
         status_set('blocked', msg)

charmhelpers/contrib/network/ip.py

@@ -477,7 +477,7 @@ def ns_query(address):
 
     try:
         answers = dns.resolver.query(address, rtype)
-    except dns.resolver.NXDOMAIN:
+    except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers):
         return None
 
     if answers:
@@ -552,7 +552,7 @@ def port_has_listener(address, port):
     """
     cmd = ['nc', '-z', address, str(port)]
     result = subprocess.call(cmd)
-    return not(bool(result))
+    return not (bool(result))
 
 
 def assert_charm_supports_ipv6():

charmhelpers/contrib/openstack/cert_utils.py

@@ -409,13 +409,33 @@ def get_requests_for_local_unit(relation_name=None):
     relation_name = relation_name or 'certificates'
     bundles = []
     for rid in relation_ids(relation_name):
+        sent = relation_get(rid=rid, unit=local_unit())
+        legacy_keys = ['certificate_name', 'common_name']
+        is_legacy_request = set(sent).intersection(legacy_keys)
         for unit in related_units(rid):
             data = relation_get(rid=rid, unit=unit)
-            if data.get(raw_certs_key):
+            # Note: Bug#2028683 - data may not be available if the certificates
-                bundles.append({
+            # relation hasn't been populated by the providing charm. If no 'ca'
-                    'ca': data['ca'],
+            # in the data then don't attempt the bundle at all.
-                    'chain': data.get('chain'),
+            if data.get('ca'):
-                    'certs': json.loads(data[raw_certs_key])})
+                if data.get(raw_certs_key):
+                    bundles.append({
+                        'ca': data['ca'],
+                        'chain': data.get('chain'),
+                        'certs': json.loads(data[raw_certs_key])
+                    })
+                elif is_legacy_request:
+                    bundles.append({
+                        'ca': data['ca'],
+                        'chain': data.get('chain'),
+                        'certs': {
+                            sent['common_name']: {
+                                'cert': data.get(local_name + '.server.cert'),
+                                'key': data.get(local_name + '.server.key')
+                            }
+                        }
+                    })
+
     return bundles
 
 

charmhelpers/contrib/openstack/context.py

@@ -434,6 +434,9 @@ class IdentityServiceContext(OSContextGenerator):
             ('password', ctxt.get('admin_password', '')),
             ('signing_dir', ctxt.get('signing_dir', '')),))
 
+        if ctxt.get('service_type'):
+            c.update((('service_type', ctxt.get('service_type')),))
+
         return c
 
     def __call__(self):
@@ -460,6 +463,7 @@ class IdentityServiceContext(OSContextGenerator):
                 int_host = format_ipv6_addr(int_host) or int_host
                 svc_protocol = rdata.get('service_protocol') or 'http'
                 auth_protocol = rdata.get('auth_protocol') or 'http'
+                admin_role = rdata.get('admin_role') or 'Admin'
                 int_protocol = rdata.get('internal_protocol') or 'http'
                 api_version = rdata.get('api_version') or '2.0'
                 ctxt.update({'service_port': rdata.get('service_port'),
@@ -471,11 +475,15 @@ class IdentityServiceContext(OSContextGenerator):
                              'admin_tenant_name': rdata.get('service_tenant'),
                              'admin_user': rdata.get('service_username'),
                              'admin_password': rdata.get('service_password'),
+                             'admin_role': admin_role,
                              'service_protocol': svc_protocol,
                              'auth_protocol': auth_protocol,
                              'internal_protocol': int_protocol,
                              'api_version': api_version})
 
+                if rdata.get('service_type'):
+                    ctxt['service_type'] = rdata.get('service_type')
+
                 if float(api_version) > 2:
                     ctxt.update({
                         'admin_domain_name': rdata.get('service_domain'),
@@ -547,6 +555,9 @@ class IdentityCredentialsContext(IdentityServiceContext):
                     'api_version': api_version
                 })
 
+                if rdata.get('service_type'):
+                    ctxt['service_type'] = rdata.get('service_type')
+
                 if float(api_version) > 2:
                     ctxt.update({'admin_domain_name':
                                  rdata.get('domain')})

charmhelpers/contrib/openstack/ssh_migrations.py

@@ -310,7 +310,7 @@ def ssh_known_hosts_lines(application_name, user=None):
         for hosts_line in hosts:
             if hosts_line.rstrip():
                 known_hosts_list.append(hosts_line.rstrip())
-    return(known_hosts_list)
+    return known_hosts_list
 
 
 def ssh_authorized_keys_lines(application_name, user=None):
@@ -327,7 +327,7 @@ def ssh_authorized_keys_lines(application_name, user=None):
         for authkey_line in keys:
             if authkey_line.rstrip():
                 authorized_keys_list.append(authkey_line.rstrip())
-    return(authorized_keys_list)
+    return authorized_keys_list
 
 
 def ssh_compute_remove(public_key, application_name, user=None):

charmhelpers/contrib/openstack/templates/haproxy.cfg

@@ -82,7 +82,11 @@ backend {{ service }}_{{ frontend }}
     {% endif -%}
     {% endif -%}
     {% for unit, address in frontends[frontend]['backends'].items() -%}
+    {% if https -%}
+    server {{ unit }} {{ address }}:{{ ports[1] }} check check-ssl verify none
+    {% else -%}
     server {{ unit }} {{ address }}:{{ ports[1] }} check
+    {% endif -%}
     {% endfor %}
 {% endfor -%}
 {% endfor -%}

charmhelpers/contrib/openstack/templates/openstack_https_frontend

@@ -22,6 +22,8 @@ Listen {{ ext_port }}
     ProxyPassReverse / http://localhost:{{ int }}/
     ProxyPreserveHost on
     RequestHeader set X-Forwarded-Proto "https"
+    KeepAliveTimeout 75
+    MaxKeepAliveRequests 1000
 </VirtualHost>
 {% endfor -%}
 <Proxy *>

charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf

@@ -22,6 +22,8 @@ Listen {{ ext_port }}
     ProxyPassReverse / http://localhost:{{ int }}/
     ProxyPreserveHost on
     RequestHeader set X-Forwarded-Proto "https"
+    KeepAliveTimeout 75
+    MaxKeepAliveRequests 1000
 </VirtualHost>
 {% endfor -%}
 <Proxy *>

charmhelpers/contrib/openstack/templates/section-keystone-authtoken

@@ -9,4 +9,9 @@ project_name = {{ admin_tenant_name }}
 username = {{ admin_user }}
 password = {{ admin_password }}
 signing_dir = {{ signing_dir }}
+{% if service_type -%}
+service_type = {{ service_type }}
+{% endif -%}
+service_token_roles = {{ admin_role }}
+service_token_roles_required = True
 {% endif -%}

charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka

@@ -6,6 +6,9 @@ auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/v3
 auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v3
 project_domain_name = {{ admin_domain_name }}
 user_domain_name = {{ admin_domain_name }}
+{% if service_type -%}
+service_type = {{ service_type }}
+{% endif -%}
 {% else -%}
 auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
 auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
@@ -19,4 +22,6 @@ signing_dir = {{ signing_dir }}
 {% if use_memcache == true %}
 memcached_servers = {{ memcache_url }}
 {% endif -%}
+service_token_roles = {{ admin_role }}
+service_token_roles_required = True
 {% endif -%}

charmhelpers/contrib/openstack/templates/section-service-user

@@ -0,0 +1,11 @@
+{% if auth_host -%}
+[service_user]
+send_service_user_token = true
+auth_type = password
+auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
+project_domain_name = service_domain
+user_domain_name = service_domain
+project_name = {{ admin_tenant_name }}
+username = {{ admin_user }}
+password = {{ admin_password }}
+{% endif -%}

charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf

@@ -20,6 +20,8 @@ Listen {{ public_port }}
     WSGIScriptAlias / {{ script }}
     WSGIApplicationGroup %{GLOBAL}
     WSGIPassAuthorization On
+    KeepAliveTimeout 75
+    MaxKeepAliveRequests 1000
     <IfVersion >= 2.4>
       ErrorLogFormat "%{cu}t %M"
     </IfVersion>
@@ -46,6 +48,8 @@ Listen {{ public_port }}
     WSGIScriptAlias / {{ admin_script }}
     WSGIApplicationGroup %{GLOBAL}
     WSGIPassAuthorization On
+    KeepAliveTimeout 75
+    MaxKeepAliveRequests 1000
     <IfVersion >= 2.4>
       ErrorLogFormat "%{cu}t %M"
     </IfVersion>
@@ -72,6 +76,8 @@ Listen {{ public_port }}
     WSGIScriptAlias / {{ public_script }}
     WSGIApplicationGroup %{GLOBAL}
     WSGIPassAuthorization On
+    KeepAliveTimeout 75
+    MaxKeepAliveRequests 1000
     <IfVersion >= 2.4>
       ErrorLogFormat "%{cu}t %M"
     </IfVersion>

charmhelpers/contrib/openstack/templates/wsgi-openstack-metadata.conf

@@ -20,6 +20,8 @@ Listen {{ public_port }}
     WSGIScriptAlias / {{ script }}
     WSGIApplicationGroup %{GLOBAL}
     WSGIPassAuthorization On
+    KeepAliveTimeout 75
+    MaxKeepAliveRequests 1000
     <IfVersion >= 2.4>
       ErrorLogFormat "%{cu}t %M"
     </IfVersion>
@@ -46,6 +48,8 @@ Listen {{ public_port }}
     WSGIScriptAlias / {{ admin_script }}
     WSGIApplicationGroup %{GLOBAL}
     WSGIPassAuthorization On
+    KeepAliveTimeout 75
+    MaxKeepAliveRequests 1000
     <IfVersion >= 2.4>
       ErrorLogFormat "%{cu}t %M"
     </IfVersion>
@@ -72,6 +76,8 @@ Listen {{ public_port }}
     WSGIScriptAlias / {{ public_script }}
     WSGIApplicationGroup %{GLOBAL}
     WSGIPassAuthorization On
+    KeepAliveTimeout 75
+    MaxKeepAliveRequests 1000
     <IfVersion >= 2.4>
       ErrorLogFormat "%{cu}t %M"
     </IfVersion>

charmhelpers/contrib/openstack/utils.py

@@ -1039,7 +1039,7 @@ def _determine_os_workload_status(
             state, message, lambda: charm_func(configs))
 
     if state is None:
-        state, message = _ows_check_services_running(services, ports)
+        state, message = ows_check_services_running(services, ports)
 
     if state is None:
         state = 'active'
@@ -1213,7 +1213,12 @@ def _ows_check_charm_func(state, message, charm_func_with_configs):
     return state, message
 
 
+@deprecate("use ows_check_services_running() instead", "2022-05", log=juju_log)
 def _ows_check_services_running(services, ports):
+    return ows_check_services_running(services, ports)
+
+
+def ows_check_services_running(services, ports):
     """Check that the services that should be running are actually running
     and that any ports specified are being listened to.
 
@@ -1320,7 +1325,7 @@ def _check_listening_on_services_ports(services, test=False):
     @param test: default=False, if False, test for closed, otherwise open.
     @returns OrderedDict(service: [port-not-open, ...]...), [boolean]
     """
-    test = not(not(test))  # ensure test is True or False
+    test = not (not (test))  # ensure test is True or False
     all_ports = list(itertools.chain(*services.values()))
     ports_states = [port_has_listener('0.0.0.0', p) for p in all_ports]
     map_ports = OrderedDict()
@@ -1544,7 +1549,7 @@ def is_unit_paused_set():
         with unitdata.HookData()() as t:
             kv = t[0]
             # transform something truth-y into a Boolean.
-            return not(not(kv.get('unit-paused')))
+            return not (not (kv.get('unit-paused')))
     except Exception:
         return False
 
@@ -2143,7 +2148,7 @@ def is_unit_upgrading_set():
         with unitdata.HookData()() as t:
             kv = t[0]
             # transform something truth-y into a Boolean.
-            return not(not(kv.get('unit-upgrading')))
+            return not (not (kv.get('unit-upgrading')))
     except Exception:
         return False
 
@@ -2599,6 +2604,23 @@ def get_subordinate_release_packages(os_release, package_type='deb'):
     return SubordinatePackages(install, purge)
 
 
+def get_subordinate_services():
+    """Iterate over subordinate relations and get service information.
+
+    In a similar fashion as with get_subordinate_release_packages(),
+    principle charms can retrieve a list of services advertised by their
+    subordinate charms. This is useful to know about subordinate services when
+    pausing, resuming or upgrading a principle unit.
+
+    :returns: Name of all services advertised by all subordinates
+    :rtype: Set[str]
+    """
+    services = set()
+    for rdata in container_scoped_relation_get('services'):
+        services |= set(json.loads(rdata or '[]'))
+    return services
+
+
 os_restart_on_change = partial(
     pausable_restart_on_change,
     can_restart_now_f=deferred_events.check_and_record_restart_request,

charmhelpers/contrib/storage/linux/ceph.py

@@ -813,8 +813,10 @@ def get_mon_map(service):
              ceph command fails.
     """
     try:
+        octopus_or_later = cmp_pkgrevno('ceph-common', '15.0.0') >= 0
+        mon_status_cmd = 'quorum_status' if octopus_or_later else 'mon_status'
         mon_status = check_output(['ceph', '--id', service,
-                                   'mon_status', '--format=json'])
+                                   mon_status_cmd, '--format=json'])
         if six.PY3:
             mon_status = mon_status.decode('UTF-8')
         try:

charmhelpers/core/host.py

@@ -926,7 +926,7 @@ def pwgen(length=None):
     random_generator = random.SystemRandom()
     random_chars = [
         random_generator.choice(alphanumeric_chars) for _ in range(length)]
-    return(''.join(random_chars))
+    return ''.join(random_chars)
 
 
 def is_phy_iface(interface):

charmhelpers/fetch/snap.py

@@ -52,7 +52,7 @@ def _snap_exec(commands):
     :param commands: List commands
     :return: Integer exit code
     """
-    assert type(commands) == list
+    assert isinstance(commands, list)
 
     retry_count = 0
     return_code = None

charmhelpers/fetch/ubuntu.py

@@ -224,6 +224,10 @@ CLOUD_ARCHIVE_POCKETS = {
     'yoga/proposed': 'focal-proposed/yoga',
     'focal-yoga/proposed': 'focal-proposed/yoga',
     'focal-proposed/yoga': 'focal-proposed/yoga',
+
+    # OVN
+    'focal-ovn-22.03': 'focal-updates/ovn-22.03',
+    'focal-ovn-22.03/proposed': 'focal-proposed/ovn-22.03',
 }
 
 
@@ -683,6 +687,7 @@ def add_source(source, key=None, fail_invalid=False):
         (r"^cloud-archive:(.*)$", _add_apt_repository),
         (r"^((?:deb |http:|https:|ppa:).*)$", _add_apt_repository),
         (r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging),
+        (r"^cloud:(.*)-(ovn-.*)$", _add_cloud_distro_check),
         (r"^cloud:(.*)-(.*)$", _add_cloud_distro_check),
         (r"^cloud:(.*)$", _add_cloud_pocket),
         (r"^snap:.*-(.*)-(.*)$", _add_cloud_distro_check),
@@ -746,6 +751,11 @@ def _add_apt_repository(spec):
                       )
 
 
+def __write_sources_list_d_actual_pocket(file, actual_pocket):
+    with open('/etc/apt/sources.list.d/{}'.format(file), 'w') as apt:
+        apt.write(CLOUD_ARCHIVE.format(actual_pocket))
+
+
 def _add_cloud_pocket(pocket):
     """Add a cloud pocket as /etc/apt/sources.d/cloud-archive.list
 
@@ -765,8 +775,9 @@ def _add_cloud_pocket(pocket):
             'Unsupported cloud: source option %s' %
             pocket)
     actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
-    with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
+    __write_sources_list_d_actual_pocket(
-        apt.write(CLOUD_ARCHIVE.format(actual_pocket))
+        'cloud-archive{}.list'.format('' if 'ovn' not in pocket else '-ovn'),
+        actual_pocket)
 
 
 def _add_cloud_staging(cloud_archive_release, openstack_release):

test-requirements.txt

@@ -10,6 +10,7 @@
 pyparsing<3.0.0  # cffi needs pyparsing < 3.0.0.
 cffi==1.14.6; python_version < '3.6'  # cffi 1.15.0 drops support for py35.
 setuptools<50.0.0  # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85
+testtools<2.6.0
 
 requests>=2.18.4
 

tox.ini

@@ -23,6 +23,7 @@ skip_missing_interpreters = False
 #   lead to fetching the latest pip in the func* tox targets, see
 #   https://stackoverflow.com/a/38133283
 requires =
+  tox < 4.0.0
   pip < 20.3
   virtualenv < 20.0
   setuptools < 50.0.0
@@ -89,7 +90,8 @@ commands = stestr run --slowest {posargs}
 [testenv:pep8]
 basepython = python3
 deps = flake8==3.9.2
-       charm-tools==2.8.3
+       PyYAML==6.0.1
+       charm-tools==2.8.6
 commands = flake8 {posargs} hooks unit_tests tests actions lib files
            charm-proof
 

unit_tests/test_actions.py

@@ -294,7 +294,7 @@ class DiskUsageTestCase(CharmTestCase):
         self.check_output.assert_called_once_with(['swift-recon', '-d'])
 
         self.action_set.assert_called()
-        self.action_fail.not_called()
+        self.action_fail.assert_not_called()
 
     def test_check_output_failure(self):
         """Ensure that action_fail and action_set are called on