Merge "Updates for testing period for 20.01 release"

charmhelpers/contrib/hahelpers/apache.py

@@ -34,6 +34,10 @@ from charmhelpers.core.hookenv import (
     INFO,
 )
 
+# This file contains the CA cert from the charms ssl_ca configuration
+# option, in future the file name should be updated reflect that.
+CONFIG_CA_CERT_FILE = 'keystone_juju_ca_cert'
+
 
 def get_cert(cn=None):
     # TODO: deal with multiple https endpoints via charm config
@@ -83,4 +87,4 @@ def retrieve_ca_cert(cert_file):
 
 
 def install_ca_cert(ca_cert):
-    host.install_ca_cert(ca_cert, 'keystone_juju_ca_cert')
+    host.install_ca_cert(ca_cert, CONFIG_CA_CERT_FILE)

charmhelpers/contrib/openstack/cert_utils.py

@@ -16,6 +16,7 @@
 
 import os
 import json
+from base64 import b64decode
 
 from charmhelpers.contrib.network.ip import (
     get_hostname,
@@ -28,10 +29,12 @@ from charmhelpers.core.hookenv import (
     related_units,
     relation_get,
     relation_ids,
+    remote_service_name,
     unit_get,
     NoNetworkBinding,
     log,
     WARNING,
+    INFO,
 )
 from charmhelpers.contrib.openstack.ip import (
     resolve_address,
@@ -44,12 +47,14 @@ from charmhelpers.contrib.network.ip import (
 )
 
 from charmhelpers.core.host import (
+    CA_CERT_DIR,
+    install_ca_cert,
     mkdir,
     write_file,
 )
 
 from charmhelpers.contrib.hahelpers.apache import (
-    install_ca_cert
+    CONFIG_CA_CERT_FILE,
 )
 
 
@@ -129,38 +134,46 @@ def get_certificate_request(json_encode=True, bindings=None):
     """
     if bindings:
         # Add default API bindings to bindings list
-        bindings = set(bindings + get_default_api_bindings())
+        bindings = list(bindings + get_default_api_bindings())
     else:
         # Use default API bindings
         bindings = get_default_api_bindings()
     req = CertRequest(json_encode=json_encode)
     req.add_hostname_cn()
     # Add os-hostname entries
-    _sans = get_certificate_sans()
+    _sans = get_certificate_sans(bindings=bindings)
 
     # Handle specific hostnames per binding
     for binding in bindings:
-        hostname_override = config(ADDRESS_MAP[binding]['override'])
         try:
-            net_addr = resolve_address(endpoint_type=binding)
+            hostname_override = config(ADDRESS_MAP[binding]['override'])
-            ip = network_get_primary_address(
+        except KeyError:
-                ADDRESS_MAP[binding]['binding'])
+            hostname_override = None
+        try:
+            try:
+                net_addr = resolve_address(endpoint_type=binding)
+            except KeyError:
+                net_addr = None
+            ip = network_get_primary_address(binding)
             addresses = [net_addr, ip]
             vip = get_vip_in_network(resolve_network_cidr(ip))
             if vip:
                 addresses.append(vip)
+
+            # Clear any Nones or duplicates
+            addresses = list(set([i for i in addresses if i]))
             # Add hostname certificate request
             if hostname_override:
                 req.add_entry(
                     binding,
                     hostname_override,
                     addresses)
-            # Remove hostname specific addresses from _sans
+                # Remove hostname specific addresses from _sans
-            for addr in addresses:
+                for addr in addresses:
-                try:
+                    try:
-                    _sans.remove(addr)
+                        _sans.remove(addr)
-                except (ValueError, KeyError):
+                    except (ValueError, KeyError):
-                    pass
+                        pass
 
         except NoNetworkBinding:
             log("Skipping request for certificate for ip in {} space, no "
@@ -174,11 +187,17 @@ def get_certificate_request(json_encode=True, bindings=None):
 
 def get_certificate_sans(bindings=None):
     """Get all possible IP addresses for certificate SANs.
+
+    :param bindings: List of bindings to check in addition to default api
+                     bindings.
+    :type bindings: list of strings
+    :returns: List of binding string names
+    :rtype: List[str]
     """
     _sans = [unit_get('private-address')]
     if bindings:
         # Add default API bindings to bindings list
-        bindings = set(bindings + get_default_api_bindings())
+        bindings = list(bindings + get_default_api_bindings())
     else:
         # Use default API bindings
         bindings = get_default_api_bindings()
@@ -192,25 +211,39 @@ def get_certificate_sans(bindings=None):
             net_config = None
         # Using resolve_address is likely redundant. Keeping it here in
         # case there is an edge case it handles.
-        net_addr = resolve_address(endpoint_type=binding)
+        try:
+            net_addr = resolve_address(endpoint_type=binding)
+        except KeyError:
+            net_addr = None
         ip = get_relation_ip(binding, cidr_network=net_config)
         _sans = _sans + [net_addr, ip]
         vip = get_vip_in_network(resolve_network_cidr(ip))
         if vip:
             _sans.append(vip)
-    return set(_sans)
+    # Clear any Nones and duplicates
+    return list(set([i for i in _sans if i]))
 
 
-def create_ip_cert_links(ssl_dir, custom_hostname_link=None):
+def create_ip_cert_links(ssl_dir, custom_hostname_link=None, bindings=None):
     """Create symlinks for SAN records
 
     :param ssl_dir: str Directory to create symlinks in
     :param custom_hostname_link: str Additional link to be created
+    :param bindings: List of bindings to check in addition to default api
+                     bindings.
+    :type bindings: list of strings
     """
 
+    if bindings:
+        # Add default API bindings to bindings list
+        bindings = list(bindings + get_default_api_bindings())
+    else:
+        # Use default API bindings
+        bindings = get_default_api_bindings()
+
     # This includes the hostname cert and any specific bindng certs:
     # admin, internal, public
-    req = get_certificate_request(json_encode=False)["cert_requests"]
+    req = get_certificate_request(json_encode=False, bindings=bindings)["cert_requests"]
     # Specific certs
     for cert_req in req.keys():
         requested_cert = os.path.join(
@@ -274,8 +307,35 @@ def install_certs(ssl_dir, certs, chain=None, user='root', group='root'):
             content=bundle['key'], perms=0o640)
 
 
+def _manage_ca_certs(ca, cert_relation_id):
+    """Manage CA certs.
+
+    :param ca: CA Certificate from certificate relation.
+    :type ca: str
+    :param cert_relation_id: Relation id providing the certs
+    :type cert_relation_id: str
+    """
+    config_ssl_ca = config('ssl_ca')
+    config_cert_file = '{}/{}.crt'.format(CA_CERT_DIR, CONFIG_CA_CERT_FILE)
+    if config_ssl_ca:
+        log("Installing CA certificate from charm ssl_ca config to {}".format(
+            config_cert_file), INFO)
+        install_ca_cert(
+            b64decode(config_ssl_ca).rstrip(),
+            name=CONFIG_CA_CERT_FILE)
+    elif os.path.exists(config_cert_file):
+        log("Removing CA certificate {}".format(config_cert_file), INFO)
+        os.remove(config_cert_file)
+    log("Installing CA certificate from certificate relation", INFO)
+    install_ca_cert(
+        ca.encode(),
+        name='{}_juju_ca_cert'.format(
+            remote_service_name(relid=cert_relation_id)))
+
+
 def process_certificates(service_name, relation_id, unit,
-                         custom_hostname_link=None, user='root', group='root'):
+                         custom_hostname_link=None, user='root', group='root',
+                         bindings=None):
     """Process the certificates supplied down the relation
 
     :param service_name: str Name of service the certifcates are for.
@@ -286,9 +346,19 @@ def process_certificates(service_name, relation_id, unit,
     :type user: str
     :param group: (Optional) Group of certificate files. Defaults to 'root'
     :type group: str
+    :param bindings: List of bindings to check in addition to default api
+                     bindings.
+    :type bindings: list of strings
     :returns: True if certificates processed for local unit or False
     :rtype: bool
     """
+    if bindings:
+        # Add default API bindings to bindings list
+        bindings = list(bindings + get_default_api_bindings())
+    else:
+        # Use default API bindings
+        bindings = get_default_api_bindings()
+
     data = relation_get(rid=relation_id, unit=unit)
     ssl_dir = os.path.join('/etc/apache2/ssl/', service_name)
     mkdir(path=ssl_dir)
@@ -298,11 +368,12 @@ def process_certificates(service_name, relation_id, unit,
     ca = data.get('ca')
     if certs:
         certs = json.loads(certs)
-        install_ca_cert(ca.encode())
+        _manage_ca_certs(ca, relation_id)
         install_certs(ssl_dir, certs, chain, user=user, group=group)
         create_ip_cert_links(
             ssl_dir,
-            custom_hostname_link=custom_hostname_link)
+            custom_hostname_link=custom_hostname_link,
+            bindings=bindings)
         return True
     return False
 

charmhelpers/contrib/openstack/context.py

@@ -1534,8 +1534,23 @@ class SubordinateConfigContext(OSContextGenerator):
                                         ctxt[k][section] = config_list
                             else:
                                 ctxt[k] = v
-        log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
+        if self.context_complete(ctxt):
-        return ctxt
+            log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
+            return ctxt
+        else:
+            return {}
+
+    def context_complete(self, ctxt):
+        """Overridden here to ensure the context is actually complete.
+
+        :param ctxt: The current context members
+        :type ctxt: Dict[str, ANY]
+        :returns: True if the context is complete
+        :rtype: bool
+        """
+        if not ctxt.get('sections'):
+            return False
+        return super(SubordinateConfigContext, self).context_complete(ctxt)
 
 
 class LogLevelContext(OSContextGenerator):
@@ -3050,6 +3065,9 @@ class SRIOVContext(OSContextGenerator):
         blanket = 'blanket'
         explicit = 'explicit'
 
+    PCIDeviceNumVFs = collections.namedtuple(
+        'PCIDeviceNumVFs', ['device', 'numvfs'])
+
     def _determine_numvfs(self, device, sriov_numvfs):
         """Determine number of Virtual Functions (VFs) configured for device.
 
@@ -3165,14 +3183,15 @@ class SRIOVContext(OSContextGenerator):
                                'configuration.')
 
         self._map = {
-            device.interface_name: self._determine_numvfs(device, sriov_numvfs)
+            device.pci_address: self.PCIDeviceNumVFs(
+                device, self._determine_numvfs(device, sriov_numvfs))
             for device in devices.pci_devices
             if device.sriov and
             self._determine_numvfs(device, sriov_numvfs) is not None
         }
 
     def __call__(self):
-        """Provide SR-IOV context.
+        """Provide backward compatible SR-IOV context.
 
         :returns: Map interface name: min(configured, max) virtual functions.
         Example:
@@ -3183,6 +3202,23 @@ class SRIOVContext(OSContextGenerator):
            }
         :rtype: Dict[str,int]
         """
+        return {
+            pcidnvfs.device.interface_name: pcidnvfs.numvfs
+            for _, pcidnvfs in self._map.items()
+        }
+
+    @property
+    def get_map(self):
+        """Provide map of configured SR-IOV capable PCI devices.
+
+        :returns: Map PCI-address: (PCIDevice, min(configured, max) VFs.
+        Example:
+            {
+                '0000:81:00.0': self.PCIDeviceNumVFs(<PCIDevice object>, 32),
+                '0000:81:00.1': self.PCIDeviceNumVFs(<PCIDevice object>, 32),
+            }
+        :rtype: Dict[str, self.PCIDeviceNumVFs]
+        """
         return self._map
 
 

charmhelpers/contrib/openstack/utils.py

@@ -90,13 +90,16 @@ from charmhelpers.core.host import (
     service_start,
     restart_on_change_helper,
 )
+
 from charmhelpers.fetch import (
     apt_cache,
+    apt_install,
     import_key as fetch_import_key,
     add_source as fetch_add_source,
     SourceConfigError,
     GPGKeyError,
     get_upstream_version,
+    filter_installed_packages,
     filter_missing_packages,
     ubuntu_apt_pkg as apt,
 )
@@ -480,9 +483,14 @@ def get_swift_codename(version):
     return None
 
 
+@deprecate("moved to charmhelpers.contrib.openstack.utils.get_installed_os_version()", "2021-01", log=juju_log)
 def get_os_codename_package(package, fatal=True):
     '''Derive OpenStack release codename from an installed package.'''
 
+    codename = get_installed_os_version()
+    if codename:
+        return codename
+
     if snap_install_requested():
         cmd = ['snap', 'list', package]
         try:
@@ -570,6 +578,28 @@ def get_os_version_package(pkg, fatal=True):
     # error_out(e)
 
 
+def get_installed_os_version():
+    apt_install(filter_installed_packages(['openstack-release']), fatal=False)
+    print("OpenStack Release: {}".format(openstack_release()))
+    return openstack_release().get('OPENSTACK_CODENAME')
+
+
+@cached
+def openstack_release():
+    """Return /etc/os-release in a dict."""
+    d = {}
+    try:
+        with open('/etc/openstack-release', 'r') as lsb:
+            for l in lsb:
+                s = l.split('=')
+                if len(s) != 2:
+                    continue
+                d[s[0].strip()] = s[1].strip()
+    except FileNotFoundError:
+        pass
+    return d
+
+
 # Module local cache variable for the os_release.
 _os_rel = None
 

charmhelpers/core/host.py

@@ -60,6 +60,7 @@ elif __platform__ == "centos":
     )  # flake8: noqa -- ignore F401 for this import
 
 UPDATEDB_PATH = '/etc/updatedb.conf'
+CA_CERT_DIR = '/usr/local/share/ca-certificates'
 
 
 def service_start(service_name, **kwargs):
@@ -1082,7 +1083,7 @@ def install_ca_cert(ca_cert, name=None):
         ca_cert = ca_cert.encode('utf8')
     if not name:
         name = 'juju-{}'.format(charm_name())
-    cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name)
+    cert_file = '{}/{}.crt'.format(CA_CERT_DIR, name)
     new_hash = hashlib.md5(ca_cert).hexdigest()
     if file_hash(cert_file) == new_hash:
         return

charmhelpers/fetch/ubuntu.py

@@ -646,7 +646,7 @@ def _add_apt_repository(spec):
     # passed as environment variables (See lp:1433761). This is not the case
     # LTS and non-LTS releases below bionic.
     _run_with_retries(['add-apt-repository', '--yes', spec],
-                      cmd_env=env_proxy_settings(['https']))
+                      cmd_env=env_proxy_settings(['https', 'http']))
 
 
 def _add_cloud_pocket(pocket):

charmhelpers/fetch/ubuntu_apt_pkg.py

@@ -129,7 +129,7 @@ class Cache(object):
             else:
                 data = line.split(None, 4)
                 status = data.pop(0)
-                if status != 'ii':
+                if status not in ('ii', 'hi'):
                     continue
                 pkg = {}
                 pkg.update({k.lower(): v for k, v in zip(headings, data)})
@@ -265,3 +265,48 @@ def version_compare(a, b):
         raise RuntimeError('Unable to compare "{}" and "{}", according to '
                            'our logic they are neither greater, equal nor '
                            'less than each other.')
+
+
+class PkgVersion():
+    """Allow package versions to be compared.
+
+    For example::
+
+        >>> import charmhelpers.fetch as fetch
+        >>> (fetch.apt_pkg.PkgVersion('2:20.4.0') <
+        ...  fetch.apt_pkg.PkgVersion('2:20.5.0'))
+        True
+        >>> pkgs = [fetch.apt_pkg.PkgVersion('2:20.4.0'),
+        ...         fetch.apt_pkg.PkgVersion('2:21.4.0'),
+        ...         fetch.apt_pkg.PkgVersion('2:17.4.0')]
+        >>> pkgs.sort()
+        >>> pkgs
+        [2:17.4.0, 2:20.4.0, 2:21.4.0]
+    """
+
+    def __init__(self, version):
+        self.version = version
+
+    def __lt__(self, other):
+        return version_compare(self.version, other.version) == -1
+
+    def __le__(self, other):
+        return self.__lt__(other) or self.__eq__(other)
+
+    def __gt__(self, other):
+        return version_compare(self.version, other.version) == 1
+
+    def __ge__(self, other):
+        return self.__gt__(other) or self.__eq__(other)
+
+    def __eq__(self, other):
+        return version_compare(self.version, other.version) == 0
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __repr__(self):
+        return self.version
+
+    def __hash__(self):
+        return hash(repr(self))

test-requirements.txt

@@ -37,6 +37,8 @@ importlib-resources<3.0.0; python_version < '3.6'
 # dropped support for python 3.5:
 osprofiler<2.7.0;python_version<'3.6'
 stevedore<1.31.0;python_version<'3.6'
+debtcollector<1.22.0;python_version<'3.6'
+oslo.utils<=3.41.0;python_version<'3.6'
 
 coverage>=4.5.2
 pyudev              # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking)