From 4114828b930102c407554307a924047141df1a01 Mon Sep 17 00:00:00 2001 From: Marius Oprin Date: Tue, 24 Nov 2020 18:05:15 +0200 Subject: [PATCH] Charmhelpers sync This is required to get the updated broker request handler containing the new rbd-mirroring-mode flag. Change-Id: Ia4c1e9741ae8eab2e41d7ba4e4615475914c43e5 --- charmhelpers/contrib/network/ip.py | 3 +- charmhelpers/contrib/openstack/cert_utils.py | 124 +++++++++++++----- charmhelpers/contrib/openstack/ip.py | 16 +++ .../openstack/templates/section-placement | 1 + charmhelpers/contrib/openstack/utils.py | 23 +++- charmhelpers/contrib/storage/linux/ceph.py | 6 + charmhelpers/core/decorators.py | 38 ++++++ charmhelpers/core/host.py | 24 ++-- 8 files changed, 193 insertions(+), 42 deletions(-) diff --git a/charmhelpers/contrib/network/ip.py b/charmhelpers/contrib/network/ip.py index b13277b..63e91cc 100644 --- a/charmhelpers/contrib/network/ip.py +++ b/charmhelpers/contrib/network/ip.py @@ -396,7 +396,8 @@ def get_ipv6_addr(iface=None, inc_aliases=False, fatal=True, exc_list=None, if global_addrs: # Make sure any found global addresses are not temporary cmd = ['ip', 'addr', 'show', iface] - out = subprocess.check_output(cmd).decode('UTF-8') + out = subprocess.check_output( + cmd).decode('UTF-8', errors='replace') if dynamic_only: key = re.compile("inet6 (.+)/[0-9]+ scope global.* dynamic.*") else: diff --git a/charmhelpers/contrib/openstack/cert_utils.py b/charmhelpers/contrib/openstack/cert_utils.py index b494af6..1eb2154 100644 --- a/charmhelpers/contrib/openstack/cert_utils.py +++ b/charmhelpers/contrib/openstack/cert_utils.py @@ -34,12 +34,14 @@ from charmhelpers.core.hookenv import ( WARNING, ) from charmhelpers.contrib.openstack.ip import ( - ADMIN, resolve_address, get_vip_in_network, - INTERNAL, - PUBLIC, - ADDRESS_MAP) + ADDRESS_MAP, + get_default_api_bindings, +) +from charmhelpers.contrib.network.ip import ( + get_relation_ip, +) from charmhelpers.core.host import ( mkdir, @@ -113,44 +115,118 @@ class CertRequest(object): return req -def get_certificate_request(json_encode=True): - """Generate a certificatee requests based on the network confioguration +def get_certificate_request(json_encode=True, bindings=None): + """Generate a certificate requests based on the network configuration + :param json_encode: Encode request in JSON or not. Used for setting + directly on a relation. + :type json_encode: boolean + :param bindings: List of bindings to check in addition to default api + bindings. + :type bindings: list of strings + :returns: CertRequest request as dictionary or JSON string. + :rtype: Union[dict, json] """ + if bindings: + # Add default API bindings to bindings list + bindings = set(bindings + get_default_api_bindings()) + else: + # Use default API bindings + bindings = get_default_api_bindings() req = CertRequest(json_encode=json_encode) req.add_hostname_cn() # Add os-hostname entries - for net_type in [INTERNAL, ADMIN, PUBLIC]: - net_config = config(ADDRESS_MAP[net_type]['override']) + _sans = get_certificate_sans() + + # Handle specific hostnames per binding + for binding in bindings: + hostname_override = config(ADDRESS_MAP[binding]['override']) try: - net_addr = resolve_address(endpoint_type=net_type) + net_addr = resolve_address(endpoint_type=binding) ip = network_get_primary_address( - ADDRESS_MAP[net_type]['binding']) + ADDRESS_MAP[binding]['binding']) addresses = [net_addr, ip] vip = get_vip_in_network(resolve_network_cidr(ip)) if vip: addresses.append(vip) - if net_config: + # Add hostname certificate request + if hostname_override: req.add_entry( - net_type, - net_config, + binding, + hostname_override, addresses) - else: - # There is network address with no corresponding hostname. - # Add the ip to the hostname cert to allow for this. - req.add_hostname_cn_ip(addresses) + # Remove hostname specific addresses from _sans + for addr in addresses: + try: + _sans.remove(addr) + except (ValueError, KeyError): + pass + except NoNetworkBinding: log("Skipping request for certificate for ip in {} space, no " - "local address found".format(net_type), WARNING) + "local address found".format(binding), WARNING) + # Gurantee all SANs are covered + # These are network addresses with no corresponding hostname. + # Add the ips to the hostname cert to allow for this. + req.add_hostname_cn_ip(_sans) return req.get_request() +def get_certificate_sans(bindings=None): + """Get all possible IP addresses for certificate SANs. + """ + _sans = [unit_get('private-address')] + if bindings: + # Add default API bindings to bindings list + bindings = set(bindings + get_default_api_bindings()) + else: + # Use default API bindings + bindings = get_default_api_bindings() + + for binding in bindings: + # Check for config override + try: + net_config = config(ADDRESS_MAP[binding]['config']) + except KeyError: + # There is no configuration network for this binding name + net_config = None + # Using resolve_address is likely redundant. Keeping it here in + # case there is an edge case it handles. + net_addr = resolve_address(endpoint_type=binding) + ip = get_relation_ip(binding, cidr_network=net_config) + _sans = _sans + [net_addr, ip] + vip = get_vip_in_network(resolve_network_cidr(ip)) + if vip: + _sans.append(vip) + return set(_sans) + + def create_ip_cert_links(ssl_dir, custom_hostname_link=None): """Create symlinks for SAN records :param ssl_dir: str Directory to create symlinks in :param custom_hostname_link: str Additional link to be created """ + + # This includes the hostname cert and any specific bindng certs: + # admin, internal, public + req = get_certificate_request(json_encode=False)["cert_requests"] + # Specific certs + for cert_req in req.keys(): + requested_cert = os.path.join( + ssl_dir, + 'cert_{}'.format(cert_req)) + requested_key = os.path.join( + ssl_dir, + 'key_{}'.format(cert_req)) + for addr in req[cert_req]['sans']: + cert = os.path.join(ssl_dir, 'cert_{}'.format(addr)) + key = os.path.join(ssl_dir, 'key_{}'.format(addr)) + if os.path.isfile(requested_cert) and not os.path.isfile(cert): + os.symlink(requested_cert, cert) + os.symlink(requested_key, key) + + # Handle custom hostnames hostname = get_hostname(unit_get('private-address')) hostname_cert = os.path.join( ssl_dir, @@ -158,18 +234,6 @@ def create_ip_cert_links(ssl_dir, custom_hostname_link=None): hostname_key = os.path.join( ssl_dir, 'key_{}'.format(hostname)) - # Add links to hostname cert, used if os-hostname vars not set - for net_type in [INTERNAL, ADMIN, PUBLIC]: - try: - addr = resolve_address(endpoint_type=net_type) - cert = os.path.join(ssl_dir, 'cert_{}'.format(addr)) - key = os.path.join(ssl_dir, 'key_{}'.format(addr)) - if os.path.isfile(hostname_cert) and not os.path.isfile(cert): - os.symlink(hostname_cert, cert) - os.symlink(hostname_key, key) - except NoNetworkBinding: - log("Skipping creating cert symlink for ip in {} space, no " - "local address found".format(net_type), WARNING) if custom_hostname_link: custom_cert = os.path.join( ssl_dir, diff --git a/charmhelpers/contrib/openstack/ip.py b/charmhelpers/contrib/openstack/ip.py index 723aebc..89cf276 100644 --- a/charmhelpers/contrib/openstack/ip.py +++ b/charmhelpers/contrib/openstack/ip.py @@ -33,6 +33,7 @@ INTERNAL = 'int' ADMIN = 'admin' ACCESS = 'access' +# TODO: reconcile 'int' vs 'internal' binding names ADDRESS_MAP = { PUBLIC: { 'binding': 'public', @@ -58,6 +59,14 @@ ADDRESS_MAP = { 'fallback': 'private-address', 'override': 'os-access-hostname', }, + # Note (thedac) bridge to begin the reconciliation between 'int' vs + # 'internal' binding names + 'internal': { + 'binding': 'internal', + 'config': 'os-internal-network', + 'fallback': 'private-address', + 'override': 'os-internal-hostname', + }, } @@ -195,3 +204,10 @@ def get_vip_in_network(network): if is_address_in_network(network, vip): matching_vip = vip return matching_vip + + +def get_default_api_bindings(): + _default_bindings = [] + for binding in [INTERNAL, ADMIN, PUBLIC]: + _default_bindings.append(ADDRESS_MAP[binding]['binding']) + return _default_bindings diff --git a/charmhelpers/contrib/openstack/templates/section-placement b/charmhelpers/contrib/openstack/templates/section-placement index 97724bd..8c224ec 100644 --- a/charmhelpers/contrib/openstack/templates/section-placement +++ b/charmhelpers/contrib/openstack/templates/section-placement @@ -15,5 +15,6 @@ password = {{ admin_password }} {% endif -%} {% if region -%} os_region_name = {{ region }} +region_name = {{ region }} {% endif -%} randomize_allocation_candidates = true diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index 0aa797c..f4c7621 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -18,6 +18,7 @@ from functools import wraps import subprocess import json +import operator import os import sys import re @@ -33,7 +34,7 @@ from charmhelpers import deprecate from charmhelpers.contrib.network import ip -from charmhelpers.core import unitdata +from charmhelpers.core import decorators, unitdata from charmhelpers.core.hookenv import ( WORKLOAD_STATES, @@ -230,7 +231,7 @@ SWIFT_CODENAMES = OrderedDict([ ('ussuri', ['2.24.0', '2.25.0']), ('victoria', - ['2.25.0']), + ['2.25.0', '2.26.0']), ]) # >= Liberty version->codename mapping @@ -1295,7 +1296,7 @@ def _check_listening_on_ports_list(ports): Returns a list of ports being listened to and a list of the booleans. - @param ports: LIST or port numbers. + @param ports: LIST of port numbers. @returns [(port_num, boolean), ...], [boolean] """ ports_open = [port_has_listener('0.0.0.0', p) for p in ports] @@ -1564,6 +1565,21 @@ def manage_payload_services(action, services=None, charm_func=None): return success, messages +def make_wait_for_ports_barrier(ports, retry_count=5): + """Make a function to wait for port shutdowns. + + Create a function which closes over the provided ports. The function will + retry probing ports until they are closed or the retry count has been reached. + + """ + @decorators.retry_on_predicate(retry_count, operator.not_, base_delay=0.1) + def retry_port_check(): + _, ports_states = _check_listening_on_ports_list(ports) + juju_log("Probe ports {}, result: {}".format(ports, ports_states), level="DEBUG") + return any(ports_states) + return retry_port_check + + def pause_unit(assess_status_func, services=None, ports=None, charm_func=None): """Pause a unit by stopping the services and setting 'unit-paused' @@ -1599,6 +1615,7 @@ def pause_unit(assess_status_func, services=None, ports=None, services=services, charm_func=charm_func) set_unit_paused() + if assess_status_func: message = assess_status_func() if message: diff --git a/charmhelpers/contrib/storage/linux/ceph.py b/charmhelpers/contrib/storage/linux/ceph.py index 0f69631..d1c6175 100644 --- a/charmhelpers/contrib/storage/linux/ceph.py +++ b/charmhelpers/contrib/storage/linux/ceph.py @@ -268,6 +268,7 @@ class BasePool(object): 'compression-max-blob-size': (int, None), 'compression-max-blob-size-hdd': (int, None), 'compression-max-blob-size-ssd': (int, None), + 'rbd-mirroring-mode': (str, ('image', 'pool')) } def __init__(self, service, name=None, percent_data=None, app_name=None, @@ -1767,6 +1768,7 @@ class CephBrokerRq(object): max_bytes=None, max_objects=None, namespace=None, + rbd_mirroring_mode='pool', weight=None): """Build common part of a create pool operation. @@ -1825,6 +1827,9 @@ class CephBrokerRq(object): :type max_objects: Optional[int] :param namespace: Group namespace :type namespace: Optional[str] + :param rbd_mirroring_mode: Pool mirroring mode used when Ceph RBD + mirroring is enabled. + :type rbd_mirroring_mode: Optional[str] :param weight: The percentage of data that is expected to be contained in the pool from the total available space on the OSDs. Used to calculate number of Placement Groups to create @@ -1849,6 +1854,7 @@ class CephBrokerRq(object): 'max-bytes': max_bytes, 'max-objects': max_objects, 'group-namespace': namespace, + 'rbd-mirroring-mode': rbd_mirroring_mode, 'weight': weight, } diff --git a/charmhelpers/core/decorators.py b/charmhelpers/core/decorators.py index 6ad41ee..e7e95d1 100644 --- a/charmhelpers/core/decorators.py +++ b/charmhelpers/core/decorators.py @@ -53,3 +53,41 @@ def retry_on_exception(num_retries, base_delay=0, exc_type=Exception): return _retry_on_exception_inner_2 return _retry_on_exception_inner_1 + + +def retry_on_predicate(num_retries, predicate_fun, base_delay=0): + """Retry based on return value + + The return value of the decorated function is passed to the given predicate_fun. If the + result of the predicate is False, retry the decorated function up to num_retries times + + An exponential backoff up to base_delay^num_retries seconds can be introduced by setting + base_delay to a nonzero value. The default is to run with a zero (i.e. no) delay + + :param num_retries: Max. number of retries to perform + :type num_retries: int + :param predicate_fun: Predicate function to determine if a retry is necessary + :type predicate_fun: callable + :param base_delay: Starting value in seconds for exponential delay, defaults to 0 (no delay) + :type base_delay: float + """ + def _retry_on_pred_inner_1(f): + def _retry_on_pred_inner_2(*args, **kwargs): + retries = num_retries + multiplier = 1 + delay = base_delay + while True: + result = f(*args, **kwargs) + if predicate_fun(result) or retries <= 0: + return result + delay *= multiplier + multiplier += 1 + log("Result {}, retrying '{}' {} more times (delay={})".format( + result, f.__name__, retries, delay), level=INFO) + retries -= 1 + if delay: + time.sleep(delay) + + return _retry_on_pred_inner_2 + + return _retry_on_pred_inner_1 diff --git a/charmhelpers/core/host.py b/charmhelpers/core/host.py index a785efd..697a5f4 100644 --- a/charmhelpers/core/host.py +++ b/charmhelpers/core/host.py @@ -19,6 +19,7 @@ # Nick Moffitt # Matthew Wedgwood +import errno import os import re import pwd @@ -677,7 +678,7 @@ def check_hash(path, checksum, hash_type='md5'): :param str checksum: Value of the checksum used to validate the file. :param str hash_type: Hash algorithm used to generate `checksum`. - Can be any hash alrgorithm supported by :mod:`hashlib`, + Can be any hash algorithm supported by :mod:`hashlib`, such as md5, sha1, sha256, sha512, etc. :raises ChecksumError: If the file fails the checksum @@ -825,7 +826,8 @@ def list_nics(nic_type=None): if nic_type: for int_type in int_types: cmd = ['ip', 'addr', 'show', 'label', int_type + '*'] - ip_output = subprocess.check_output(cmd).decode('UTF-8') + ip_output = subprocess.check_output( + cmd).decode('UTF-8', errors='replace') ip_output = ip_output.split('\n') ip_output = (line for line in ip_output if line) for line in ip_output: @@ -841,7 +843,8 @@ def list_nics(nic_type=None): interfaces.append(iface) else: cmd = ['ip', 'a'] - ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') + ip_output = subprocess.check_output( + cmd).decode('UTF-8', errors='replace').split('\n') ip_output = (line.strip() for line in ip_output if line) key = re.compile(r'^[0-9]+:\s+(.+):') @@ -865,7 +868,8 @@ def set_nic_mtu(nic, mtu): def get_nic_mtu(nic): """Return the Maximum Transmission Unit (MTU) for a network interface.""" cmd = ['ip', 'addr', 'show', nic] - ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') + ip_output = subprocess.check_output( + cmd).decode('UTF-8', errors='replace').split('\n') mtu = "" for line in ip_output: words = line.split() @@ -877,7 +881,7 @@ def get_nic_mtu(nic): def get_nic_hwaddr(nic): """Return the Media Access Control (MAC) for a network interface.""" cmd = ['ip', '-o', '-0', 'addr', 'show', nic] - ip_output = subprocess.check_output(cmd).decode('UTF-8') + ip_output = subprocess.check_output(cmd).decode('UTF-8', errors='replace') hwaddr = "" words = ip_output.split() if 'link/ether' in words: @@ -889,7 +893,7 @@ def get_nic_hwaddr(nic): def chdir(directory): """Change the current working directory to a different directory for a code block and return the previous directory after the block exits. Useful to - run commands from a specificed directory. + run commands from a specified directory. :param str directory: The directory path to change to for this context. """ @@ -924,9 +928,13 @@ def chownr(path, owner, group, follow_links=True, chowntopdir=False): for root, dirs, files in os.walk(path, followlinks=follow_links): for name in dirs + files: full = os.path.join(root, name) - broken_symlink = os.path.lexists(full) and not os.path.exists(full) - if not broken_symlink: + try: chown(full, uid, gid) + except (IOError, OSError) as e: + # Intended to ignore "file not found". Catching both to be + # compatible with both Python 2.7 and 3.x. + if e.errno == errno.ENOENT: + pass def lchownr(path, owner, group):