From 45b444e073932ccdb57a18862f9b0f1a5c279018 Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Wed, 7 Nov 2018 15:34:18 -0600 Subject: [PATCH] Sync charm-helpers Change-Id: I3f35600d1a6b61ce75726e2652239b2657464257 --- charmhelpers/__init__.py | 8 ++-- charmhelpers/contrib/openstack/cert_utils.py | 48 +++++++++++++++++++ charmhelpers/contrib/openstack/context.py | 2 +- charmhelpers/contrib/openstack/utils.py | 27 ++++++++--- .../contrib/storage/linux/loopback.py | 2 +- charmhelpers/core/host.py | 17 ++++--- charmhelpers/core/kernel.py | 4 +- charmhelpers/fetch/ubuntu.py | 2 +- 8 files changed, 87 insertions(+), 23 deletions(-) diff --git a/charmhelpers/__init__.py b/charmhelpers/__init__.py index e7aa471..61ef907 100644 --- a/charmhelpers/__init__.py +++ b/charmhelpers/__init__.py @@ -23,22 +23,22 @@ import subprocess import sys try: - import six # flake8: noqa + import six # NOQA:F401 except ImportError: if sys.version_info.major == 2: subprocess.check_call(['apt-get', 'install', '-y', 'python-six']) else: subprocess.check_call(['apt-get', 'install', '-y', 'python3-six']) - import six # flake8: noqa + import six # NOQA:F401 try: - import yaml # flake8: noqa + import yaml # NOQA:F401 except ImportError: if sys.version_info.major == 2: subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml']) else: subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml']) - import yaml # flake8: noqa + import yaml # NOQA:F401 # Holds a list of mapping of mangled function names that have been deprecated diff --git a/charmhelpers/contrib/openstack/cert_utils.py b/charmhelpers/contrib/openstack/cert_utils.py index de853b5..3e07870 100644 --- a/charmhelpers/contrib/openstack/cert_utils.py +++ b/charmhelpers/contrib/openstack/cert_utils.py @@ -25,7 +25,9 @@ from charmhelpers.core.hookenv import ( local_unit, network_get_primary_address, config, + related_units, relation_get, + relation_ids, unit_get, NoNetworkBinding, log, @@ -225,3 +227,49 @@ def process_certificates(service_name, relation_id, unit, create_ip_cert_links( ssl_dir, custom_hostname_link=custom_hostname_link) + + +def get_requests_for_local_unit(relation_name=None): + """Extract any certificates data targeted at this unit down relation_name. + + :param relation_name: str Name of relation to check for data. + :returns: List of bundles of certificates. + :rtype: List of dicts + """ + local_name = local_unit().replace('/', '_') + raw_certs_key = '{}.processed_requests'.format(local_name) + relation_name = relation_name or 'certificates' + bundles = [] + for rid in relation_ids(relation_name): + for unit in related_units(rid): + data = relation_get(rid=rid, unit=unit) + if data.get(raw_certs_key): + bundles.append({ + 'ca': data['ca'], + 'chain': data.get('chain'), + 'certs': json.loads(data[raw_certs_key])}) + return bundles + + +def get_bundle_for_cn(cn, relation_name=None): + """Extract certificates for the given cn. + + :param cn: str Canonical Name on certificate. + :param relation_name: str Relation to check for certificates down. + :returns: Dictionary of certificate data, + :rtype: dict. + """ + entries = get_requests_for_local_unit(relation_name) + cert_bundle = {} + for entry in entries: + for _cn, bundle in entry['certs'].items(): + if _cn == cn: + cert_bundle = { + 'cert': bundle['cert'], + 'key': bundle['key'], + 'chain': entry['chain'], + 'ca': entry['ca']} + break + if cert_bundle: + break + return cert_bundle diff --git a/charmhelpers/contrib/openstack/context.py b/charmhelpers/contrib/openstack/context.py index 6feb3f9..72084cb 100644 --- a/charmhelpers/contrib/openstack/context.py +++ b/charmhelpers/contrib/openstack/context.py @@ -642,7 +642,7 @@ class HAProxyContext(OSContextGenerator): return {} l_unit = local_unit().replace('/', '-') - cluster_hosts = {} + cluster_hosts = collections.OrderedDict() # NOTE(jamespage): build out map of configured network endpoints # and associated backends diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index ae48d6b..29cad08 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -375,7 +375,7 @@ def get_swift_codename(version): return codenames[0] # NOTE: fallback - attempt to match with just major.minor version - match = re.match('^(\d+)\.(\d+)', version) + match = re.match(r'^(\d+)\.(\d+)', version) if match: major_minor_version = match.group(0) for codename, versions in six.iteritems(SWIFT_CODENAMES): @@ -395,7 +395,7 @@ def get_os_codename_package(package, fatal=True): out = subprocess.check_output(cmd) if six.PY3: out = out.decode('UTF-8') - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: return None lines = out.split('\n') for line in lines: @@ -427,11 +427,11 @@ def get_os_codename_package(package, fatal=True): vers = apt.upstream_version(pkg.current_ver.ver_str) if 'swift' in pkg.name: # Fully x.y.z match for swift versions - match = re.match('^(\d+)\.(\d+)\.(\d+)', vers) + match = re.match(r'^(\d+)\.(\d+)\.(\d+)', vers) else: # x.y match only for 20XX.X # and ignore patch level for other packages - match = re.match('^(\d+)\.(\d+)', vers) + match = re.match(r'^(\d+)\.(\d+)', vers) if match: vers = match.group(0) @@ -1450,20 +1450,33 @@ def pausable_restart_on_change(restart_map, stopstart=False, see core.utils.restart_on_change() for more details. + Note restart_map can be a callable, in which case, restart_map is only + evaluated at runtime. This means that it is lazy and the underlying + function won't be called if the decorated function is never called. Note, + retains backwards compatibility for passing a non-callable dictionary. + @param f: the function to decorate - @param restart_map: the restart map {conf_file: [services]} + @param restart_map: (optionally callable, which then returns the + restart_map) the restart map {conf_file: [services]} @param stopstart: DEFAULT false; whether to stop, start or just restart @returns decorator to use a restart_on_change with pausability """ def wrap(f): + # py27 compatible nonlocal variable. When py3 only, replace with + # nonlocal keyword + __restart_map_cache = {'cache': None} + @functools.wraps(f) def wrapped_f(*args, **kwargs): if is_unit_paused_set(): return f(*args, **kwargs) + if __restart_map_cache['cache'] is None: + __restart_map_cache['cache'] = restart_map() \ + if callable(restart_map) else restart_map # otherwise, normal restart_on_change functionality return restart_on_change_helper( - (lambda: f(*args, **kwargs)), restart_map, stopstart, - restart_functions) + (lambda: f(*args, **kwargs)), __restart_map_cache['cache'], + stopstart, restart_functions) return wrapped_f return wrap diff --git a/charmhelpers/contrib/storage/linux/loopback.py b/charmhelpers/contrib/storage/linux/loopback.py index 1d6ae6f..0dfdae5 100644 --- a/charmhelpers/contrib/storage/linux/loopback.py +++ b/charmhelpers/contrib/storage/linux/loopback.py @@ -39,7 +39,7 @@ def loopback_devices(): devs = [d.strip().split(' ') for d in check_output(cmd).splitlines() if d != ''] for dev, _, f in devs: - loopbacks[dev.replace(':', '')] = re.search('\((\S+)\)', f).groups()[0] + loopbacks[dev.replace(':', '')] = re.search(r'\((\S+)\)', f).groups()[0] return loopbacks diff --git a/charmhelpers/core/host.py b/charmhelpers/core/host.py index 51b6702..79953a4 100644 --- a/charmhelpers/core/host.py +++ b/charmhelpers/core/host.py @@ -40,7 +40,7 @@ from charmhelpers.osplatform import get_platform __platform__ = get_platform() if __platform__ == "ubuntu": - from charmhelpers.core.host_factory.ubuntu import ( + from charmhelpers.core.host_factory.ubuntu import ( # NOQA:F401 service_available, add_new_group, lsb_release, @@ -48,7 +48,7 @@ if __platform__ == "ubuntu": CompareHostReleases, ) # flake8: noqa -- ignore F401 for this import elif __platform__ == "centos": - from charmhelpers.core.host_factory.centos import ( + from charmhelpers.core.host_factory.centos import ( # NOQA:F401 service_available, add_new_group, lsb_release, @@ -58,6 +58,7 @@ elif __platform__ == "centos": UPDATEDB_PATH = '/etc/updatedb.conf' + def service_start(service_name, **kwargs): """Start a system service. @@ -287,8 +288,8 @@ def service_running(service_name, **kwargs): for key, value in six.iteritems(kwargs): parameter = '%s=%s' % (key, value) cmd.append(parameter) - output = subprocess.check_output(cmd, - stderr=subprocess.STDOUT).decode('UTF-8') + output = subprocess.check_output( + cmd, stderr=subprocess.STDOUT).decode('UTF-8') except subprocess.CalledProcessError: return False else: @@ -442,7 +443,7 @@ def add_user_to_group(username, group): def chage(username, lastday=None, expiredate=None, inactive=None, - mindays=None, maxdays=None, root=None, warndays=None): + mindays=None, maxdays=None, root=None, warndays=None): """Change user password expiry information :param str username: User to update @@ -482,8 +483,10 @@ def chage(username, lastday=None, expiredate=None, inactive=None, cmd.append(username) subprocess.check_call(cmd) + remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1') + def rsync(from_path, to_path, flags='-r', options=None, timeout=None): """Replicate the contents of a path""" options = options or ['--delete', '--executability'] @@ -543,7 +546,7 @@ def write_file(path, content, owner='root', group='root', perms=0o444): existing_uid, existing_gid, existing_perms = ( stat.st_uid, stat.st_gid, stat.st_mode ) - except: + except Exception: pass if content != existing_content: log("Writing file {} {}:{} {:o}".format(path, owner, group, perms), @@ -833,7 +836,7 @@ def list_nics(nic_type=None): ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') ip_output = (line.strip() for line in ip_output if line) - key = re.compile('^[0-9]+:\s+(.+):') + key = re.compile(r'^[0-9]+:\s+(.+):') for line in ip_output: matched = re.search(key, line) if matched: diff --git a/charmhelpers/core/kernel.py b/charmhelpers/core/kernel.py index 2d40452..e01f4f8 100644 --- a/charmhelpers/core/kernel.py +++ b/charmhelpers/core/kernel.py @@ -26,12 +26,12 @@ from charmhelpers.core.hookenv import ( __platform__ = get_platform() if __platform__ == "ubuntu": - from charmhelpers.core.kernel_factory.ubuntu import ( + from charmhelpers.core.kernel_factory.ubuntu import ( # NOQA:F401 persistent_modprobe, update_initramfs, ) # flake8: noqa -- ignore F401 for this import elif __platform__ == "centos": - from charmhelpers.core.kernel_factory.centos import ( + from charmhelpers.core.kernel_factory.centos import ( # NOQA:F401 persistent_modprobe, update_initramfs, ) # flake8: noqa -- ignore F401 for this import diff --git a/charmhelpers/fetch/ubuntu.py b/charmhelpers/fetch/ubuntu.py index 8c56eb1..c7ad128 100644 --- a/charmhelpers/fetch/ubuntu.py +++ b/charmhelpers/fetch/ubuntu.py @@ -294,7 +294,7 @@ def apt_unhold(packages, fatal=False): def import_key(key): """Import an ASCII Armor key. - /!\ A Radix64 format keyid is also supported for backwards + A Radix64 format keyid is also supported for backwards compatibility, but should never be used; the key retrieval mechanism is insecure and subject to man-in-the-middle attacks voiding all signature checks using that key.