diff --git a/hooks/charmhelpers/__init__.py b/hooks/charmhelpers/__init__.py index e7aa4715..61ef9071 100644 --- a/hooks/charmhelpers/__init__.py +++ b/hooks/charmhelpers/__init__.py @@ -23,22 +23,22 @@ import subprocess import sys try: - import six # flake8: noqa + import six # NOQA:F401 except ImportError: if sys.version_info.major == 2: subprocess.check_call(['apt-get', 'install', '-y', 'python-six']) else: subprocess.check_call(['apt-get', 'install', '-y', 'python3-six']) - import six # flake8: noqa + import six # NOQA:F401 try: - import yaml # flake8: noqa + import yaml # NOQA:F401 except ImportError: if sys.version_info.major == 2: subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml']) else: subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml']) - import yaml # flake8: noqa + import yaml # NOQA:F401 # Holds a list of mapping of mangled function names that have been deprecated diff --git a/hooks/charmhelpers/contrib/hahelpers/apache.py b/hooks/charmhelpers/contrib/hahelpers/apache.py index 605a1bec..2c1e371e 100644 --- a/hooks/charmhelpers/contrib/hahelpers/apache.py +++ b/hooks/charmhelpers/contrib/hahelpers/apache.py @@ -23,8 +23,8 @@ # import os -import subprocess +from charmhelpers.core import host from charmhelpers.core.hookenv import ( config as config_get, relation_get, @@ -83,14 +83,4 @@ def retrieve_ca_cert(cert_file): def install_ca_cert(ca_cert): - if ca_cert: - cert_file = ('/usr/local/share/ca-certificates/' - 'keystone_juju_ca_cert.crt') - old_cert = retrieve_ca_cert(cert_file) - if old_cert and old_cert == ca_cert: - log("CA cert is the same as installed version", level=INFO) - else: - log("Installing new CA cert", level=INFO) - with open(cert_file, 'wb') as crt: - crt.write(ca_cert) - subprocess.check_call(['update-ca-certificates', '--fresh']) + host.install_ca_cert(ca_cert, 'keystone_juju_ca_cert') diff --git a/hooks/charmhelpers/contrib/hardening/apache/checks/config.py b/hooks/charmhelpers/contrib/hardening/apache/checks/config.py index 06482aac..341da9ee 100644 --- a/hooks/charmhelpers/contrib/hardening/apache/checks/config.py +++ b/hooks/charmhelpers/contrib/hardening/apache/checks/config.py @@ -14,6 +14,7 @@ import os import re +import six import subprocess @@ -95,6 +96,8 @@ class ApacheConfContext(object): ctxt = settings['hardening'] out = subprocess.check_output(['apache2', '-v']) + if six.PY3: + out = out.decode('utf-8') ctxt['apache_version'] = re.search(r'.+version: Apache/(.+?)\s.+', out).group(1) ctxt['apache_icondir'] = '/usr/share/apache2/icons/' diff --git a/hooks/charmhelpers/contrib/hardening/audits/apache.py b/hooks/charmhelpers/contrib/hardening/audits/apache.py index d32bf44e..04825f5a 100644 --- a/hooks/charmhelpers/contrib/hardening/audits/apache.py +++ b/hooks/charmhelpers/contrib/hardening/audits/apache.py @@ -15,7 +15,7 @@ import re import subprocess -from six import string_types +import six from charmhelpers.core.hookenv import ( log, @@ -35,7 +35,7 @@ class DisabledModuleAudit(BaseAudit): def __init__(self, modules): if modules is None: self.modules = [] - elif isinstance(modules, string_types): + elif isinstance(modules, six.string_types): self.modules = [modules] else: self.modules = modules @@ -69,6 +69,8 @@ class DisabledModuleAudit(BaseAudit): def _get_loaded_modules(): """Returns the modules which are enabled in Apache.""" output = subprocess.check_output(['apache2ctl', '-M']) + if six.PY3: + output = output.decode('utf-8') modules = [] for line in output.splitlines(): # Each line of the enabled module output looks like: diff --git a/hooks/charmhelpers/contrib/hardening/harden.py b/hooks/charmhelpers/contrib/hardening/harden.py index b55764cd..63f21b9c 100644 --- a/hooks/charmhelpers/contrib/hardening/harden.py +++ b/hooks/charmhelpers/contrib/hardening/harden.py @@ -27,6 +27,8 @@ from charmhelpers.contrib.hardening.ssh.checks import run_ssh_checks from charmhelpers.contrib.hardening.mysql.checks import run_mysql_checks from charmhelpers.contrib.hardening.apache.checks import run_apache_checks +_DISABLE_HARDENING_FOR_UNIT_TEST = False + def harden(overrides=None): """Hardening decorator. @@ -47,16 +49,28 @@ def harden(overrides=None): provided with 'harden' config. :returns: Returns value returned by decorated function once executed. """ + if overrides is None: + overrides = [] + def _harden_inner1(f): - log("Hardening function '%s'" % (f.__name__), level=DEBUG) + # As this has to be py2.7 compat, we can't use nonlocal. Use a trick + # to capture the dictionary that can then be updated. + _logged = {'done': False} def _harden_inner2(*args, **kwargs): + # knock out hardening via a config var; normally it won't get + # disabled. + if _DISABLE_HARDENING_FOR_UNIT_TEST: + return f(*args, **kwargs) + if not _logged['done']: + log("Hardening function '%s'" % (f.__name__), level=DEBUG) + _logged['done'] = True RUN_CATALOG = OrderedDict([('os', run_os_checks), ('ssh', run_ssh_checks), ('mysql', run_mysql_checks), ('apache', run_apache_checks)]) - enabled = overrides or (config("harden") or "").split() + enabled = overrides[:] or (config("harden") or "").split() if enabled: modules_to_run = [] # modules will always be performed in the following order diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 936b4036..9133e9b3 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -618,12 +618,12 @@ class OpenStackAmuletUtils(AmuletUtils): return self.authenticate_keystone(keystone_ip, user, password, project_name=tenant) - def authenticate_glance_admin(self, keystone): + def authenticate_glance_admin(self, keystone, force_v1_client=False): """Authenticates admin user with glance.""" self.log.debug('Authenticating glance admin...') ep = keystone.service_catalog.url_for(service_type='image', interface='adminURL') - if keystone.session: + if not force_v1_client and keystone.session: return glance_clientv2.Client("2", session=keystone.session) else: return glance_client.Client(ep, token=keystone.auth_token) @@ -680,18 +680,30 @@ class OpenStackAmuletUtils(AmuletUtils): nova.flavors.create(name, ram, vcpus, disk, flavorid, ephemeral, swap, rxtx_factor, is_public) - def create_cirros_image(self, glance, image_name): - """Download the latest cirros image and upload it to glance, - validate and return a resource pointer. + def glance_create_image(self, glance, image_name, image_url, + download_dir='tests', + hypervisor_type=None, + disk_format='qcow2', + architecture='x86_64', + container_format='bare'): + """Download an image and upload it to glance, validate its status + and return an image object pointer. KVM defaults, can override for + LXD. - :param glance: pointer to authenticated glance connection + :param glance: pointer to authenticated glance api connection :param image_name: display name for new image + :param image_url: url to retrieve + :param download_dir: directory to store downloaded image file + :param hypervisor_type: glance image hypervisor property + :param disk_format: glance image disk format + :param architecture: glance image architecture property + :param container_format: glance image container format :returns: glance image pointer """ - self.log.debug('Creating glance cirros image ' - '({})...'.format(image_name)) + self.log.debug('Creating glance image ({}) from ' + '{}...'.format(image_name, image_url)) - # Download cirros image + # Download image http_proxy = os.getenv('AMULET_HTTP_PROXY') self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) if http_proxy: @@ -700,31 +712,34 @@ class OpenStackAmuletUtils(AmuletUtils): else: opener = urllib.FancyURLopener() - f = opener.open('http://download.cirros-cloud.net/version/released') - version = f.read().strip() - cirros_img = 'cirros-{}-x86_64-disk.img'.format(version) - local_path = os.path.join('tests', cirros_img) - - if not os.path.exists(local_path): - cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net', - version, cirros_img) - opener.retrieve(cirros_url, local_path) - f.close() + abs_file_name = os.path.join(download_dir, image_name) + if not os.path.exists(abs_file_name): + opener.retrieve(image_url, abs_file_name) # Create glance image + glance_properties = { + 'architecture': architecture, + } + if hypervisor_type: + glance_properties['hypervisor_type'] = hypervisor_type + # Create glance image if float(glance.version) < 2.0: - with open(local_path) as fimage: - image = glance.images.create(name=image_name, is_public=True, - disk_format='qcow2', - container_format='bare', - data=fimage) + with open(abs_file_name) as f: + image = glance.images.create( + name=image_name, + is_public=True, + disk_format=disk_format, + container_format=container_format, + properties=glance_properties, + data=f) else: image = glance.images.create( name=image_name, - disk_format="qcow2", visibility="public", - container_format="bare") - glance.images.upload(image.id, open(local_path, 'rb')) + disk_format=disk_format, + container_format=container_format) + glance.images.upload(image.id, open(abs_file_name, 'rb')) + glance.images.update(image.id, **glance_properties) # Wait for image to reach active status img_id = image.id @@ -753,15 +768,54 @@ class OpenStackAmuletUtils(AmuletUtils): val_img_stat, val_img_cfmt, val_img_dfmt)) if val_img_name == image_name and val_img_stat == 'active' \ - and val_img_pub is True and val_img_cfmt == 'bare' \ - and val_img_dfmt == 'qcow2': + and val_img_pub is True and val_img_cfmt == container_format \ + and val_img_dfmt == disk_format: self.log.debug(msg_attr) else: - msg = ('Volume validation failed, {}'.format(msg_attr)) + msg = ('Image validation failed, {}'.format(msg_attr)) amulet.raise_status(amulet.FAIL, msg=msg) return image + def create_cirros_image(self, glance, image_name, hypervisor_type=None): + """Download the latest cirros image and upload it to glance, + validate and return a resource pointer. + + :param glance: pointer to authenticated glance connection + :param image_name: display name for new image + :param hypervisor_type: glance image hypervisor property + :returns: glance image pointer + """ + # /!\ DEPRECATION WARNING + self.log.warn('/!\\ DEPRECATION WARNING: use ' + 'glance_create_image instead of ' + 'create_cirros_image.') + + self.log.debug('Creating glance cirros image ' + '({})...'.format(image_name)) + + # Get cirros image URL + http_proxy = os.getenv('AMULET_HTTP_PROXY') + self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) + if http_proxy: + proxies = {'http': http_proxy} + opener = urllib.FancyURLopener(proxies) + else: + opener = urllib.FancyURLopener() + + f = opener.open('http://download.cirros-cloud.net/version/released') + version = f.read().strip() + cirros_img = 'cirros-{}-x86_64-disk.img'.format(version) + cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net', + version, cirros_img) + f.close() + + return self.glance_create_image( + glance, + image_name, + cirros_url, + hypervisor_type=hypervisor_type) + def delete_image(self, glance, image): """Delete the specified image.""" diff --git a/hooks/charmhelpers/contrib/openstack/cert_utils.py b/hooks/charmhelpers/contrib/openstack/cert_utils.py index de853b53..3e078703 100644 --- a/hooks/charmhelpers/contrib/openstack/cert_utils.py +++ b/hooks/charmhelpers/contrib/openstack/cert_utils.py @@ -25,7 +25,9 @@ from charmhelpers.core.hookenv import ( local_unit, network_get_primary_address, config, + related_units, relation_get, + relation_ids, unit_get, NoNetworkBinding, log, @@ -225,3 +227,49 @@ def process_certificates(service_name, relation_id, unit, create_ip_cert_links( ssl_dir, custom_hostname_link=custom_hostname_link) + + +def get_requests_for_local_unit(relation_name=None): + """Extract any certificates data targeted at this unit down relation_name. + + :param relation_name: str Name of relation to check for data. + :returns: List of bundles of certificates. + :rtype: List of dicts + """ + local_name = local_unit().replace('/', '_') + raw_certs_key = '{}.processed_requests'.format(local_name) + relation_name = relation_name or 'certificates' + bundles = [] + for rid in relation_ids(relation_name): + for unit in related_units(rid): + data = relation_get(rid=rid, unit=unit) + if data.get(raw_certs_key): + bundles.append({ + 'ca': data['ca'], + 'chain': data.get('chain'), + 'certs': json.loads(data[raw_certs_key])}) + return bundles + + +def get_bundle_for_cn(cn, relation_name=None): + """Extract certificates for the given cn. + + :param cn: str Canonical Name on certificate. + :param relation_name: str Relation to check for certificates down. + :returns: Dictionary of certificate data, + :rtype: dict. + """ + entries = get_requests_for_local_unit(relation_name) + cert_bundle = {} + for entry in entries: + for _cn, bundle in entry['certs'].items(): + if _cn == cn: + cert_bundle = { + 'cert': bundle['cert'], + 'key': bundle['key'], + 'chain': entry['chain'], + 'ca': entry['ca']} + break + if cert_bundle: + break + return cert_bundle diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 92cb742e..72084cb3 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -642,7 +642,7 @@ class HAProxyContext(OSContextGenerator): return {} l_unit = local_unit().replace('/', '-') - cluster_hosts = {} + cluster_hosts = collections.OrderedDict() # NOTE(jamespage): build out map of configured network endpoints # and associated backends @@ -1523,10 +1523,6 @@ class NeutronAPIContext(OSContextGenerator): 'rel_key': 'enable-nsg-logging', 'default': False, }, - 'nsg_log_output_base': { - 'rel_key': 'nsg-log-output-base', - 'default': None, - }, } ctxt = self.get_neutron_options({}) for rid in relation_ids('neutron-plugin-api'): @@ -1538,10 +1534,15 @@ class NeutronAPIContext(OSContextGenerator): if 'l2-population' in rdata: ctxt.update(self.get_neutron_options(rdata)) + extension_drivers = [] + if ctxt['enable_qos']: - ctxt['extension_drivers'] = 'qos' - else: - ctxt['extension_drivers'] = '' + extension_drivers.append('qos') + + if ctxt['enable_nsg_logging']: + extension_drivers.append('log') + + ctxt['extension_drivers'] = ','.join(extension_drivers) return ctxt @@ -1901,7 +1902,7 @@ class EnsureDirContext(OSContextGenerator): Some software requires a user to create a target directory to be scanned for drop-in files with a specific format. This is why this context is needed to do that before rendering a template. - ''' + ''' def __init__(self, dirname, **kwargs): '''Used merely to ensure that a given directory exists.''' @@ -1911,3 +1912,23 @@ class EnsureDirContext(OSContextGenerator): def __call__(self): mkdir(self.dirname, **self.kwargs) return {} + + +class VersionsContext(OSContextGenerator): + """Context to return the openstack and operating system versions. + + """ + def __init__(self, pkg='python-keystone'): + """Initialise context. + + :param pkg: Package to extrapolate openstack version from. + :type pkg: str + """ + self.pkg = pkg + + def __call__(self): + ostack = os_release(self.pkg, base='icehouse') + osystem = lsb_release()['DISTRIB_CODENAME'].lower() + return { + 'openstack_release': ostack, + 'operating_system_release': osystem} diff --git a/hooks/charmhelpers/contrib/openstack/ha/utils.py b/hooks/charmhelpers/contrib/openstack/ha/utils.py index 6060ae50..add8eb9a 100644 --- a/hooks/charmhelpers/contrib/openstack/ha/utils.py +++ b/hooks/charmhelpers/contrib/openstack/ha/utils.py @@ -28,6 +28,7 @@ import json import re from charmhelpers.core.hookenv import ( + expected_related_units, log, relation_set, charm_name, @@ -110,12 +111,17 @@ def assert_charm_supports_dns_ha(): def expect_ha(): """ Determine if the unit expects to be in HA - Check for VIP or dns-ha settings which indicate the unit should expect to - be related to hacluster. + Check juju goal-state if ha relation is expected, check for VIP or dns-ha + settings which indicate the unit should expect to be related to hacluster. @returns boolean """ - return config('vip') or config('dns-ha') + ha_related_units = [] + try: + ha_related_units = list(expected_related_units(reltype='ha')) + except (NotImplementedError, KeyError): + pass + return len(ha_related_units) > 0 or config('vip') or config('dns-ha') def generate_ha_relation_data(service): diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka index 8e6889e0..c281868b 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka +++ b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka @@ -1,12 +1,14 @@ {% if auth_host -%} [keystone_authtoken] -auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }} -auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }} auth_type = password {% if api_version == "3" -%} +auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/v3 +auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v3 project_domain_name = {{ admin_domain_name }} user_domain_name = {{ admin_domain_name }} {% else -%} +auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }} +auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }} project_domain_name = default user_domain_name = default {% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index ae48d6b4..29cad083 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -375,7 +375,7 @@ def get_swift_codename(version): return codenames[0] # NOTE: fallback - attempt to match with just major.minor version - match = re.match('^(\d+)\.(\d+)', version) + match = re.match(r'^(\d+)\.(\d+)', version) if match: major_minor_version = match.group(0) for codename, versions in six.iteritems(SWIFT_CODENAMES): @@ -395,7 +395,7 @@ def get_os_codename_package(package, fatal=True): out = subprocess.check_output(cmd) if six.PY3: out = out.decode('UTF-8') - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: return None lines = out.split('\n') for line in lines: @@ -427,11 +427,11 @@ def get_os_codename_package(package, fatal=True): vers = apt.upstream_version(pkg.current_ver.ver_str) if 'swift' in pkg.name: # Fully x.y.z match for swift versions - match = re.match('^(\d+)\.(\d+)\.(\d+)', vers) + match = re.match(r'^(\d+)\.(\d+)\.(\d+)', vers) else: # x.y match only for 20XX.X # and ignore patch level for other packages - match = re.match('^(\d+)\.(\d+)', vers) + match = re.match(r'^(\d+)\.(\d+)', vers) if match: vers = match.group(0) @@ -1450,20 +1450,33 @@ def pausable_restart_on_change(restart_map, stopstart=False, see core.utils.restart_on_change() for more details. + Note restart_map can be a callable, in which case, restart_map is only + evaluated at runtime. This means that it is lazy and the underlying + function won't be called if the decorated function is never called. Note, + retains backwards compatibility for passing a non-callable dictionary. + @param f: the function to decorate - @param restart_map: the restart map {conf_file: [services]} + @param restart_map: (optionally callable, which then returns the + restart_map) the restart map {conf_file: [services]} @param stopstart: DEFAULT false; whether to stop, start or just restart @returns decorator to use a restart_on_change with pausability """ def wrap(f): + # py27 compatible nonlocal variable. When py3 only, replace with + # nonlocal keyword + __restart_map_cache = {'cache': None} + @functools.wraps(f) def wrapped_f(*args, **kwargs): if is_unit_paused_set(): return f(*args, **kwargs) + if __restart_map_cache['cache'] is None: + __restart_map_cache['cache'] = restart_map() \ + if callable(restart_map) else restart_map # otherwise, normal restart_on_change functionality return restart_on_change_helper( - (lambda: f(*args, **kwargs)), restart_map, stopstart, - restart_functions) + (lambda: f(*args, **kwargs)), __restart_map_cache['cache'], + stopstart, restart_functions) return wrapped_f return wrap diff --git a/hooks/charmhelpers/contrib/storage/linux/loopback.py b/hooks/charmhelpers/contrib/storage/linux/loopback.py index 1d6ae6f0..0dfdae52 100644 --- a/hooks/charmhelpers/contrib/storage/linux/loopback.py +++ b/hooks/charmhelpers/contrib/storage/linux/loopback.py @@ -39,7 +39,7 @@ def loopback_devices(): devs = [d.strip().split(' ') for d in check_output(cmd).splitlines() if d != ''] for dev, _, f in devs: - loopbacks[dev.replace(':', '')] = re.search('\((\S+)\)', f).groups()[0] + loopbacks[dev.replace(':', '')] = re.search(r'\((\S+)\)', f).groups()[0] return loopbacks diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 9abf2a45..2e287659 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -510,6 +510,67 @@ def related_units(relid=None): subprocess.check_output(units_cmd_line).decode('UTF-8')) or [] +def expected_peer_units(): + """Get a generator for units we expect to join peer relation based on + goal-state. + + The local unit is excluded from the result to make it easy to gauge + completion of all peers joining the relation with existing hook tools. + + Example usage: + log('peer {} of {} joined peer relation' + .format(len(related_units()), + len(list(expected_peer_units())))) + + This function will raise NotImplementedError if used with juju versions + without goal-state support. + + :returns: iterator + :rtype: types.GeneratorType + :raises: NotImplementedError + """ + if not has_juju_version("2.4.0"): + # goal-state first appeared in 2.4.0. + raise NotImplementedError("goal-state") + _goal_state = goal_state() + return (key for key in _goal_state['units'] + if '/' in key and key != local_unit()) + + +def expected_related_units(reltype=None): + """Get a generator for units we expect to join relation based on + goal-state. + + Note that you can not use this function for the peer relation, take a look + at expected_peer_units() for that. + + This function will raise KeyError if you request information for a + relation type for which juju goal-state does not have information. It will + raise NotImplementedError if used with juju versions without goal-state + support. + + Example usage: + log('participant {} of {} joined relation {}' + .format(len(related_units()), + len(list(expected_related_units())), + relation_type())) + + :param reltype: Relation type to list data for, default is to list data for + the realtion type we are currently executing a hook for. + :type reltype: str + :returns: iterator + :rtype: types.GeneratorType + :raises: KeyError, NotImplementedError + """ + if not has_juju_version("2.4.4"): + # goal-state existed in 2.4.0, but did not list individual units to + # join a relation in 2.4.1 through 2.4.3. (LP: #1794739) + raise NotImplementedError("goal-state relation unit count") + reltype = reltype or relation_type() + _goal_state = goal_state() + return (key for key in _goal_state['relations'][reltype] if '/' in key) + + @cached def relation_for_unit(unit=None, rid=None): """Get the json represenation of a unit's relation""" @@ -998,6 +1059,7 @@ def application_version_set(version): @translate_exc(from_exc=OSError, to_exc=NotImplementedError) +@cached def goal_state(): """Juju goal state values""" cmd = ['goal-state', '--format=json'] diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index e9fd38a0..79953a44 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -34,13 +34,13 @@ import six from contextlib import contextmanager from collections import OrderedDict -from .hookenv import log, DEBUG, local_unit +from .hookenv import log, INFO, DEBUG, local_unit, charm_name from .fstab import Fstab from charmhelpers.osplatform import get_platform __platform__ = get_platform() if __platform__ == "ubuntu": - from charmhelpers.core.host_factory.ubuntu import ( + from charmhelpers.core.host_factory.ubuntu import ( # NOQA:F401 service_available, add_new_group, lsb_release, @@ -48,7 +48,7 @@ if __platform__ == "ubuntu": CompareHostReleases, ) # flake8: noqa -- ignore F401 for this import elif __platform__ == "centos": - from charmhelpers.core.host_factory.centos import ( + from charmhelpers.core.host_factory.centos import ( # NOQA:F401 service_available, add_new_group, lsb_release, @@ -58,6 +58,7 @@ elif __platform__ == "centos": UPDATEDB_PATH = '/etc/updatedb.conf' + def service_start(service_name, **kwargs): """Start a system service. @@ -287,8 +288,8 @@ def service_running(service_name, **kwargs): for key, value in six.iteritems(kwargs): parameter = '%s=%s' % (key, value) cmd.append(parameter) - output = subprocess.check_output(cmd, - stderr=subprocess.STDOUT).decode('UTF-8') + output = subprocess.check_output( + cmd, stderr=subprocess.STDOUT).decode('UTF-8') except subprocess.CalledProcessError: return False else: @@ -442,7 +443,7 @@ def add_user_to_group(username, group): def chage(username, lastday=None, expiredate=None, inactive=None, - mindays=None, maxdays=None, root=None, warndays=None): + mindays=None, maxdays=None, root=None, warndays=None): """Change user password expiry information :param str username: User to update @@ -482,8 +483,10 @@ def chage(username, lastday=None, expiredate=None, inactive=None, cmd.append(username) subprocess.check_call(cmd) + remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1') + def rsync(from_path, to_path, flags='-r', options=None, timeout=None): """Replicate the contents of a path""" options = options or ['--delete', '--executability'] @@ -535,13 +538,15 @@ def write_file(path, content, owner='root', group='root', perms=0o444): # lets see if we can grab the file and compare the context, to avoid doing # a write. existing_content = None - existing_uid, existing_gid = None, None + existing_uid, existing_gid, existing_perms = None, None, None try: with open(path, 'rb') as target: existing_content = target.read() stat = os.stat(path) - existing_uid, existing_gid = stat.st_uid, stat.st_gid - except: + existing_uid, existing_gid, existing_perms = ( + stat.st_uid, stat.st_gid, stat.st_mode + ) + except Exception: pass if content != existing_content: log("Writing file {} {}:{} {:o}".format(path, owner, group, perms), @@ -554,7 +559,7 @@ def write_file(path, content, owner='root', group='root', perms=0o444): target.write(content) return # the contents were the same, but we might still need to change the - # ownership. + # ownership or permissions. if existing_uid != uid: log("Changing uid on already existing content: {} -> {}" .format(existing_uid, uid), level=DEBUG) @@ -563,6 +568,10 @@ def write_file(path, content, owner='root', group='root', perms=0o444): log("Changing gid on already existing content: {} -> {}" .format(existing_gid, gid), level=DEBUG) os.chown(path, -1, gid) + if existing_perms != perms: + log("Changing permissions on existing content: {} -> {}" + .format(existing_perms, perms), level=DEBUG) + os.chmod(path, perms) def fstab_remove(mp): @@ -827,7 +836,7 @@ def list_nics(nic_type=None): ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') ip_output = (line.strip() for line in ip_output if line) - key = re.compile('^[0-9]+:\s+(.+):') + key = re.compile(r'^[0-9]+:\s+(.+):') for line in ip_output: matched = re.search(key, line) if matched: @@ -1040,3 +1049,27 @@ def modulo_distribution(modulo=3, wait=30, non_zero_wait=False): return modulo * wait else: return calculated_wait_time + + +def install_ca_cert(ca_cert, name=None): + """ + Install the given cert as a trusted CA. + + The ``name`` is the stem of the filename where the cert is written, and if + not provided, it will default to ``juju-{charm_name}``. + + If the cert is empty or None, or is unchanged, nothing is done. + """ + if not ca_cert: + return + if not isinstance(ca_cert, bytes): + ca_cert = ca_cert.encode('utf8') + if not name: + name = 'juju-{}'.format(charm_name()) + cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name) + new_hash = hashlib.md5(ca_cert).hexdigest() + if file_hash(cert_file) == new_hash: + return + log("Installing new CA cert at: {}".format(cert_file), level=INFO) + write_file(cert_file, ca_cert) + subprocess.check_call(['update-ca-certificates', '--fresh']) diff --git a/hooks/charmhelpers/core/kernel.py b/hooks/charmhelpers/core/kernel.py index 2d404528..e01f4f8b 100644 --- a/hooks/charmhelpers/core/kernel.py +++ b/hooks/charmhelpers/core/kernel.py @@ -26,12 +26,12 @@ from charmhelpers.core.hookenv import ( __platform__ = get_platform() if __platform__ == "ubuntu": - from charmhelpers.core.kernel_factory.ubuntu import ( + from charmhelpers.core.kernel_factory.ubuntu import ( # NOQA:F401 persistent_modprobe, update_initramfs, ) # flake8: noqa -- ignore F401 for this import elif __platform__ == "centos": - from charmhelpers.core.kernel_factory.centos import ( + from charmhelpers.core.kernel_factory.centos import ( # NOQA:F401 persistent_modprobe, update_initramfs, ) # flake8: noqa -- ignore F401 for this import diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py index ec08cbc2..c7ad128c 100644 --- a/hooks/charmhelpers/fetch/ubuntu.py +++ b/hooks/charmhelpers/fetch/ubuntu.py @@ -294,7 +294,7 @@ def apt_unhold(packages, fatal=False): def import_key(key): """Import an ASCII Armor key. - /!\ A Radix64 format keyid is also supported for backwards + A Radix64 format keyid is also supported for backwards compatibility, but should never be used; the key retrieval mechanism is insecure and subject to man-in-the-middle attacks voiding all signature checks using that key. @@ -454,6 +454,9 @@ def _add_apt_repository(spec): :param spec: the parameter to pass to add_apt_repository """ + if '{series}' in spec: + series = lsb_release()['DISTRIB_CODENAME'] + spec = spec.replace('{series}', series) _run_with_retries(['add-apt-repository', '--yes', spec]) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 23ae53df..bed92a7f 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -155,8 +155,18 @@ class CinderBasicDeployment(OpenStackAmuletDeployment): api_version = 1 self.cinder = u.authenticate_cinder_admin(self.keystone, api_version) + force_v1_client = False + if self._get_openstack_release() == self.trusty_icehouse: + # Updating image properties (such as arch or hypervisor) using the + # v2 api in icehouse results in: + # https://bugs.launchpad.net/python-glanceclient/+bug/1371559 + u.log.debug('Forcing glance to use v1 api') + force_v1_client = True + # Authenticate admin with glance endpoint - self.glance = u.authenticate_glance_admin(self.keystone) + self.glance = u.authenticate_glance_admin( + self.keystone, + force_v1_client=force_v1_client) def _extend_cinder_volume(self, vol_id, new_size=2): """Extend an existing cinder volume size. @@ -571,14 +581,7 @@ class CinderBasicDeployment(OpenStackAmuletDeployment): unit = self.cinder_sentry conf = '/etc/cinder/cinder.conf' unit_mq = self.rabbitmq_sentry - unit_ks = self.keystone_sentry rel_mq_ci = unit_mq.relation('amqp', 'cinder:amqp') - rel_ks_ci = unit_ks.relation('identity-service', - 'cinder:identity-service') - auth_uri = ('http://%s:%s/' % - (rel_ks_ci['auth_host'], rel_ks_ci['service_port'])) - auth_url = ('http://%s:%s/' % - (rel_ks_ci['auth_host'], rel_ks_ci['auth_port'])) expected = { 'DEFAULT': { @@ -588,13 +591,6 @@ class CinderBasicDeployment(OpenStackAmuletDeployment): 'iscsi_helper': 'tgtadm', 'auth_strategy': 'keystone', }, - 'keystone_authtoken': { - 'admin_user': rel_ks_ci['service_username'], - 'admin_password': rel_ks_ci['service_password'], - 'admin_tenant_name': rel_ks_ci['service_tenant'], - 'auth_uri': auth_uri, - 'signing_dir': '/var/cache/cinder' - } } if self._get_openstack_release() < self.xenial_ocata: expected['DEFAULT']['volume_group'] = 'cinder-volumes' @@ -613,47 +609,6 @@ class CinderBasicDeployment(OpenStackAmuletDeployment): 'rabbit_password': rel_mq_ci['password'], 'rabbit_host': rel_mq_ci['hostname'], } - if self._get_openstack_release() >= self.xenial_queens: - expected['keystone_authtoken'] = { - 'auth_uri': auth_uri.rstrip('/'), - 'auth_url': auth_url.rstrip('/'), - 'auth_type': 'password', - 'project_domain_name': 'service_domain', - 'user_domain_name': 'service_domain', - 'project_name': 'services', - 'username': rel_ks_ci['service_username'], - 'password': rel_ks_ci['service_password'], - 'signing_dir': '/var/cache/cinder' - } - elif self._get_openstack_release() >= self.trusty_mitaka: - expected['keystone_authtoken'] = { - 'auth_uri': auth_uri.rstrip('/'), - 'auth_url': auth_url.rstrip('/'), - 'auth_type': 'password', - 'project_domain_name': 'default', - 'user_domain_name': 'default', - 'project_name': 'services', - 'username': rel_ks_ci['service_username'], - 'password': rel_ks_ci['service_password'], - 'signing_dir': '/var/cache/cinder' - } - elif self._get_openstack_release() >= self.trusty_liberty: - expected['keystone_authtoken'] = { - 'auth_uri': auth_uri.rstrip('/'), - 'auth_url': auth_url.rstrip('/'), - 'auth_plugin': 'password', - 'project_domain_id': 'default', - 'user_domain_id': 'default', - 'project_name': 'services', - 'username': rel_ks_ci['service_username'], - 'password': rel_ks_ci['service_password'], - 'signing_dir': '/var/cache/cinder' - } - - if self._get_openstack_release() == self.trusty_kilo: - expected['keystone_authtoken']['auth_uri'] = auth_uri - expected['keystone_authtoken']['identity_uri'] = \ - auth_url.strip('/') if self._get_openstack_release() >= self.trusty_kilo: # Kilo or later @@ -661,8 +616,6 @@ class CinderBasicDeployment(OpenStackAmuletDeployment): else: # Juno or earlier expected['DEFAULT'].update(expected_rmq) - expected['keystone_authtoken']['auth_host'] = \ - rel_ks_ci['auth_host'] for section, pairs in expected.iteritems(): ret = u.validate_config_data(unit, conf, section, pairs)