From 66707ffa31e19d789aa08cdf1bb70861edecd2ce Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Tue, 14 Feb 2017 11:57:16 -0800 Subject: [PATCH] Pre-release charm-helpers sync 17.02 Get each charm up to date with lp:charm-helpers for release testing. Add necessary relation between nova cloud controller and neutron gateway Closes-Bug: 1665008 Change-Id: I3fb3769b147d42d36e53293b759d394f2dc63071 --- hooks/charmhelpers/contrib/network/ip.py | 6 +- .../contrib/openstack/amulet/utils.py | 174 ++++++++++++-- .../charmhelpers/contrib/openstack/context.py | 74 ++++++ .../openstack/templates/memcached.conf | 53 ++++ .../section-keystone-authtoken-mitaka | 3 + .../templates/wsgi-openstack-api.conf | 100 ++++++++ hooks/charmhelpers/contrib/openstack/utils.py | 70 +++++- .../contrib/storage/linux/ceph.py | 16 +- hooks/charmhelpers/core/hookenv.py | 45 ++++ hooks/charmhelpers/core/host.py | 227 +++++++++++++++--- hooks/charmhelpers/osplatform.py | 6 + tests/basic_deployment.py | 2 + 12 files changed, 721 insertions(+), 55 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/templates/memcached.conf create mode 100644 hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 2d2026e..e141fc1 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -424,7 +424,11 @@ def ns_query(address): else: return None - answers = dns.resolver.query(address, rtype) + try: + answers = dns.resolver.query(address, rtype) + except dns.resolver.NXDOMAIN: + return None + if answers: return str(answers[0]) return None diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 6a0ba83..401c032 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -20,6 +20,7 @@ import re import six import time import urllib +import urlparse import cinderclient.v1.client as cinder_client import glanceclient.v1.client as glance_client @@ -37,6 +38,7 @@ import swiftclient from charmhelpers.contrib.amulet.utils import ( AmuletUtils ) +from charmhelpers.core.decorators import retry_on_exception DEBUG = logging.DEBUG ERROR = logging.ERROR @@ -303,6 +305,46 @@ class OpenStackAmuletUtils(AmuletUtils): self.log.debug('Checking if tenant exists ({})...'.format(tenant)) return tenant in [t.name for t in keystone.tenants.list()] + @retry_on_exception(5, base_delay=10) + def keystone_wait_for_propagation(self, sentry_relation_pairs, + api_version): + """Iterate over list of sentry and relation tuples and verify that + api_version has the expected value. + + :param sentry_relation_pairs: list of sentry, relation name tuples used + for monitoring propagation of relation + data + :param api_version: api_version to expect in relation data + :returns: None if successful. Raise on error. + """ + for (sentry, relation_name) in sentry_relation_pairs: + rel = sentry.relation('identity-service', + relation_name) + self.log.debug('keystone relation data: {}'.format(rel)) + if rel['api_version'] != str(api_version): + raise Exception("api_version not propagated through relation" + " data yet ('{}' != '{}')." + "".format(rel['api_version'], api_version)) + + def keystone_configure_api_version(self, sentry_relation_pairs, deployment, + api_version): + """Configure preferred-api-version of keystone in deployment and + monitor provided list of relation objects for propagation + before returning to caller. + + :param sentry_relation_pairs: list of sentry, relation tuples used for + monitoring propagation of relation data + :param deployment: deployment to configure + :param api_version: value preferred-api-version will be set to + :returns: None if successful. Raise on error. + """ + self.log.debug("Setting keystone preferred-api-version: '{}'" + "".format(api_version)) + + config = {'preferred-api-version': api_version} + deployment.d.configure('keystone', config) + self.keystone_wait_for_propagation(sentry_relation_pairs, api_version) + def authenticate_cinder_admin(self, keystone_sentry, username, password, tenant): """Authenticates admin user with cinder.""" @@ -311,6 +353,37 @@ class OpenStackAmuletUtils(AmuletUtils): ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8')) return cinder_client.Client(username, password, tenant, ept) + def authenticate_keystone(self, keystone_ip, username, password, + api_version=False, admin_port=False, + user_domain_name=None, domain_name=None, + project_domain_name=None, project_name=None): + """Authenticate with Keystone""" + self.log.debug('Authenticating with keystone...') + port = 5000 + if admin_port: + port = 35357 + base_ep = "http://{}:{}".format(keystone_ip.strip().decode('utf-8'), + port) + if not api_version or api_version == 2: + ep = base_ep + "/v2.0" + return keystone_client.Client(username=username, password=password, + tenant_name=project_name, + auth_url=ep) + else: + ep = base_ep + "/v3" + auth = keystone_id_v3.Password( + user_domain_name=user_domain_name, + username=username, + password=password, + domain_name=domain_name, + project_domain_name=project_domain_name, + project_name=project_name, + auth_url=ep + ) + return keystone_client_v3.Client( + session=keystone_session.Session(auth=auth) + ) + def authenticate_keystone_admin(self, keystone_sentry, user, password, tenant=None, api_version=None, keystone_ip=None): @@ -319,30 +392,28 @@ class OpenStackAmuletUtils(AmuletUtils): if not keystone_ip: keystone_ip = keystone_sentry.info['public-address'] - base_ep = "http://{}:35357".format(keystone_ip.strip().decode('utf-8')) - if not api_version or api_version == 2: - ep = base_ep + "/v2.0" - return keystone_client.Client(username=user, password=password, - tenant_name=tenant, auth_url=ep) - else: - ep = base_ep + "/v3" - auth = keystone_id_v3.Password( - user_domain_name='admin_domain', - username=user, - password=password, - domain_name='admin_domain', - auth_url=ep, - ) - sess = keystone_session.Session(auth=auth) - return keystone_client_v3.Client(session=sess) + user_domain_name = None + domain_name = None + if api_version == 3: + user_domain_name = 'admin_domain' + domain_name = user_domain_name + + return self.authenticate_keystone(keystone_ip, user, password, + project_name=tenant, + api_version=api_version, + user_domain_name=user_domain_name, + domain_name=domain_name, + admin_port=True) def authenticate_keystone_user(self, keystone, user, password, tenant): """Authenticates a regular user with the keystone public endpoint.""" self.log.debug('Authenticating keystone user ({})...'.format(user)) ep = keystone.service_catalog.url_for(service_type='identity', endpoint_type='publicURL') - return keystone_client.Client(username=user, password=password, - tenant_name=tenant, auth_url=ep) + keystone_ip = urlparse.urlparse(ep).hostname + + return self.authenticate_keystone(keystone_ip, user, password, + project_name=tenant) def authenticate_glance_admin(self, keystone): """Authenticates admin user with glance.""" @@ -1133,3 +1204,70 @@ class OpenStackAmuletUtils(AmuletUtils): else: msg = 'No message retrieved.' amulet.raise_status(amulet.FAIL, msg) + + def validate_memcache(self, sentry_unit, conf, os_release, + earliest_release=5, section='keystone_authtoken', + check_kvs=None): + """Check Memcache is running and is configured to be used + + Example call from Amulet test: + + def test_110_memcache(self): + u.validate_memcache(self.neutron_api_sentry, + '/etc/neutron/neutron.conf', + self._get_openstack_release()) + + :param sentry_unit: sentry unit + :param conf: OpenStack config file to check memcache settings + :param os_release: Current OpenStack release int code + :param earliest_release: Earliest Openstack release to check int code + :param section: OpenStack config file section to check + :param check_kvs: Dict of settings to check in config file + :returns: None + """ + if os_release < earliest_release: + self.log.debug('Skipping memcache checks for deployment. {} <' + 'mitaka'.format(os_release)) + return + _kvs = check_kvs or {'memcached_servers': 'inet6:[::1]:11211'} + self.log.debug('Checking memcached is running') + ret = self.validate_services_by_name({sentry_unit: ['memcached']}) + if ret: + amulet.raise_status(amulet.FAIL, msg='Memcache running check' + 'failed {}'.format(ret)) + else: + self.log.debug('OK') + self.log.debug('Checking memcache url is configured in {}'.format( + conf)) + if self.validate_config_data(sentry_unit, conf, section, _kvs): + message = "Memcache config error in: {}".format(conf) + amulet.raise_status(amulet.FAIL, msg=message) + else: + self.log.debug('OK') + self.log.debug('Checking memcache configuration in ' + '/etc/memcached.conf') + contents = self.file_contents_safe(sentry_unit, '/etc/memcached.conf', + fatal=True) + ubuntu_release, _ = self.run_cmd_unit(sentry_unit, 'lsb_release -cs') + if ubuntu_release <= 'trusty': + memcache_listen_addr = 'ip6-localhost' + else: + memcache_listen_addr = '::1' + expected = { + '-p': '11211', + '-l': memcache_listen_addr} + found = [] + for key, value in expected.items(): + for line in contents.split('\n'): + if line.startswith(key): + self.log.debug('Checking {} is set to {}'.format( + key, + value)) + assert value == line.split()[-1] + self.log.debug(line.split()[-1]) + found.append(key) + if sorted(found) == sorted(expected.keys()): + self.log.debug('OK') + else: + message = "Memcache config error in: /etc/memcached.conf" + amulet.raise_status(amulet.FAIL, msg=message) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index d5b3a33..4231633 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -14,6 +14,7 @@ import glob import json +import math import os import re import time @@ -90,6 +91,9 @@ from charmhelpers.contrib.network.ip import ( from charmhelpers.contrib.openstack.utils import ( config_flags_parser, get_host_ip, + git_determine_usr_bin, + git_determine_python_path, + enable_memcache, ) from charmhelpers.core.unitdata import kv @@ -1207,6 +1211,43 @@ class WorkerConfigContext(OSContextGenerator): return ctxt +class WSGIWorkerConfigContext(WorkerConfigContext): + + def __init__(self, name=None, script=None, admin_script=None, + public_script=None, process_weight=1.00, + admin_process_weight=0.75, public_process_weight=0.25): + self.service_name = name + self.user = name + self.group = name + self.script = script + self.admin_script = admin_script + self.public_script = public_script + self.process_weight = process_weight + self.admin_process_weight = admin_process_weight + self.public_process_weight = public_process_weight + + def __call__(self): + multiplier = config('worker-multiplier') or 1 + total_processes = self.num_cpus * multiplier + ctxt = { + "service_name": self.service_name, + "user": self.user, + "group": self.group, + "script": self.script, + "admin_script": self.admin_script, + "public_script": self.public_script, + "processes": int(math.ceil(self.process_weight * total_processes)), + "admin_processes": int(math.ceil(self.admin_process_weight * + total_processes)), + "public_processes": int(math.ceil(self.public_process_weight * + total_processes)), + "threads": 1, + "usr_bin": git_determine_usr_bin(), + "python_path": git_determine_python_path(), + } + return ctxt + + class ZeroMQContext(OSContextGenerator): interfaces = ['zeromq-configuration'] @@ -1512,3 +1553,36 @@ class AppArmorContext(OSContextGenerator): "".format(self.ctxt['aa_profile'], self.ctxt['aa_profile_mode'])) raise e + + +class MemcacheContext(OSContextGenerator): + """Memcache context + + This context provides options for configuring a local memcache client and + server + """ + + def __init__(self, package=None): + """ + @param package: Package to examine to extrapolate OpenStack release. + Used when charms have no openstack-origin config + option (ie subordinates) + """ + self.package = package + + def __call__(self): + ctxt = {} + ctxt['use_memcache'] = enable_memcache(package=self.package) + if ctxt['use_memcache']: + # Trusty version of memcached does not support ::1 as a listen + # address so use host file entry instead + if lsb_release()['DISTRIB_CODENAME'].lower() > 'trusty': + ctxt['memcache_server'] = '::1' + else: + ctxt['memcache_server'] = 'ip6-localhost' + ctxt['memcache_server_formatted'] = '[::1]' + ctxt['memcache_port'] = '11211' + ctxt['memcache_url'] = 'inet6:{}:{}'.format( + ctxt['memcache_server_formatted'], + ctxt['memcache_port']) + return ctxt diff --git a/hooks/charmhelpers/contrib/openstack/templates/memcached.conf b/hooks/charmhelpers/contrib/openstack/templates/memcached.conf new file mode 100644 index 0000000..26cb037 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/memcached.conf @@ -0,0 +1,53 @@ +############################################################################### +# [ WARNING ] +# memcached configuration file maintained by Juju +# local changes may be overwritten. +############################################################################### + +# memcached default config file +# 2003 - Jay Bonci +# This configuration file is read by the start-memcached script provided as +# part of the Debian GNU/Linux distribution. + +# Run memcached as a daemon. This command is implied, and is not needed for the +# daemon to run. See the README.Debian that comes with this package for more +# information. +-d + +# Log memcached's output to /var/log/memcached +logfile /var/log/memcached.log + +# Be verbose +# -v + +# Be even more verbose (print client commands as well) +# -vv + +# Start with a cap of 64 megs of memory. It's reasonable, and the daemon default +# Note that the daemon will grow to this size, but does not start out holding this much +# memory +-m 64 + +# Default connection port is 11211 +-p {{ memcache_port }} + +# Run the daemon as root. The start-memcached will default to running as root if no +# -u command is present in this config file +-u memcache + +# Specify which IP address to listen on. The default is to listen on all IP addresses +# This parameter is one of the only security measures that memcached has, so make sure +# it's listening on a firewalled interface. +-l {{ memcache_server }} + +# Limit the number of simultaneous incoming connections. The daemon default is 1024 +# -c 1024 + +# Lock down all paged memory. Consult with the README and homepage before you do this +# -k + +# Return error when memory is exhausted (rather than removing items) +# -M + +# Maximize core file limit +# -r diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka index 7c6f0c3..8e6889e 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka +++ b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka @@ -14,4 +14,7 @@ project_name = {{ admin_tenant_name }} username = {{ admin_user }} password = {{ admin_password }} signing_dir = {{ signing_dir }} +{% if use_memcache == true %} +memcached_servers = {{ memcache_url }} +{% endif -%} {% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf b/hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf new file mode 100644 index 0000000..315b2a3 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf @@ -0,0 +1,100 @@ +# Configuration file maintained by Juju. Local changes may be overwritten. + +{% if port -%} +Listen {{ port }} +{% endif -%} + +{% if admin_port -%} +Listen {{ admin_port }} +{% endif -%} + +{% if public_port -%} +Listen {{ public_port }} +{% endif -%} + +{% if port -%} + + WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \ +{% if python_path -%} + python-path={{ python_path }} \ +{% endif -%} + display-name=%{GROUP} + WSGIProcessGroup {{ service_name }} + WSGIScriptAlias / {{ script }} + WSGIApplicationGroup %{GLOBAL} + WSGIPassAuthorization On + = 2.4> + ErrorLogFormat "%{cu}t %M" + + ErrorLog /var/log/apache2/{{ service_name }}_error.log + CustomLog /var/log/apache2/{{ service_name }}_access.log combined + + + = 2.4> + Require all granted + + + Order allow,deny + Allow from all + + + +{% endif -%} + +{% if admin_port -%} + + WSGIDaemonProcess {{ service_name }}-admin processes={{ admin_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \ +{% if python_path -%} + python-path={{ python_path }} \ +{% endif -%} + display-name=%{GROUP} + WSGIProcessGroup {{ service_name }}-admin + WSGIScriptAlias / {{ admin_script }} + WSGIApplicationGroup %{GLOBAL} + WSGIPassAuthorization On + = 2.4> + ErrorLogFormat "%{cu}t %M" + + ErrorLog /var/log/apache2/{{ service_name }}_error.log + CustomLog /var/log/apache2/{{ service_name }}_access.log combined + + + = 2.4> + Require all granted + + + Order allow,deny + Allow from all + + + +{% endif -%} + +{% if public_port -%} + + WSGIDaemonProcess {{ service_name }}-public processes={{ public_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \ +{% if python_path -%} + python-path={{ python_path }} \ +{% endif -%} + display-name=%{GROUP} + WSGIProcessGroup {{ service_name }}-public + WSGIScriptAlias / {{ public_script }} + WSGIApplicationGroup %{GLOBAL} + WSGIPassAuthorization On + = 2.4> + ErrorLogFormat "%{cu}t %M" + + ErrorLog /var/log/apache2/{{ service_name }}_error.log + CustomLog /var/log/apache2/{{ service_name }}_access.log combined + + + = 2.4> + Require all granted + + + Order allow,deny + Allow from all + + + +{% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 6d544e7..80219d6 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -153,7 +153,7 @@ SWIFT_CODENAMES = OrderedDict([ ('newton', ['2.8.0', '2.9.0', '2.10.0']), ('ocata', - ['2.11.0']), + ['2.11.0', '2.12.0']), ]) # >= Liberty version->codename mapping @@ -549,9 +549,9 @@ def configure_installation_source(rel): 'newton': 'xenial-updates/newton', 'newton/updates': 'xenial-updates/newton', 'newton/proposed': 'xenial-proposed/newton', - 'zesty': 'zesty-updates/ocata', - 'zesty/updates': 'xenial-updates/ocata', - 'zesty/proposed': 'xenial-proposed/ocata', + 'ocata': 'xenial-updates/ocata', + 'ocata/updates': 'xenial-updates/ocata', + 'ocata/proposed': 'xenial-proposed/ocata', } try: @@ -1119,6 +1119,35 @@ def git_generate_systemd_init_files(templates_dir): shutil.copyfile(service_source, service_dest) +def git_determine_usr_bin(): + """Return the /usr/bin path for Apache2 config. + + The /usr/bin path will be located in the virtualenv if the charm + is configured to deploy from source. + """ + if git_install_requested(): + projects_yaml = config('openstack-origin-git') + projects_yaml = git_default_repos(projects_yaml) + return os.path.join(git_pip_venv_dir(projects_yaml), 'bin') + else: + return '/usr/bin' + + +def git_determine_python_path(): + """Return the python-path for Apache2 config. + + Returns 'None' unless the charm is configured to deploy from source, + in which case the path of the virtualenv's site-packages is returned. + """ + if git_install_requested(): + projects_yaml = config('openstack-origin-git') + projects_yaml = git_default_repos(projects_yaml) + return os.path.join(git_pip_venv_dir(projects_yaml), + 'lib/python2.7/site-packages') + else: + return None + + def os_workload_status(configs, required_interfaces, charm_func=None): """ Decorator to set workload status based on complete contexts @@ -1925,3 +1954,36 @@ def os_application_version_set(package): application_version_set(os_release(package)) else: application_version_set(application_version) + + +def enable_memcache(source=None, release=None, package=None): + """Determine if memcache should be enabled on the local unit + + @param release: release of OpenStack currently deployed + @param package: package to derive OpenStack version deployed + @returns boolean Whether memcache should be enabled + """ + _release = None + if release: + _release = release + else: + _release = os_release(package, base='icehouse') + if not _release: + _release = get_os_codename_install_source(source) + + # TODO: this should be changed to a numeric comparison using a known list + # of releases and comparing by index. + return _release >= 'mitaka' + + +def token_cache_pkgs(source=None, release=None): + """Determine additional packages needed for token caching + + @param source: source string for charm + @param release: release of OpenStack currently deployed + @returns List of package to enable token caching + """ + packages = [] + if enable_memcache(source=source, release=release): + packages.extend(['memcached', 'python-memcache']) + return packages diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index edb536c..ae7f3f9 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -40,6 +40,7 @@ from subprocess import ( ) from charmhelpers.core.hookenv import ( config, + service_name, local_unit, relation_get, relation_ids, @@ -1043,8 +1044,18 @@ class CephBrokerRq(object): self.request_id = str(uuid.uuid1()) self.ops = [] + def add_op_request_access_to_group(self, name, namespace=None, + permission=None, key_name=None): + """ + Adds the requested permissions to the current service's Ceph key, + allowing the key to access only the specified pools + """ + self.ops.append({'op': 'add-permissions-to-key', 'group': name, + 'namespace': namespace, 'name': key_name or service_name(), + 'group-permission': permission}) + def add_op_create_pool(self, name, replica_count=3, pg_num=None, - weight=None): + weight=None, group=None, namespace=None): """Adds an operation to create a pool. @param pg_num setting: optional setting. If not provided, this value @@ -1058,7 +1069,8 @@ class CephBrokerRq(object): self.ops.append({'op': 'create-pool', 'name': name, 'replicas': replica_count, 'pg_num': pg_num, - 'weight': weight}) + 'weight': weight, 'group': group, + 'group-namespace': namespace}) def set_ops(self, ops): """Set request ops to provided value. diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 94fc996..e44e22b 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -616,6 +616,20 @@ def close_port(port, protocol="TCP"): subprocess.check_call(_args) +def open_ports(start, end, protocol="TCP"): + """Opens a range of service network ports""" + _args = ['open-port'] + _args.append('{}-{}/{}'.format(start, end, protocol)) + subprocess.check_call(_args) + + +def close_ports(start, end, protocol="TCP"): + """Close a range of service network ports""" + _args = ['close-port'] + _args.append('{}-{}/{}'.format(start, end, protocol)) + subprocess.check_call(_args) + + @cached def unit_get(attribute): """Get the unit ID for the remote unit""" @@ -1021,3 +1035,34 @@ def network_get_primary_address(binding): ''' cmd = ['network-get', '--primary-address', binding] return subprocess.check_output(cmd).decode('UTF-8').strip() + + +def add_metric(*args, **kwargs): + """Add metric values. Values may be expressed with keyword arguments. For + metric names containing dashes, these may be expressed as one or more + 'key=value' positional arguments. May only be called from the collect-metrics + hook.""" + _args = ['add-metric'] + _kvpairs = [] + _kvpairs.extend(args) + _kvpairs.extend(['{}={}'.format(k, v) for k, v in kwargs.items()]) + _args.extend(sorted(_kvpairs)) + try: + subprocess.check_call(_args) + return + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + log_message = 'add-metric failed: {}'.format(' '.join(_kvpairs)) + log(log_message, level='INFO') + + +def meter_status(): + """Get the meter status, if running in the meter-status-changed hook.""" + return os.environ.get('JUJU_METER_STATUS') + + +def meter_info(): + """Get the meter status information, if running in the meter-status-changed + hook.""" + return os.environ.get('JUJU_METER_INFO') diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 04cadb3..edbb72f 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -54,38 +54,138 @@ elif __platform__ == "centos": cmp_pkgrevno, ) # flake8: noqa -- ignore F401 for this import +UPDATEDB_PATH = '/etc/updatedb.conf' -def service_start(service_name): - """Start a system service""" - return service('start', service_name) +def service_start(service_name, **kwargs): + """Start a system service. + + The specified service name is managed via the system level init system. + Some init systems (e.g. upstart) require that additional arguments be + provided in order to directly control service instances whereas other init + systems allow for addressing instances of a service directly by name (e.g. + systemd). + + The kwargs allow for the additional parameters to be passed to underlying + init systems for those systems which require/allow for them. For example, + the ceph-osd upstart script requires the id parameter to be passed along + in order to identify which running daemon should be reloaded. The follow- + ing example stops the ceph-osd service for instance id=4: + + service_stop('ceph-osd', id=4) + + :param service_name: the name of the service to stop + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for systemd enabled systems. + """ + return service('start', service_name, **kwargs) -def service_stop(service_name): - """Stop a system service""" - return service('stop', service_name) +def service_stop(service_name, **kwargs): + """Stop a system service. + + The specified service name is managed via the system level init system. + Some init systems (e.g. upstart) require that additional arguments be + provided in order to directly control service instances whereas other init + systems allow for addressing instances of a service directly by name (e.g. + systemd). + + The kwargs allow for the additional parameters to be passed to underlying + init systems for those systems which require/allow for them. For example, + the ceph-osd upstart script requires the id parameter to be passed along + in order to identify which running daemon should be reloaded. The follow- + ing example stops the ceph-osd service for instance id=4: + + service_stop('ceph-osd', id=4) + + :param service_name: the name of the service to stop + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for systemd enabled systems. + """ + return service('stop', service_name, **kwargs) -def service_restart(service_name): - """Restart a system service""" +def service_restart(service_name, **kwargs): + """Restart a system service. + + The specified service name is managed via the system level init system. + Some init systems (e.g. upstart) require that additional arguments be + provided in order to directly control service instances whereas other init + systems allow for addressing instances of a service directly by name (e.g. + systemd). + + The kwargs allow for the additional parameters to be passed to underlying + init systems for those systems which require/allow for them. For example, + the ceph-osd upstart script requires the id parameter to be passed along + in order to identify which running daemon should be restarted. The follow- + ing example restarts the ceph-osd service for instance id=4: + + service_restart('ceph-osd', id=4) + + :param service_name: the name of the service to restart + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for init systems not allowing additional + parameters via the commandline (systemd). + """ return service('restart', service_name) -def service_reload(service_name, restart_on_failure=False): +def service_reload(service_name, restart_on_failure=False, **kwargs): """Reload a system service, optionally falling back to restart if - reload fails""" - service_result = service('reload', service_name) + reload fails. + + The specified service name is managed via the system level init system. + Some init systems (e.g. upstart) require that additional arguments be + provided in order to directly control service instances whereas other init + systems allow for addressing instances of a service directly by name (e.g. + systemd). + + The kwargs allow for the additional parameters to be passed to underlying + init systems for those systems which require/allow for them. For example, + the ceph-osd upstart script requires the id parameter to be passed along + in order to identify which running daemon should be reloaded. The follow- + ing example restarts the ceph-osd service for instance id=4: + + service_reload('ceph-osd', id=4) + + :param service_name: the name of the service to reload + :param restart_on_failure: boolean indicating whether to fallback to a + restart if the reload fails. + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for init systems not allowing additional + parameters via the commandline (systemd). + """ + service_result = service('reload', service_name, **kwargs) if not service_result and restart_on_failure: - service_result = service('restart', service_name) + service_result = service('restart', service_name, **kwargs) return service_result -def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"): +def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d", + **kwargs): """Pause a system service. - Stop it, and prevent it from starting again at boot.""" + Stop it, and prevent it from starting again at boot. + + :param service_name: the name of the service to pause + :param init_dir: path to the upstart init directory + :param initd_dir: path to the sysv init directory + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for init systems which do not support + key=value arguments via the commandline. + """ stopped = True - if service_running(service_name): - stopped = service_stop(service_name) + if service_running(service_name, **kwargs): + stopped = service_stop(service_name, **kwargs) upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) sysv_file = os.path.join(initd_dir, service_name) if init_is_systemd(): @@ -106,10 +206,19 @@ def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"): def service_resume(service_name, init_dir="/etc/init", - initd_dir="/etc/init.d"): + initd_dir="/etc/init.d", **kwargs): """Resume a system service. - Reenable starting again at boot. Start the service""" + Reenable starting again at boot. Start the service. + + :param service_name: the name of the service to resume + :param init_dir: the path to the init dir + :param initd dir: the path to the initd dir + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for systemd enabled systems. + """ upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) sysv_file = os.path.join(initd_dir, service_name) if init_is_systemd(): @@ -126,19 +235,28 @@ def service_resume(service_name, init_dir="/etc/init", "Unable to detect {0} as SystemD, Upstart {1} or" " SysV {2}".format( service_name, upstart_file, sysv_file)) + started = service_running(service_name, **kwargs) - started = service_running(service_name) if not started: - started = service_start(service_name) + started = service_start(service_name, **kwargs) return started -def service(action, service_name): - """Control a system service""" +def service(action, service_name, **kwargs): + """Control a system service. + + :param action: the action to take on the service + :param service_name: the name of the service to perform th action on + :param **kwargs: additional params to be passed to the service command in + the form of key=value. + """ if init_is_systemd(): cmd = ['systemctl', action, service_name] else: cmd = ['service', service_name, action] + for key, value in six.iteritems(kwargs): + parameter = '%s=%s' % (key, value) + cmd.append(parameter) return subprocess.call(cmd) == 0 @@ -146,15 +264,26 @@ _UPSTART_CONF = "/etc/init/{}.conf" _INIT_D_CONF = "/etc/init.d/{}" -def service_running(service_name): - """Determine whether a system service is running""" +def service_running(service_name, **kwargs): + """Determine whether a system service is running. + + :param service_name: the name of the service + :param **kwargs: additional args to pass to the service command. This is + used to pass additional key=value arguments to the + service command line for managing specific instance + units (e.g. service ceph-osd status id=2). The kwargs + are ignored in systemd services. + """ if init_is_systemd(): return service('is-active', service_name) else: if os.path.exists(_UPSTART_CONF.format(service_name)): try: - output = subprocess.check_output( - ['status', service_name], + cmd = ['status', service_name] + for key, value in six.iteritems(kwargs): + parameter = '%s=%s' % (key, value) + cmd.append(parameter) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8') except subprocess.CalledProcessError: return False @@ -306,15 +435,17 @@ def add_user_to_group(username, group): subprocess.check_call(cmd) -def rsync(from_path, to_path, flags='-r', options=None): +def rsync(from_path, to_path, flags='-r', options=None, timeout=None): """Replicate the contents of a path""" options = options or ['--delete', '--executability'] cmd = ['/usr/bin/rsync', flags] + if timeout: + cmd = ['timeout', str(timeout)] + cmd cmd.extend(options) cmd.append(from_path) cmd.append(to_path) log(" ".join(cmd)) - return subprocess.check_output(cmd).decode('UTF-8').strip() + return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8').strip() def symlink(source, destination): @@ -684,7 +815,7 @@ def chownr(path, owner, group, follow_links=True, chowntopdir=False): :param str path: The string path to start changing ownership. :param str owner: The owner string to use when looking up the uid. :param str group: The group string to use when looking up the gid. - :param bool follow_links: Also Chown links if True + :param bool follow_links: Also follow and chown links if True :param bool chowntopdir: Also chown path itself if True """ uid = pwd.getpwnam(owner).pw_uid @@ -698,7 +829,7 @@ def chownr(path, owner, group, follow_links=True, chowntopdir=False): broken_symlink = os.path.lexists(path) and not os.path.exists(path) if not broken_symlink: chown(path, uid, gid) - for root, dirs, files in os.walk(path): + for root, dirs, files in os.walk(path, followlinks=follow_links): for name in dirs + files: full = os.path.join(root, name) broken_symlink = os.path.lexists(full) and not os.path.exists(full) @@ -718,6 +849,20 @@ def lchownr(path, owner, group): chownr(path, owner, group, follow_links=False) +def owner(path): + """Returns a tuple containing the username & groupname owning the path. + + :param str path: the string path to retrieve the ownership + :return tuple(str, str): A (username, groupname) tuple containing the + name of the user and group owning the path. + :raises OSError: if the specified path does not exist + """ + stat = os.stat(path) + username = pwd.getpwuid(stat.st_uid)[0] + groupname = grp.getgrgid(stat.st_gid)[0] + return username, groupname + + def get_total_ram(): """The total amount of system RAM in bytes. @@ -749,3 +894,25 @@ def is_container(): else: # Detect using upstart container file marker return os.path.exists(UPSTART_CONTAINER_TYPE) + + +def add_to_updatedb_prunepath(path, updatedb_path=UPDATEDB_PATH): + with open(updatedb_path, 'r+') as f_id: + updatedb_text = f_id.read() + output = updatedb(updatedb_text, path) + f_id.seek(0) + f_id.write(output) + f_id.truncate() + + +def updatedb(updatedb_text, new_path): + lines = [line for line in updatedb_text.split("\n")] + for i, line in enumerate(lines): + if line.startswith("PRUNEPATHS="): + paths_line = line.split("=")[1].replace('"', '') + paths = paths_line.split(" ") + if new_path not in paths: + paths.append(new_path) + lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths)) + output = "\n".join(lines) + return output diff --git a/hooks/charmhelpers/osplatform.py b/hooks/charmhelpers/osplatform.py index ea490bb..d9a4d5c 100644 --- a/hooks/charmhelpers/osplatform.py +++ b/hooks/charmhelpers/osplatform.py @@ -8,12 +8,18 @@ def get_platform(): will be returned (which is the name of the module). This string is used to decide which platform module should be imported. """ + # linux_distribution is deprecated and will be removed in Python 3.7 + # Warings *not* disabled, as we certainly need to fix this. tuple_platform = platform.linux_distribution() current_platform = tuple_platform[0] if "Ubuntu" in current_platform: return "ubuntu" elif "CentOS" in current_platform: return "centos" + elif "debian" in current_platform: + # Stock Python does not detect Ubuntu and instead returns debian. + # Or at least it does in some build environments like Travis CI + return "ubuntu" else: raise RuntimeError("This module is not supported on {}." .format(current_platform)) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 2182260..c1490f9 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -201,6 +201,8 @@ class NovaBasicDeployment(OpenStackAmuletDeployment): 'neutron-api:neutron-plugin-api': 'neutron-gateway:' 'neutron-plugin-api', 'neutron-api:identity-service': 'keystone:identity-service', + 'nova-cloud-controller:quantum-network-service': + 'neutron-gateway:quantum-network-service', } super(NovaBasicDeployment, self)._add_relations(relations)