diff --git a/Makefile b/Makefile
index c57d2db..ab19a71 100644
--- a/Makefile
+++ b/Makefile
@@ -1,13 +1,29 @@
#!/usr/bin/make
PYTHON := /usr/bin/env python
+CHARM_DIR := $(PWD)
+HOOKS_DIR := $(PWD)/hooks
+TEST_PREFIX := PYTHONPATH=$(HOOKS_DIR)
clean:
- @rm -fv files/*
+ rm -f .coverage .testrepository
+ find . -name '*.pyc' -delete
+
+lint:
+ @tox -e pep8
bin/charm_helpers_sync.py:
@mkdir -p bin
@curl -o bin/charm_helpers_sync.py https://raw.githubusercontent.com/juju/charm-helpers/master/tools/charm_helpers_sync/charm_helpers_sync.py
+
sync: bin/charm_helpers_sync.py
@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-hooks.yaml
+ @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-tests.yaml
+test:
+ @echo Starting unit tests...
+ @tox -e py27
+
+functional_test:
+ @echo Starting amulet tests...
+ @tox -e func27
diff --git a/bin/charm_helpers_sync.py b/bin/charm_helpers_sync.py
index f67fdb9..e3f0e74 100644
--- a/bin/charm_helpers_sync.py
+++ b/bin/charm_helpers_sync.py
@@ -2,19 +2,17 @@
# Copyright 2014-2015 Canonical Limited.
#
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# charm-helpers is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# charm-helpers is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers. If not, see .
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
# Authors:
# Adam Gandelman
@@ -31,7 +29,7 @@ from fnmatch import fnmatch
import six
-CHARM_HELPERS_BRANCH = 'lp:charm-helpers'
+CHARM_HELPERS_REPO = 'https://github.com/juju/charm-helpers'
def parse_config(conf_file):
@@ -41,10 +39,16 @@ def parse_config(conf_file):
return yaml.load(open(conf_file).read())
-def clone_helpers(work_dir, branch):
+def clone_helpers(work_dir, repo):
dest = os.path.join(work_dir, 'charm-helpers')
- logging.info('Checking out %s to %s.' % (branch, dest))
- cmd = ['bzr', 'checkout', '--lightweight', branch, dest]
+ logging.info('Cloning out %s to %s.' % (repo, dest))
+ branch = None
+ if '@' in repo:
+ repo, branch = repo.split('@', 1)
+ cmd = ['git', 'clone', '--depth=1']
+ if branch is not None:
+ cmd += ['--branch', branch]
+ cmd += [repo, dest]
subprocess.check_call(cmd)
return dest
@@ -193,14 +197,15 @@ def sync_helpers(include, src, dest, options=None):
inc, opts = extract_options(m, global_options)
sync(src, dest, '%s.%s' % (k, inc), opts)
+
if __name__ == '__main__':
parser = optparse.OptionParser()
parser.add_option('-c', '--config', action='store', dest='config',
default=None, help='helper config file')
parser.add_option('-D', '--debug', action='store_true', dest='debug',
default=False, help='debug')
- parser.add_option('-b', '--branch', action='store', dest='branch',
- help='charm-helpers bzr branch (overrides config)')
+ parser.add_option('-r', '--repository', action='store', dest='repo',
+ help='charm-helpers git repository (overrides config)')
parser.add_option('-d', '--destination', action='store', dest='dest_dir',
help='sync destination dir (overrides config)')
(opts, args) = parser.parse_args()
@@ -219,10 +224,10 @@ if __name__ == '__main__':
else:
config = {}
- if 'branch' not in config:
- config['branch'] = CHARM_HELPERS_BRANCH
- if opts.branch:
- config['branch'] = opts.branch
+ if 'repo' not in config:
+ config['repo'] = CHARM_HELPERS_REPO
+ if opts.repo:
+ config['repo'] = opts.repo
if opts.dest_dir:
config['destination'] = opts.dest_dir
@@ -242,7 +247,7 @@ if __name__ == '__main__':
sync_options = config['options']
tmpd = tempfile.mkdtemp()
try:
- checkout = clone_helpers(tmpd, config['branch'])
+ checkout = clone_helpers(tmpd, config['repo'])
sync_helpers(config['include'], checkout, config['destination'],
options=sync_options)
except Exception as e:
diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml
new file mode 100644
index 0000000..cabe7b0
--- /dev/null
+++ b/charm-helpers-tests.yaml
@@ -0,0 +1,13 @@
+repo: https://github.com/juju/charm-helpers
+destination: tests/charmhelpers
+include:
+ - fetch
+ - core
+ - contrib.amulet
+ - contrib.openstack.amulet
+ - contrib.openstack.utils
+ - contrib.openstack.exceptions
+ - contrib.network.ip
+ - contrib.storage|inc=*
+ - contrib.python|inc=*
+ - osplatform
diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py
index 80d574d..1c55b30 100644
--- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py
+++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py
@@ -30,6 +30,7 @@ import yaml
from charmhelpers.core.hookenv import (
config,
+ hook_name,
local_unit,
log,
relation_ids,
@@ -285,7 +286,7 @@ class NRPE(object):
try:
nagios_uid = pwd.getpwnam('nagios').pw_uid
nagios_gid = grp.getgrnam('nagios').gr_gid
- except:
+ except Exception:
log("Nagios user not set up, nrpe checks not updated")
return
@@ -302,7 +303,12 @@ class NRPE(object):
"command": nrpecheck.command,
}
- service('restart', 'nagios-nrpe-server')
+ # update-status hooks are configured to firing every 5 minutes by
+ # default. When nagios-nrpe-server is restarted, the nagios server
+ # reports checks failing causing unneccessary alerts. Let's not restart
+ # on update-status hooks.
+ if not hook_name() == 'update-status':
+ service('restart', 'nagios-nrpe-server')
monitor_ids = relation_ids("local-monitors") + \
relation_ids("nrpe-external-master")
diff --git a/hooks/charmhelpers/contrib/hahelpers/apache.py b/hooks/charmhelpers/contrib/hahelpers/apache.py
index d0c6994..22acb68 100644
--- a/hooks/charmhelpers/contrib/hahelpers/apache.py
+++ b/hooks/charmhelpers/contrib/hahelpers/apache.py
@@ -90,6 +90,6 @@ def install_ca_cert(ca_cert):
log("CA cert is the same as installed version", level=INFO)
else:
log("Installing new CA cert", level=INFO)
- with open(cert_file, 'w') as crt:
+ with open(cert_file, 'wb') as crt:
crt.write(ca_cert)
subprocess.check_call(['update-ca-certificates', '--fresh'])
diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py
index e02350e..4207e42 100644
--- a/hooks/charmhelpers/contrib/hahelpers/cluster.py
+++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py
@@ -27,6 +27,7 @@ clustering-related helpers.
import subprocess
import os
+import time
from socket import gethostname as get_unit_hostname
@@ -45,6 +46,9 @@ from charmhelpers.core.hookenv import (
is_leader as juju_is_leader,
status_set,
)
+from charmhelpers.core.host import (
+ modulo_distribution,
+)
from charmhelpers.core.decorators import (
retry_on_exception,
)
@@ -361,3 +365,29 @@ def canonical_url(configs, vip_setting='vip'):
else:
addr = unit_get('private-address')
return '%s://%s' % (scheme, addr)
+
+
+def distributed_wait(modulo=None, wait=None, operation_name='operation'):
+ ''' Distribute operations by waiting based on modulo_distribution
+
+ If modulo and or wait are not set, check config_get for those values.
+
+ :param modulo: int The modulo number creates the group distribution
+ :param wait: int The constant time wait value
+ :param operation_name: string Operation name for status message
+ i.e. 'restart'
+ :side effect: Calls config_get()
+ :side effect: Calls log()
+ :side effect: Calls status_set()
+ :side effect: Calls time.sleep()
+ '''
+ if modulo is None:
+ modulo = config_get('modulo-nodes')
+ if wait is None:
+ wait = config_get('known-wait')
+ calculated_wait = modulo_distribution(modulo=modulo, wait=wait)
+ msg = "Waiting {} seconds for {} ...".format(calculated_wait,
+ operation_name)
+ log(msg, DEBUG)
+ status_set('maintenance', msg)
+ time.sleep(calculated_wait)
diff --git a/hooks/charmhelpers/contrib/hardening/audits/apache.py b/hooks/charmhelpers/contrib/hardening/audits/apache.py
index d812948..d32bf44 100644
--- a/hooks/charmhelpers/contrib/hardening/audits/apache.py
+++ b/hooks/charmhelpers/contrib/hardening/audits/apache.py
@@ -70,12 +70,12 @@ class DisabledModuleAudit(BaseAudit):
"""Returns the modules which are enabled in Apache."""
output = subprocess.check_output(['apache2ctl', '-M'])
modules = []
- for line in output.strip().split():
+ for line in output.splitlines():
# Each line of the enabled module output looks like:
# module_name (static|shared)
# Plus a header line at the top of the output which is stripped
# out by the regex.
- matcher = re.search(r'^ (\S*)', line)
+ matcher = re.search(r'^ (\S*)_module (\S*)', line)
if matcher:
modules.append(matcher.group(1))
return modules
diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py
index d7e6deb..a871ce3 100644
--- a/hooks/charmhelpers/contrib/network/ip.py
+++ b/hooks/charmhelpers/contrib/network/ip.py
@@ -490,7 +490,7 @@ def get_host_ip(hostname, fallback=None):
if not ip_addr:
try:
ip_addr = socket.gethostbyname(hostname)
- except:
+ except Exception:
log("Failed to resolve hostname '%s'" % (hostname),
level=WARNING)
return fallback
@@ -518,7 +518,7 @@ def get_hostname(address, fqdn=True):
if not result:
try:
result = socket.gethostbyaddr(address)[0]
- except:
+ except Exception:
return None
else:
result = address
diff --git a/hooks/charmhelpers/contrib/openstack/alternatives.py b/hooks/charmhelpers/contrib/openstack/alternatives.py
index 1501641..547de09 100644
--- a/hooks/charmhelpers/contrib/openstack/alternatives.py
+++ b/hooks/charmhelpers/contrib/openstack/alternatives.py
@@ -29,3 +29,16 @@ def install_alternative(name, target, source, priority=50):
target, name, source, str(priority)
]
subprocess.check_call(cmd)
+
+
+def remove_alternative(name, source):
+ """Remove an installed alternative configuration file
+
+ :param name: string name of the alternative to remove
+ :param source: string full path to alternative to remove
+ """
+ cmd = [
+ 'update-alternatives', '--remove',
+ name, source
+ ]
+ subprocess.check_call(cmd)
diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
index 5c041d2..5afbbd8 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -13,6 +13,7 @@
# limitations under the License.
import logging
+import os
import re
import sys
import six
@@ -185,7 +186,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
self.d.configure(service, config)
def _auto_wait_for_status(self, message=None, exclude_services=None,
- include_only=None, timeout=1800):
+ include_only=None, timeout=None):
"""Wait for all units to have a specific extended status, except
for any defined as excluded. Unless specified via message, any
status containing any case of 'ready' will be considered a match.
@@ -215,7 +216,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
:param timeout: Maximum time in seconds to wait for status match
:returns: None. Raises if timeout is hit.
"""
- self.log.info('Waiting for extended status on units...')
+ if not timeout:
+ timeout = int(os.environ.get('AMULET_SETUP_TIMEOUT', 1800))
+ self.log.info('Waiting for extended status on units for {}s...'
+ ''.format(timeout))
all_services = self.d.services.keys()
@@ -250,7 +254,14 @@ class OpenStackAmuletDeployment(AmuletDeployment):
self.log.debug('Waiting up to {}s for extended status on services: '
'{}'.format(timeout, services))
service_messages = {service: message for service in services}
+
+ # Check for idleness
+ self.d.sentry.wait(timeout=timeout)
+ # Check for error states and bail early
+ self.d.sentry.wait_for_status(self.d.juju_env, services, timeout=timeout)
+ # Check for ready messages
self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+
self.log.info('OK')
def _get_openstack_release(self):
@@ -263,7 +274,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
(self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty,
self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton,
self.yakkety_newton, self.xenial_ocata, self.zesty_ocata,
- self.xenial_pike, self.artful_pike) = range(11)
+ self.xenial_pike, self.artful_pike, self.xenial_queens,
+ self.bionic_queens,) = range(13)
releases = {
('trusty', None): self.trusty_icehouse,
@@ -274,9 +286,11 @@ class OpenStackAmuletDeployment(AmuletDeployment):
('xenial', 'cloud:xenial-newton'): self.xenial_newton,
('xenial', 'cloud:xenial-ocata'): self.xenial_ocata,
('xenial', 'cloud:xenial-pike'): self.xenial_pike,
+ ('xenial', 'cloud:xenial-queens'): self.xenial_queens,
('yakkety', None): self.yakkety_newton,
('zesty', None): self.zesty_ocata,
('artful', None): self.artful_pike,
+ ('bionic', None): self.bionic_queens,
}
return releases[(self.series, self.openstack)]
@@ -291,6 +305,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
('yakkety', 'newton'),
('zesty', 'ocata'),
('artful', 'pike'),
+ ('bionic', 'queens'),
])
if self.openstack:
os_origin = self.openstack.split(':')[1]
@@ -303,20 +318,27 @@ class OpenStackAmuletDeployment(AmuletDeployment):
test scenario, based on OpenStack release and whether ceph radosgw
is flagged as present or not."""
- if self._get_openstack_release() >= self.trusty_kilo:
- # Kilo or later
- pools = [
- 'rbd',
- 'cinder',
- 'glance'
- ]
- else:
- # Juno or earlier
+ if self._get_openstack_release() == self.trusty_icehouse:
+ # Icehouse
pools = [
'data',
'metadata',
'rbd',
- 'cinder',
+ 'cinder-ceph',
+ 'glance'
+ ]
+ elif (self.trusty_kilo <= self._get_openstack_release() <=
+ self.zesty_ocata):
+ # Kilo through Ocata
+ pools = [
+ 'rbd',
+ 'cinder-ceph',
+ 'glance'
+ ]
+ else:
+ # Pike and later
+ pools = [
+ 'cinder-ceph',
'glance'
]
diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py
index c8edbf6..b71b2b1 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py
@@ -23,6 +23,7 @@ import urllib
import urlparse
import cinderclient.v1.client as cinder_client
+import cinderclient.v2.client as cinder_clientv2
import glanceclient.v1.client as glance_client
import heatclient.v1.client as heat_client
from keystoneclient.v2_0 import client as keystone_client
@@ -42,7 +43,6 @@ import swiftclient
from charmhelpers.contrib.amulet.utils import (
AmuletUtils
)
-from charmhelpers.core.decorators import retry_on_exception
from charmhelpers.core.host import CompareHostReleases
DEBUG = logging.DEBUG
@@ -310,7 +310,6 @@ class OpenStackAmuletUtils(AmuletUtils):
self.log.debug('Checking if tenant exists ({})...'.format(tenant))
return tenant in [t.name for t in keystone.tenants.list()]
- @retry_on_exception(5, base_delay=10)
def keystone_wait_for_propagation(self, sentry_relation_pairs,
api_version):
"""Iterate over list of sentry and relation tuples and verify that
@@ -326,7 +325,7 @@ class OpenStackAmuletUtils(AmuletUtils):
rel = sentry.relation('identity-service',
relation_name)
self.log.debug('keystone relation data: {}'.format(rel))
- if rel['api_version'] != str(api_version):
+ if rel.get('api_version') != str(api_version):
raise Exception("api_version not propagated through relation"
" data yet ('{}' != '{}')."
"".format(rel['api_version'], api_version))
@@ -348,15 +347,19 @@ class OpenStackAmuletUtils(AmuletUtils):
config = {'preferred-api-version': api_version}
deployment.d.configure('keystone', config)
+ deployment._auto_wait_for_status()
self.keystone_wait_for_propagation(sentry_relation_pairs, api_version)
def authenticate_cinder_admin(self, keystone_sentry, username,
- password, tenant):
+ password, tenant, api_version=2):
"""Authenticates admin user with cinder."""
# NOTE(beisner): cinder python client doesn't accept tokens.
keystone_ip = keystone_sentry.info['public-address']
ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8'))
- return cinder_client.Client(username, password, tenant, ept)
+ _clients = {
+ 1: cinder_client.Client,
+ 2: cinder_clientv2.Client}
+ return _clients[api_version](username, password, tenant, ept)
def authenticate_keystone(self, keystone_ip, username, password,
api_version=False, admin_port=False,
@@ -617,13 +620,25 @@ class OpenStackAmuletUtils(AmuletUtils):
self.log.debug('Keypair ({}) already exists, '
'using it.'.format(keypair_name))
return _keypair
- except:
+ except Exception:
self.log.debug('Keypair ({}) does not exist, '
'creating it.'.format(keypair_name))
_keypair = nova.keypairs.create(name=keypair_name)
return _keypair
+ def _get_cinder_obj_name(self, cinder_object):
+ """Retrieve name of cinder object.
+
+ :param cinder_object: cinder snapshot or volume object
+ :returns: str cinder object name
+ """
+ # v1 objects store name in 'display_name' attr but v2+ use 'name'
+ try:
+ return cinder_object.display_name
+ except AttributeError:
+ return cinder_object.name
+
def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1,
img_id=None, src_vol_id=None, snap_id=None):
"""Create cinder volume, optionally from a glance image, OR
@@ -674,6 +689,13 @@ class OpenStackAmuletUtils(AmuletUtils):
source_volid=src_vol_id,
snapshot_id=snap_id)
vol_id = vol_new.id
+ except TypeError:
+ vol_new = cinder.volumes.create(name=vol_name,
+ imageRef=img_id,
+ size=vol_size,
+ source_volid=src_vol_id,
+ snapshot_id=snap_id)
+ vol_id = vol_new.id
except Exception as e:
msg = 'Failed to create volume: {}'.format(e)
amulet.raise_status(amulet.FAIL, msg=msg)
@@ -688,7 +710,7 @@ class OpenStackAmuletUtils(AmuletUtils):
# Re-validate new volume
self.log.debug('Validating volume attributes...')
- val_vol_name = cinder.volumes.get(vol_id).display_name
+ val_vol_name = self._get_cinder_obj_name(cinder.volumes.get(vol_id))
val_vol_boot = cinder.volumes.get(vol_id).bootable
val_vol_stat = cinder.volumes.get(vol_id).status
val_vol_size = cinder.volumes.get(vol_id).size
diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py
index f67f326..e6c0e9f 100644
--- a/hooks/charmhelpers/contrib/openstack/context.py
+++ b/hooks/charmhelpers/contrib/openstack/context.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import collections
import glob
import json
import math
@@ -292,7 +293,7 @@ class PostgresqlDBContext(OSContextGenerator):
def db_ssl(rdata, ctxt, ssl_dir):
if 'ssl_ca' in rdata and ssl_dir:
ca_path = os.path.join(ssl_dir, 'db-client.ca')
- with open(ca_path, 'w') as fh:
+ with open(ca_path, 'wb') as fh:
fh.write(b64decode(rdata['ssl_ca']))
ctxt['database_ssl_ca'] = ca_path
@@ -307,12 +308,12 @@ def db_ssl(rdata, ctxt, ssl_dir):
log("Waiting 1m for ssl client cert validity", level=INFO)
time.sleep(60)
- with open(cert_path, 'w') as fh:
+ with open(cert_path, 'wb') as fh:
fh.write(b64decode(rdata['ssl_cert']))
ctxt['database_ssl_cert'] = cert_path
key_path = os.path.join(ssl_dir, 'db-client.key')
- with open(key_path, 'w') as fh:
+ with open(key_path, 'wb') as fh:
fh.write(b64decode(rdata['ssl_key']))
ctxt['database_ssl_key'] = key_path
@@ -458,7 +459,7 @@ class AMQPContext(OSContextGenerator):
ca_path = os.path.join(
self.ssl_dir, 'rabbit-client-ca.pem')
- with open(ca_path, 'w') as fh:
+ with open(ca_path, 'wb') as fh:
fh.write(b64decode(ctxt['rabbit_ssl_ca']))
ctxt['rabbit_ssl_ca'] = ca_path
@@ -578,11 +579,14 @@ class HAProxyContext(OSContextGenerator):
laddr = get_address_in_network(config(cfg_opt))
if laddr:
netmask = get_netmask_for_address(laddr)
- cluster_hosts[laddr] = {'network': "{}/{}".format(laddr,
- netmask),
- 'backends': {l_unit: laddr}}
+ cluster_hosts[laddr] = {
+ 'network': "{}/{}".format(laddr,
+ netmask),
+ 'backends': collections.OrderedDict([(l_unit,
+ laddr)])
+ }
for rid in relation_ids('cluster'):
- for unit in related_units(rid):
+ for unit in sorted(related_units(rid)):
_laddr = relation_get('{}-address'.format(addr_type),
rid=rid, unit=unit)
if _laddr:
@@ -594,10 +598,13 @@ class HAProxyContext(OSContextGenerator):
# match in the frontend
cluster_hosts[addr] = {}
netmask = get_netmask_for_address(addr)
- cluster_hosts[addr] = {'network': "{}/{}".format(addr, netmask),
- 'backends': {l_unit: addr}}
+ cluster_hosts[addr] = {
+ 'network': "{}/{}".format(addr, netmask),
+ 'backends': collections.OrderedDict([(l_unit,
+ addr)])
+ }
for rid in relation_ids('cluster'):
- for unit in related_units(rid):
+ for unit in sorted(related_units(rid)):
_laddr = relation_get('private-address',
rid=rid, unit=unit)
if _laddr:
@@ -628,6 +635,8 @@ class HAProxyContext(OSContextGenerator):
ctxt['local_host'] = '127.0.0.1'
ctxt['haproxy_host'] = '0.0.0.0'
+ ctxt['ipv6_enabled'] = not is_ipv6_disabled()
+
ctxt['stat_port'] = '8888'
db = kv()
@@ -802,8 +811,9 @@ class ApacheSSLContext(OSContextGenerator):
else:
# Expect cert/key provided in config (currently assumed that ca
# uses ip for cn)
- cn = resolve_address(endpoint_type=INTERNAL)
- self.configure_cert(cn)
+ for net_type in (INTERNAL, ADMIN, PUBLIC):
+ cn = resolve_address(endpoint_type=net_type)
+ self.configure_cert(cn)
addresses = self.get_network_addresses()
for address, endpoint in addresses:
@@ -843,15 +853,6 @@ class NeutronContext(OSContextGenerator):
for pkgs in self.packages:
ensure_packages(pkgs)
- def _save_flag_file(self):
- if self.network_manager == 'quantum':
- _file = '/etc/nova/quantum_plugin.conf'
- else:
- _file = '/etc/nova/neutron_plugin.conf'
-
- with open(_file, 'wb') as out:
- out.write(self.plugin + '\n')
-
def ovs_ctxt(self):
driver = neutron_plugin_attribute(self.plugin, 'driver',
self.network_manager)
@@ -996,7 +997,6 @@ class NeutronContext(OSContextGenerator):
flags = config_flags_parser(alchemy_flags)
ctxt['neutron_alchemy_flags'] = flags
- self._save_flag_file()
return ctxt
@@ -1176,7 +1176,7 @@ class SubordinateConfigContext(OSContextGenerator):
if sub_config and sub_config != '':
try:
sub_config = json.loads(sub_config)
- except:
+ except Exception:
log('Could not parse JSON from '
'subordinate_configuration setting from %s'
% rid, level=ERROR)
diff --git a/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh b/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh
index 0df0717..7aab129 100755
--- a/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh
+++ b/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh
@@ -9,7 +9,7 @@
CRITICAL=0
NOTACTIVE=''
LOGFILE=/var/log/nagios/check_haproxy.log
-AUTH=$(grep -r "stats auth" /etc/haproxy | awk 'NR=1{print $4}')
+AUTH=$(grep -r "stats auth" /etc/haproxy/haproxy.cfg | awk 'NR=1{print $4}')
typeset -i N_INSTANCES=0
for appserver in $(awk '/^\s+server/{print $2}' /etc/haproxy/haproxy.cfg)
diff --git a/hooks/charmhelpers/contrib/openstack/ha/utils.py b/hooks/charmhelpers/contrib/openstack/ha/utils.py
index 254a90e..9a4d79c 100644
--- a/hooks/charmhelpers/contrib/openstack/ha/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/ha/utils.py
@@ -82,15 +82,18 @@ def update_dns_ha_resource_params(resources, resource_params,
continue
m = re.search('os-(.+?)-hostname', setting)
if m:
- networkspace = m.group(1)
+ endpoint_type = m.group(1)
+ # resolve_address's ADDRESS_MAP uses 'int' not 'internal'
+ if endpoint_type == 'internal':
+ endpoint_type = 'int'
else:
msg = ('Unexpected DNS hostname setting: {}. '
- 'Cannot determine network space name'
+ 'Cannot determine endpoint_type name'
''.format(setting))
status_set('blocked', msg)
raise DNSHAException(msg)
- hostname_key = 'res_{}_{}_hostname'.format(charm_name(), networkspace)
+ hostname_key = 'res_{}_{}_hostname'.format(charm_name(), endpoint_type)
if hostname_key in hostname_group:
log('DNS HA: Resource {}: {} already exists in '
'hostname group - skipping'.format(hostname_key, hostname),
@@ -101,7 +104,7 @@ def update_dns_ha_resource_params(resources, resource_params,
resources[hostname_key] = crm_ocf
resource_params[hostname_key] = (
'params fqdn="{}" ip_address="{}" '
- ''.format(hostname, resolve_address(endpoint_type=networkspace,
+ ''.format(hostname, resolve_address(endpoint_type=endpoint_type,
override=False)))
if len(hostname_group) >= 1:
diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py
index 37fa0eb..0f847f5 100644
--- a/hooks/charmhelpers/contrib/openstack/neutron.py
+++ b/hooks/charmhelpers/contrib/openstack/neutron.py
@@ -59,18 +59,13 @@ def determine_dkms_package():
def quantum_plugins():
- from charmhelpers.contrib.openstack import context
return {
'ovs': {
'config': '/etc/quantum/plugins/openvswitch/'
'ovs_quantum_plugin.ini',
'driver': 'quantum.plugins.openvswitch.ovs_quantum_plugin.'
'OVSQuantumPluginV2',
- 'contexts': [
- context.SharedDBContext(user=config('neutron-database-user'),
- database=config('neutron-database'),
- relation_prefix='neutron',
- ssl_dir=QUANTUM_CONF_DIR)],
+ 'contexts': [],
'services': ['quantum-plugin-openvswitch-agent'],
'packages': [determine_dkms_package(),
['quantum-plugin-openvswitch-agent']],
@@ -82,11 +77,7 @@ def quantum_plugins():
'config': '/etc/quantum/plugins/nicira/nvp.ini',
'driver': 'quantum.plugins.nicira.nicira_nvp_plugin.'
'QuantumPlugin.NvpPluginV2',
- 'contexts': [
- context.SharedDBContext(user=config('neutron-database-user'),
- database=config('neutron-database'),
- relation_prefix='neutron',
- ssl_dir=QUANTUM_CONF_DIR)],
+ 'contexts': [],
'services': [],
'packages': [],
'server_packages': ['quantum-server',
@@ -100,7 +91,6 @@ NEUTRON_CONF_DIR = '/etc/neutron'
def neutron_plugins():
- from charmhelpers.contrib.openstack import context
release = os_release('nova-common')
plugins = {
'ovs': {
@@ -108,11 +98,7 @@ def neutron_plugins():
'ovs_neutron_plugin.ini',
'driver': 'neutron.plugins.openvswitch.ovs_neutron_plugin.'
'OVSNeutronPluginV2',
- 'contexts': [
- context.SharedDBContext(user=config('neutron-database-user'),
- database=config('neutron-database'),
- relation_prefix='neutron',
- ssl_dir=NEUTRON_CONF_DIR)],
+ 'contexts': [],
'services': ['neutron-plugin-openvswitch-agent'],
'packages': [determine_dkms_package(),
['neutron-plugin-openvswitch-agent']],
@@ -124,11 +110,7 @@ def neutron_plugins():
'config': '/etc/neutron/plugins/nicira/nvp.ini',
'driver': 'neutron.plugins.nicira.nicira_nvp_plugin.'
'NeutronPlugin.NvpPluginV2',
- 'contexts': [
- context.SharedDBContext(user=config('neutron-database-user'),
- database=config('neutron-database'),
- relation_prefix='neutron',
- ssl_dir=NEUTRON_CONF_DIR)],
+ 'contexts': [],
'services': [],
'packages': [],
'server_packages': ['neutron-server',
@@ -138,11 +120,7 @@ def neutron_plugins():
'nsx': {
'config': '/etc/neutron/plugins/vmware/nsx.ini',
'driver': 'vmware',
- 'contexts': [
- context.SharedDBContext(user=config('neutron-database-user'),
- database=config('neutron-database'),
- relation_prefix='neutron',
- ssl_dir=NEUTRON_CONF_DIR)],
+ 'contexts': [],
'services': [],
'packages': [],
'server_packages': ['neutron-server',
@@ -152,11 +130,7 @@ def neutron_plugins():
'n1kv': {
'config': '/etc/neutron/plugins/cisco/cisco_plugins.ini',
'driver': 'neutron.plugins.cisco.network_plugin.PluginV2',
- 'contexts': [
- context.SharedDBContext(user=config('neutron-database-user'),
- database=config('neutron-database'),
- relation_prefix='neutron',
- ssl_dir=NEUTRON_CONF_DIR)],
+ 'contexts': [],
'services': [],
'packages': [determine_dkms_package(),
['neutron-plugin-cisco']],
@@ -167,11 +141,7 @@ def neutron_plugins():
'Calico': {
'config': '/etc/neutron/plugins/ml2/ml2_conf.ini',
'driver': 'neutron.plugins.ml2.plugin.Ml2Plugin',
- 'contexts': [
- context.SharedDBContext(user=config('neutron-database-user'),
- database=config('neutron-database'),
- relation_prefix='neutron',
- ssl_dir=NEUTRON_CONF_DIR)],
+ 'contexts': [],
'services': ['calico-felix',
'bird',
'neutron-dhcp-agent',
@@ -189,11 +159,7 @@ def neutron_plugins():
'vsp': {
'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini',
'driver': 'neutron.plugins.nuage.plugin.NuagePlugin',
- 'contexts': [
- context.SharedDBContext(user=config('neutron-database-user'),
- database=config('neutron-database'),
- relation_prefix='neutron',
- ssl_dir=NEUTRON_CONF_DIR)],
+ 'contexts': [],
'services': [],
'packages': [],
'server_packages': ['neutron-server', 'neutron-plugin-nuage'],
@@ -203,10 +169,7 @@ def neutron_plugins():
'config': '/etc/neutron/plugins/plumgrid/plumgrid.ini',
'driver': ('neutron.plugins.plumgrid.plumgrid_plugin'
'.plumgrid_plugin.NeutronPluginPLUMgridV2'),
- 'contexts': [
- context.SharedDBContext(user=config('database-user'),
- database=config('database'),
- ssl_dir=NEUTRON_CONF_DIR)],
+ 'contexts': [],
'services': [],
'packages': ['plumgrid-lxc',
'iovisor-dkms'],
@@ -217,11 +180,7 @@ def neutron_plugins():
'midonet': {
'config': '/etc/neutron/plugins/midonet/midonet.ini',
'driver': 'midonet.neutron.plugin.MidonetPluginV2',
- 'contexts': [
- context.SharedDBContext(user=config('neutron-database-user'),
- database=config('neutron-database'),
- relation_prefix='neutron',
- ssl_dir=NEUTRON_CONF_DIR)],
+ 'contexts': [],
'services': [],
'packages': [determine_dkms_package()],
'server_packages': ['neutron-server',
diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf
index ed5c4f1..a11ce8a 100644
--- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf
+++ b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf
@@ -18,7 +18,7 @@ rbd default features = {{ rbd_features }}
[client]
{% if rbd_client_cache_settings -%}
-{% for key, value in rbd_client_cache_settings.iteritems() -%}
+{% for key, value in rbd_client_cache_settings.items() -%}
{{ key }} = {{ value }}
{% endfor -%}
-{%- endif %}
\ No newline at end of file
+{%- endif %}
diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg
index 2e66045..ebc8a68 100644
--- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg
+++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg
@@ -48,7 +48,9 @@ listen stats
{% for service, ports in service_ports.items() -%}
frontend tcp-in_{{ service }}
bind *:{{ ports[0] }}
+ {% if ipv6_enabled -%}
bind :::{{ ports[0] }}
+ {% endif -%}
{% for frontend in frontends -%}
acl net_{{ frontend }} dst {{ frontends[frontend]['network'] }}
use_backend {{ service }}_{{ frontend }} if net_{{ frontend }}
diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-oslo-cache b/hooks/charmhelpers/contrib/openstack/templates/section-oslo-cache
new file mode 100644
index 0000000..e056a32
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/templates/section-oslo-cache
@@ -0,0 +1,6 @@
+[cache]
+{% if memcache_url %}
+enabled = true
+backend = oslo_cache.memcache_pool
+memcache_servers = {{ memcache_url }}
+{% endif %}
diff --git a/hooks/charmhelpers/contrib/openstack/templating.py b/hooks/charmhelpers/contrib/openstack/templating.py
index d8c1fc7..77490e4 100644
--- a/hooks/charmhelpers/contrib/openstack/templating.py
+++ b/hooks/charmhelpers/contrib/openstack/templating.py
@@ -272,6 +272,8 @@ class OSConfigRenderer(object):
raise OSConfigException
_out = self.render(config_file)
+ if six.PY3:
+ _out = _out.encode('UTF-8')
with open(config_file, 'wb') as out:
out.write(_out)
diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index 837a167..8a541d4 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -95,7 +95,7 @@ from charmhelpers.fetch import (
from charmhelpers.fetch.snap import (
snap_install,
snap_refresh,
- SNAP_CHANNELS,
+ valid_snap_channel,
)
from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
@@ -140,6 +140,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
('yakkety', 'newton'),
('zesty', 'ocata'),
('artful', 'pike'),
+ ('bionic', 'queens'),
])
@@ -157,6 +158,7 @@ OPENSTACK_CODENAMES = OrderedDict([
('2016.2', 'newton'),
('2017.1', 'ocata'),
('2017.2', 'pike'),
+ ('2018.1', 'queens'),
])
# The ugly duckling - must list releases oldest to newest
@@ -187,6 +189,8 @@ SWIFT_CODENAMES = OrderedDict([
['2.11.0', '2.12.0', '2.13.0']),
('pike',
['2.13.0', '2.15.0']),
+ ('queens',
+ ['2.16.0']),
])
# >= Liberty version->codename mapping
@@ -412,6 +416,8 @@ def get_os_codename_package(package, fatal=True):
cmd = ['snap', 'list', package]
try:
out = subprocess.check_output(cmd)
+ if six.PY3:
+ out = out.decode('UTF-8')
except subprocess.CalledProcessError as e:
return None
lines = out.split('\n')
@@ -426,7 +432,7 @@ def get_os_codename_package(package, fatal=True):
try:
pkg = cache[package]
- except:
+ except Exception:
if not fatal:
return None
# the package is unknown to the current apt cache.
@@ -579,6 +585,9 @@ def configure_installation_source(source_plus_key):
Note that the behaviour on error is to log the error to the juju log and
then call sys.exit(1).
"""
+ if source_plus_key.startswith('snap'):
+ # Do nothing for snap installs
+ return
# extract the key if there is one, denoted by a '|' in the rel
source, key = get_source_and_pgp_key(source_plus_key)
@@ -615,7 +624,7 @@ def save_script_rc(script_path="scripts/scriptrc", **env_vars):
juju_rc_path = "%s/%s" % (charm_dir(), script_path)
if not os.path.exists(os.path.dirname(juju_rc_path)):
os.mkdir(os.path.dirname(juju_rc_path))
- with open(juju_rc_path, 'wb') as rc_script:
+ with open(juju_rc_path, 'wt') as rc_script:
rc_script.write(
"#!/bin/bash\n")
[rc_script.write('export %s=%s\n' % (u, p))
@@ -794,7 +803,7 @@ def git_default_repos(projects_yaml):
service = service_name()
core_project = service
- for default, branch in GIT_DEFAULT_BRANCHES.iteritems():
+ for default, branch in six.iteritems(GIT_DEFAULT_BRANCHES):
if projects_yaml == default:
# add the requirements repo first
@@ -1615,7 +1624,7 @@ def do_action_openstack_upgrade(package, upgrade_callback, configs):
upgrade_callback(configs=configs)
action_set({'outcome': 'success, upgrade completed.'})
ret = True
- except:
+ except Exception:
action_set({'outcome': 'upgrade failed, see traceback.'})
action_set({'traceback': traceback.format_exc()})
action_fail('do_openstack_upgrade resulted in an '
@@ -1720,7 +1729,7 @@ def is_unit_paused_set():
kv = t[0]
# transform something truth-y into a Boolean.
return not(not(kv.get('unit-paused')))
- except:
+ except Exception:
return False
@@ -2048,7 +2057,7 @@ def update_json_file(filename, items):
def snap_install_requested():
""" Determine if installing from snaps
- If openstack-origin is of the form snap:channel-series-release
+ If openstack-origin is of the form snap:track/channel[/branch]
and channel is in SNAPS_CHANNELS return True.
"""
origin = config('openstack-origin') or ""
@@ -2056,10 +2065,12 @@ def snap_install_requested():
return False
_src = origin[5:]
- channel, series, release = _src.split('-')
- if channel.lower() in SNAP_CHANNELS:
- return True
- return False
+ if '/' in _src:
+ channel = _src.split('/')[1]
+ else:
+ # Handle snap:track with no channel
+ channel = 'stable'
+ return valid_snap_channel(channel)
def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
@@ -2067,7 +2078,7 @@ def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
@param snaps: List of snaps
@param src: String of openstack-origin or source of the form
- snap:channel-series-track
+ snap:track/channel
@param mode: String classic, devmode or jailmode
@returns: Dictionary of snaps with channels and modes
"""
@@ -2077,8 +2088,7 @@ def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
return {}
_src = src[5:]
- _channel, _series, _release = _src.split('-')
- channel = '--channel={}/{}'.format(_release, _channel)
+ channel = '--channel={}'.format(_src)
return {snap: {'channel': channel, 'mode': mode}
for snap in snaps}
@@ -2090,8 +2100,8 @@ def install_os_snaps(snaps, refresh=False):
@param snaps: Dictionary of snaps with channels and modes of the form:
{'snap_name': {'channel': 'snap_channel',
'mode': 'snap_mode'}}
- Where channel a snapstore channel and mode is --classic, --devmode or
- --jailmode.
+ Where channel is a snapstore channel and mode is --classic, --devmode
+ or --jailmode.
@param post_snap_install: Callback function to run after snaps have been
installed
"""
diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py
index e5a01b1..3923161 100644
--- a/hooks/charmhelpers/contrib/storage/linux/ceph.py
+++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py
@@ -370,9 +370,10 @@ def get_mon_map(service):
Also raises CalledProcessError if our ceph command fails
"""
try:
- mon_status = check_output(
- ['ceph', '--id', service,
- 'mon_status', '--format=json'])
+ mon_status = check_output(['ceph', '--id', service,
+ 'mon_status', '--format=json'])
+ if six.PY3:
+ mon_status = mon_status.decode('UTF-8')
try:
return json.loads(mon_status)
except ValueError as v:
@@ -457,7 +458,7 @@ def monitor_key_get(service, key):
try:
output = check_output(
['ceph', '--id', service,
- 'config-key', 'get', str(key)])
+ 'config-key', 'get', str(key)]).decode('UTF-8')
return output
except CalledProcessError as e:
log("Monitor config-key get failed with message: {}".format(
@@ -500,6 +501,8 @@ def get_erasure_profile(service, name):
out = check_output(['ceph', '--id', service,
'osd', 'erasure-code-profile', 'get',
name, '--format=json'])
+ if six.PY3:
+ out = out.decode('UTF-8')
return json.loads(out)
except (CalledProcessError, OSError, ValueError):
return None
@@ -686,7 +689,10 @@ def get_cache_mode(service, pool_name):
"""
validator(value=service, valid_type=six.string_types)
validator(value=pool_name, valid_type=six.string_types)
- out = check_output(['ceph', '--id', service, 'osd', 'dump', '--format=json'])
+ out = check_output(['ceph', '--id', service,
+ 'osd', 'dump', '--format=json'])
+ if six.PY3:
+ out = out.decode('UTF-8')
try:
osd_json = json.loads(out)
for pool in osd_json['pools']:
@@ -700,8 +706,9 @@ def get_cache_mode(service, pool_name):
def pool_exists(service, name):
"""Check to see if a RADOS pool already exists."""
try:
- out = check_output(['rados', '--id', service,
- 'lspools']).decode('UTF-8')
+ out = check_output(['rados', '--id', service, 'lspools'])
+ if six.PY3:
+ out = out.decode('UTF-8')
except CalledProcessError:
return False
@@ -714,9 +721,12 @@ def get_osds(service):
"""
version = ceph_version()
if version and version >= '0.56':
- return json.loads(check_output(['ceph', '--id', service,
- 'osd', 'ls',
- '--format=json']).decode('UTF-8'))
+ out = check_output(['ceph', '--id', service,
+ 'osd', 'ls',
+ '--format=json'])
+ if six.PY3:
+ out = out.decode('UTF-8')
+ return json.loads(out)
return None
@@ -734,7 +744,9 @@ def rbd_exists(service, pool, rbd_img):
"""Check to see if a RADOS block device exists."""
try:
out = check_output(['rbd', 'list', '--id',
- service, '--pool', pool]).decode('UTF-8')
+ service, '--pool', pool])
+ if six.PY3:
+ out = out.decode('UTF-8')
except CalledProcessError:
return False
@@ -859,7 +871,9 @@ def configure(service, key, auth, use_syslog):
def image_mapped(name):
"""Determine whether a RADOS block device is mapped locally."""
try:
- out = check_output(['rbd', 'showmapped']).decode('UTF-8')
+ out = check_output(['rbd', 'showmapped'])
+ if six.PY3:
+ out = out.decode('UTF-8')
except CalledProcessError:
return False
@@ -1018,7 +1032,9 @@ def ceph_version():
"""Retrieve the local version of ceph."""
if os.path.exists('/usr/bin/ceph'):
cmd = ['ceph', '-v']
- output = check_output(cmd).decode('US-ASCII')
+ output = check_output(cmd)
+ if six.PY3:
+ output = output.decode('UTF-8')
output = output.split()
if len(output) > 3:
return output[2]
diff --git a/hooks/charmhelpers/contrib/storage/linux/lvm.py b/hooks/charmhelpers/contrib/storage/linux/lvm.py
index 4719f53..7f2a060 100644
--- a/hooks/charmhelpers/contrib/storage/linux/lvm.py
+++ b/hooks/charmhelpers/contrib/storage/linux/lvm.py
@@ -74,10 +74,10 @@ def list_lvm_volume_group(block_device):
'''
vg = None
pvd = check_output(['pvdisplay', block_device]).splitlines()
- for l in pvd:
- l = l.decode('UTF-8')
- if l.strip().startswith('VG Name'):
- vg = ' '.join(l.strip().split()[2:])
+ for lvm in pvd:
+ lvm = lvm.decode('UTF-8')
+ if lvm.strip().startswith('VG Name'):
+ vg = ' '.join(lvm.strip().split()[2:])
return vg
diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py
index 3dc0df6..c942889 100644
--- a/hooks/charmhelpers/contrib/storage/linux/utils.py
+++ b/hooks/charmhelpers/contrib/storage/linux/utils.py
@@ -64,6 +64,6 @@ def is_device_mounted(device):
'''
try:
out = check_output(['lsblk', '-P', device]).decode('UTF-8')
- except:
+ except Exception:
return False
return bool(re.search(r'MOUNTPOINT=".+"', out))
diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
index 12f37b2..5a88f79 100644
--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -22,6 +22,7 @@ from __future__ import print_function
import copy
from distutils.version import LooseVersion
from functools import wraps
+from collections import namedtuple
import glob
import os
import json
@@ -218,6 +219,8 @@ def principal_unit():
for rid in relation_ids(reltype):
for unit in related_units(rid):
md = _metadata_unit(unit)
+ if not md:
+ continue
subordinate = md.pop('subordinate', None)
if not subordinate:
return unit
@@ -511,7 +514,10 @@ def _metadata_unit(unit):
"""
basedir = os.sep.join(charm_dir().split(os.sep)[:-2])
unitdir = 'unit-{}'.format(unit.replace(os.sep, '-'))
- with open(os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')) as md:
+ joineddir = os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')
+ if not os.path.exists(joineddir):
+ return None
+ with open(joineddir) as md:
return yaml.safe_load(md)
@@ -639,18 +645,31 @@ def is_relation_made(relation, keys='private-address'):
return False
+def _port_op(op_name, port, protocol="TCP"):
+ """Open or close a service network port"""
+ _args = [op_name]
+ icmp = protocol.upper() == "ICMP"
+ if icmp:
+ _args.append(protocol)
+ else:
+ _args.append('{}/{}'.format(port, protocol))
+ try:
+ subprocess.check_call(_args)
+ except subprocess.CalledProcessError:
+ # Older Juju pre 2.3 doesn't support ICMP
+ # so treat it as a no-op if it fails.
+ if not icmp:
+ raise
+
+
def open_port(port, protocol="TCP"):
"""Open a service network port"""
- _args = ['open-port']
- _args.append('{}/{}'.format(port, protocol))
- subprocess.check_call(_args)
+ _port_op('open-port', port, protocol)
def close_port(port, protocol="TCP"):
"""Close a service network port"""
- _args = ['close-port']
- _args.append('{}/{}'.format(port, protocol))
- subprocess.check_call(_args)
+ _port_op('close-port', port, protocol)
def open_ports(start, end, protocol="TCP"):
@@ -667,6 +686,17 @@ def close_ports(start, end, protocol="TCP"):
subprocess.check_call(_args)
+def opened_ports():
+ """Get the opened ports
+
+ *Note that this will only show ports opened in a previous hook*
+
+ :returns: Opened ports as a list of strings: ``['8080/tcp', '8081-8083/tcp']``
+ """
+ _args = ['opened-ports', '--format=json']
+ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
+
+
@cached
def unit_get(attribute):
"""Get the unit ID for the remote unit"""
@@ -1077,6 +1107,35 @@ def network_get_primary_address(binding):
return subprocess.check_output(cmd).decode('UTF-8').strip()
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def network_get(endpoint, relation_id=None):
+ """
+ Retrieve the network details for a relation endpoint
+
+ :param endpoint: string. The name of a relation endpoint
+ :param relation_id: int. The ID of the relation for the current context.
+ :return: dict. The loaded YAML output of the network-get query.
+ :raise: NotImplementedError if run on Juju < 2.1
+ """
+ cmd = ['network-get', endpoint, '--format', 'yaml']
+ if relation_id:
+ cmd.append('-r')
+ cmd.append(relation_id)
+ try:
+ response = subprocess.check_output(
+ cmd,
+ stderr=subprocess.STDOUT).decode('UTF-8').strip()
+ except CalledProcessError as e:
+ # Early versions of Juju 2.0.x required the --primary-address argument.
+ # We catch that condition here and raise NotImplementedError since
+ # the requested semantics are not available - the caller can then
+ # use the network_get_primary_address() method instead.
+ if '--primary-address is currently required' in e.output.decode('UTF-8'):
+ raise NotImplementedError
+ raise
+ return yaml.safe_load(response)
+
+
def add_metric(*args, **kwargs):
"""Add metric values. Values may be expressed with keyword arguments. For
metric names containing dashes, these may be expressed as one or more
@@ -1106,3 +1165,42 @@ def meter_info():
"""Get the meter status information, if running in the meter-status-changed
hook."""
return os.environ.get('JUJU_METER_INFO')
+
+
+def iter_units_for_relation_name(relation_name):
+ """Iterate through all units in a relation
+
+ Generator that iterates through all the units in a relation and yields
+ a named tuple with rid and unit field names.
+
+ Usage:
+ data = [(u.rid, u.unit)
+ for u in iter_units_for_relation_name(relation_name)]
+
+ :param relation_name: string relation name
+ :yield: Named Tuple with rid and unit field names
+ """
+ RelatedUnit = namedtuple('RelatedUnit', 'rid, unit')
+ for rid in relation_ids(relation_name):
+ for unit in related_units(rid):
+ yield RelatedUnit(rid, unit)
+
+
+def ingress_address(rid=None, unit=None):
+ """
+ Retrieve the ingress-address from a relation when available. Otherwise,
+ return the private-address. This function is to be used on the consuming
+ side of the relation.
+
+ Usage:
+ addresses = [ingress_address(rid=u.rid, unit=u.unit)
+ for u in iter_units_for_relation_name(relation_name)]
+
+ :param rid: string relation id
+ :param unit: string unit name
+ :side effect: calls relation_get
+ :return: string IP address
+ """
+ settings = relation_get(rid=rid, unit=unit)
+ return (settings.get('ingress-address') or
+ settings.get('private-address'))
diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py
index 5656e2f..5cc5c86 100644
--- a/hooks/charmhelpers/core/host.py
+++ b/hooks/charmhelpers/core/host.py
@@ -34,7 +34,7 @@ import six
from contextlib import contextmanager
from collections import OrderedDict
-from .hookenv import log, DEBUG
+from .hookenv import log, DEBUG, local_unit
from .fstab import Fstab
from charmhelpers.osplatform import get_platform
@@ -441,6 +441,49 @@ def add_user_to_group(username, group):
subprocess.check_call(cmd)
+def chage(username, lastday=None, expiredate=None, inactive=None,
+ mindays=None, maxdays=None, root=None, warndays=None):
+ """Change user password expiry information
+
+ :param str username: User to update
+ :param str lastday: Set when password was changed in YYYY-MM-DD format
+ :param str expiredate: Set when user's account will no longer be
+ accessible in YYYY-MM-DD format.
+ -1 will remove an account expiration date.
+ :param str inactive: Set the number of days of inactivity after a password
+ has expired before the account is locked.
+ -1 will remove an account's inactivity.
+ :param str mindays: Set the minimum number of days between password
+ changes to MIN_DAYS.
+ 0 indicates the password can be changed anytime.
+ :param str maxdays: Set the maximum number of days during which a
+ password is valid.
+ -1 as MAX_DAYS will remove checking maxdays
+ :param str root: Apply changes in the CHROOT_DIR directory
+ :param str warndays: Set the number of days of warning before a password
+ change is required
+ :raises subprocess.CalledProcessError: if call to chage fails
+ """
+ cmd = ['chage']
+ if root:
+ cmd.extend(['--root', root])
+ if lastday:
+ cmd.extend(['--lastday', lastday])
+ if expiredate:
+ cmd.extend(['--expiredate', expiredate])
+ if inactive:
+ cmd.extend(['--inactive', inactive])
+ if mindays:
+ cmd.extend(['--mindays', mindays])
+ if maxdays:
+ cmd.extend(['--maxdays', maxdays])
+ if warndays:
+ cmd.extend(['--warndays', warndays])
+ cmd.append(username)
+ subprocess.check_call(cmd)
+
+remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1')
+
def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
"""Replicate the contents of a path"""
options = options or ['--delete', '--executability']
@@ -946,3 +989,31 @@ def updatedb(updatedb_text, new_path):
lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths))
output = "\n".join(lines)
return output
+
+
+def modulo_distribution(modulo=3, wait=30):
+ """ Modulo distribution
+
+ This helper uses the unit number, a modulo value and a constant wait time
+ to produce a calculated wait time distribution. This is useful in large
+ scale deployments to distribute load during an expensive operation such as
+ service restarts.
+
+ If you have 1000 nodes that need to restart 100 at a time 1 minute at a
+ time:
+
+ time.wait(modulo_distribution(modulo=100, wait=60))
+ restart()
+
+ If you need restarts to happen serially set modulo to the exact number of
+ nodes and set a high constant wait time:
+
+ time.wait(modulo_distribution(modulo=10, wait=120))
+ restart()
+
+ @param modulo: int The modulo number creates the group distribution
+ @param wait: int The constant time wait value
+ @return: int Calculated time to wait for unit operation
+ """
+ unit_number = int(local_unit().split('/')[1])
+ return (unit_number % modulo) * wait
diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py
index 685dabd..e8df045 100644
--- a/hooks/charmhelpers/core/strutils.py
+++ b/hooks/charmhelpers/core/strutils.py
@@ -61,13 +61,19 @@ def bytes_from_string(value):
if isinstance(value, six.string_types):
value = six.text_type(value)
else:
- msg = "Unable to interpret non-string value '%s' as boolean" % (value)
+ msg = "Unable to interpret non-string value '%s' as bytes" % (value)
raise ValueError(msg)
matches = re.match("([0-9]+)([a-zA-Z]+)", value)
- if not matches:
- msg = "Unable to interpret string value '%s' as bytes" % (value)
- raise ValueError(msg)
- return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
+ if matches:
+ size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
+ else:
+ # Assume that value passed in is bytes
+ try:
+ size = int(value)
+ except ValueError:
+ msg = "Unable to interpret string value '%s' as bytes" % (value)
+ raise ValueError(msg)
+ return size
class BasicStringComparator(object):
diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py
index 54ec969..7af875c 100644
--- a/hooks/charmhelpers/core/unitdata.py
+++ b/hooks/charmhelpers/core/unitdata.py
@@ -358,7 +358,7 @@ class Storage(object):
try:
yield self.revision
self.revision = None
- except:
+ except Exception:
self.flush(False)
self.revision = None
raise
diff --git a/hooks/charmhelpers/fetch/snap.py b/hooks/charmhelpers/fetch/snap.py
index 112a54c..395836c 100644
--- a/hooks/charmhelpers/fetch/snap.py
+++ b/hooks/charmhelpers/fetch/snap.py
@@ -41,6 +41,10 @@ class CouldNotAcquireLockException(Exception):
pass
+class InvalidSnapChannel(Exception):
+ pass
+
+
def _snap_exec(commands):
"""
Execute snap commands.
@@ -132,3 +136,15 @@ def snap_refresh(packages, *flags):
log(message, level='INFO')
return _snap_exec(['refresh'] + flags + packages)
+
+
+def valid_snap_channel(channel):
+ """ Validate snap channel exists
+
+ :raises InvalidSnapChannel: When channel does not exist
+ :return: Boolean
+ """
+ if channel.lower() in SNAP_CHANNELS:
+ return True
+ else:
+ raise InvalidSnapChannel("Invalid Snap Channel: {}".format(channel))
diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py
index 40e1cb5..910e96a 100644
--- a/hooks/charmhelpers/fetch/ubuntu.py
+++ b/hooks/charmhelpers/fetch/ubuntu.py
@@ -572,7 +572,7 @@ def get_upstream_version(package):
cache = apt_cache()
try:
pkg = cache[package]
- except:
+ except Exception:
# the package is unknown to the current apt cache.
return None
diff --git a/requirements.txt b/requirements.txt
index 67d7085..6a3271b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,5 +10,3 @@ Jinja2>=2.6 # BSD License (3 clause)
six>=1.9.0
dnspython>=1.12.0
psutil>=1.1.1,<2.0.0
-bzr+lp:charm-helpers#egg=charmhelpers
-juju-wait==2.5.0
diff --git a/tests/charmhelpers/__init__.py b/tests/charmhelpers/__init__.py
new file mode 100644
index 0000000..e7aa471
--- /dev/null
+++ b/tests/charmhelpers/__init__.py
@@ -0,0 +1,97 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Bootstrap charm-helpers, installing its dependencies if necessary using
+# only standard libraries.
+from __future__ import print_function
+from __future__ import absolute_import
+
+import functools
+import inspect
+import subprocess
+import sys
+
+try:
+ import six # flake8: noqa
+except ImportError:
+ if sys.version_info.major == 2:
+ subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
+ else:
+ subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
+ import six # flake8: noqa
+
+try:
+ import yaml # flake8: noqa
+except ImportError:
+ if sys.version_info.major == 2:
+ subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
+ else:
+ subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
+ import yaml # flake8: noqa
+
+
+# Holds a list of mapping of mangled function names that have been deprecated
+# using the @deprecate decorator below. This is so that the warning is only
+# printed once for each usage of the function.
+__deprecated_functions = {}
+
+
+def deprecate(warning, date=None, log=None):
+ """Add a deprecation warning the first time the function is used.
+ The date, which is a string in semi-ISO8660 format indicate the year-month
+ that the function is officially going to be removed.
+
+ usage:
+
+ @deprecate('use core/fetch/add_source() instead', '2017-04')
+ def contributed_add_source_thing(...):
+ ...
+
+ And it then prints to the log ONCE that the function is deprecated.
+ The reason for passing the logging function (log) is so that hookenv.log
+ can be used for a charm if needed.
+
+ :param warning: String to indicat where it has moved ot.
+ :param date: optional sting, in YYYY-MM format to indicate when the
+ function will definitely (probably) be removed.
+ :param log: The log function to call to log. If not, logs to stdout
+ """
+ def wrap(f):
+
+ @functools.wraps(f)
+ def wrapped_f(*args, **kwargs):
+ try:
+ module = inspect.getmodule(f)
+ file = inspect.getsourcefile(f)
+ lines = inspect.getsourcelines(f)
+ f_name = "{}-{}-{}..{}-{}".format(
+ module.__name__, file, lines[0], lines[-1], f.__name__)
+ except (IOError, TypeError):
+ # assume it was local, so just use the name of the function
+ f_name = f.__name__
+ if f_name not in __deprecated_functions:
+ __deprecated_functions[f_name] = True
+ s = "DEPRECATION WARNING: Function {} is being removed".format(
+ f.__name__)
+ if date:
+ s = "{} on/around {}".format(s, date)
+ if warning:
+ s = "{} : {}".format(s, warning)
+ if log:
+ log(s)
+ else:
+ print(s)
+ return f(*args, **kwargs)
+ return wrapped_f
+ return wrap
diff --git a/tests/charmhelpers/contrib/__init__.py b/tests/charmhelpers/contrib/__init__.py
new file mode 100644
index 0000000..d7567b8
--- /dev/null
+++ b/tests/charmhelpers/contrib/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/tests/charmhelpers/contrib/amulet/__init__.py b/tests/charmhelpers/contrib/amulet/__init__.py
new file mode 100644
index 0000000..d7567b8
--- /dev/null
+++ b/tests/charmhelpers/contrib/amulet/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/tests/charmhelpers/contrib/amulet/deployment.py b/tests/charmhelpers/contrib/amulet/deployment.py
new file mode 100644
index 0000000..9c65518
--- /dev/null
+++ b/tests/charmhelpers/contrib/amulet/deployment.py
@@ -0,0 +1,97 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import amulet
+import os
+import six
+
+
+class AmuletDeployment(object):
+ """Amulet deployment.
+
+ This class provides generic Amulet deployment and test runner
+ methods.
+ """
+
+ def __init__(self, series=None):
+ """Initialize the deployment environment."""
+ self.series = None
+
+ if series:
+ self.series = series
+ self.d = amulet.Deployment(series=self.series)
+ else:
+ self.d = amulet.Deployment()
+
+ def _add_services(self, this_service, other_services):
+ """Add services.
+
+ Add services to the deployment where this_service is the local charm
+ that we're testing and other_services are the other services that
+ are being used in the local amulet tests.
+ """
+ if this_service['name'] != os.path.basename(os.getcwd()):
+ s = this_service['name']
+ msg = "The charm's root directory name needs to be {}".format(s)
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ if 'units' not in this_service:
+ this_service['units'] = 1
+
+ self.d.add(this_service['name'], units=this_service['units'],
+ constraints=this_service.get('constraints'))
+
+ for svc in other_services:
+ if 'location' in svc:
+ branch_location = svc['location']
+ elif self.series:
+ branch_location = 'cs:{}/{}'.format(self.series, svc['name']),
+ else:
+ branch_location = None
+
+ if 'units' not in svc:
+ svc['units'] = 1
+
+ self.d.add(svc['name'], charm=branch_location, units=svc['units'],
+ constraints=svc.get('constraints'))
+
+ def _add_relations(self, relations):
+ """Add all of the relations for the services."""
+ for k, v in six.iteritems(relations):
+ self.d.relate(k, v)
+
+ def _configure_services(self, configs):
+ """Configure all of the services."""
+ for service, config in six.iteritems(configs):
+ self.d.configure(service, config)
+
+ def _deploy(self):
+ """Deploy environment and wait for all hooks to finish executing."""
+ timeout = int(os.environ.get('AMULET_SETUP_TIMEOUT', 900))
+ try:
+ self.d.setup(timeout=timeout)
+ self.d.sentry.wait(timeout=timeout)
+ except amulet.helpers.TimeoutError:
+ amulet.raise_status(
+ amulet.FAIL,
+ msg="Deployment timed out ({}s)".format(timeout)
+ )
+ except Exception:
+ raise
+
+ def run_tests(self):
+ """Run all of the methods that are prefixed with 'test_'."""
+ for test in dir(self):
+ if test.startswith('test_'):
+ getattr(self, test)()
diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py
new file mode 100644
index 0000000..8a6b764
--- /dev/null
+++ b/tests/charmhelpers/contrib/amulet/utils.py
@@ -0,0 +1,821 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import io
+import json
+import logging
+import os
+import re
+import socket
+import subprocess
+import sys
+import time
+import uuid
+
+import amulet
+import distro_info
+import six
+from six.moves import configparser
+if six.PY3:
+ from urllib import parse as urlparse
+else:
+ import urlparse
+
+
+class AmuletUtils(object):
+ """Amulet utilities.
+
+ This class provides common utility functions that are used by Amulet
+ tests.
+ """
+
+ def __init__(self, log_level=logging.ERROR):
+ self.log = self.get_logger(level=log_level)
+ self.ubuntu_releases = self.get_ubuntu_releases()
+
+ def get_logger(self, name="amulet-logger", level=logging.DEBUG):
+ """Get a logger object that will log to stdout."""
+ log = logging
+ logger = log.getLogger(name)
+ fmt = log.Formatter("%(asctime)s %(funcName)s "
+ "%(levelname)s: %(message)s")
+
+ handler = log.StreamHandler(stream=sys.stdout)
+ handler.setLevel(level)
+ handler.setFormatter(fmt)
+
+ logger.addHandler(handler)
+ logger.setLevel(level)
+
+ return logger
+
+ def valid_ip(self, ip):
+ if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip):
+ return True
+ else:
+ return False
+
+ def valid_url(self, url):
+ p = re.compile(
+ r'^(?:http|ftp)s?://'
+ r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # noqa
+ r'localhost|'
+ r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'
+ r'(?::\d+)?'
+ r'(?:/?|[/?]\S+)$',
+ re.IGNORECASE)
+ if p.match(url):
+ return True
+ else:
+ return False
+
+ def get_ubuntu_release_from_sentry(self, sentry_unit):
+ """Get Ubuntu release codename from sentry unit.
+
+ :param sentry_unit: amulet sentry/service unit pointer
+ :returns: list of strings - release codename, failure message
+ """
+ msg = None
+ cmd = 'lsb_release -cs'
+ release, code = sentry_unit.run(cmd)
+ if code == 0:
+ self.log.debug('{} lsb_release: {}'.format(
+ sentry_unit.info['unit_name'], release))
+ else:
+ msg = ('{} `{}` returned {} '
+ '{}'.format(sentry_unit.info['unit_name'],
+ cmd, release, code))
+ if release not in self.ubuntu_releases:
+ msg = ("Release ({}) not found in Ubuntu releases "
+ "({})".format(release, self.ubuntu_releases))
+ return release, msg
+
+ def validate_services(self, commands):
+ """Validate that lists of commands succeed on service units. Can be
+ used to verify system services are running on the corresponding
+ service units.
+
+ :param commands: dict with sentry keys and arbitrary command list vals
+ :returns: None if successful, Failure string message otherwise
+ """
+ self.log.debug('Checking status of system services...')
+
+ # /!\ DEPRECATION WARNING (beisner):
+ # New and existing tests should be rewritten to use
+ # validate_services_by_name() as it is aware of init systems.
+ self.log.warn('DEPRECATION WARNING: use '
+ 'validate_services_by_name instead of validate_services '
+ 'due to init system differences.')
+
+ for k, v in six.iteritems(commands):
+ for cmd in v:
+ output, code = k.run(cmd)
+ self.log.debug('{} `{}` returned '
+ '{}'.format(k.info['unit_name'],
+ cmd, code))
+ if code != 0:
+ return "command `{}` returned {}".format(cmd, str(code))
+ return None
+
+ def validate_services_by_name(self, sentry_services):
+ """Validate system service status by service name, automatically
+ detecting init system based on Ubuntu release codename.
+
+ :param sentry_services: dict with sentry keys and svc list values
+ :returns: None if successful, Failure string message otherwise
+ """
+ self.log.debug('Checking status of system services...')
+
+ # Point at which systemd became a thing
+ systemd_switch = self.ubuntu_releases.index('vivid')
+
+ for sentry_unit, services_list in six.iteritems(sentry_services):
+ # Get lsb_release codename from unit
+ release, ret = self.get_ubuntu_release_from_sentry(sentry_unit)
+ if ret:
+ return ret
+
+ for service_name in services_list:
+ if (self.ubuntu_releases.index(release) >= systemd_switch or
+ service_name in ['rabbitmq-server', 'apache2',
+ 'memcached']):
+ # init is systemd (or regular sysv)
+ cmd = 'sudo service {} status'.format(service_name)
+ output, code = sentry_unit.run(cmd)
+ service_running = code == 0
+ elif self.ubuntu_releases.index(release) < systemd_switch:
+ # init is upstart
+ cmd = 'sudo status {}'.format(service_name)
+ output, code = sentry_unit.run(cmd)
+ service_running = code == 0 and "start/running" in output
+
+ self.log.debug('{} `{}` returned '
+ '{}'.format(sentry_unit.info['unit_name'],
+ cmd, code))
+ if not service_running:
+ return u"command `{}` returned {} {}".format(
+ cmd, output, str(code))
+ return None
+
+ def _get_config(self, unit, filename):
+ """Get a ConfigParser object for parsing a unit's config file."""
+ file_contents = unit.file_contents(filename)
+
+ # NOTE(beisner): by default, ConfigParser does not handle options
+ # with no value, such as the flags used in the mysql my.cnf file.
+ # https://bugs.python.org/issue7005
+ config = configparser.ConfigParser(allow_no_value=True)
+ config.readfp(io.StringIO(file_contents))
+ return config
+
+ def validate_config_data(self, sentry_unit, config_file, section,
+ expected):
+ """Validate config file data.
+
+ Verify that the specified section of the config file contains
+ the expected option key:value pairs.
+
+ Compare expected dictionary data vs actual dictionary data.
+ The values in the 'expected' dictionary can be strings, bools, ints,
+ longs, or can be a function that evaluates a variable and returns a
+ bool.
+ """
+ self.log.debug('Validating config file data ({} in {} on {})'
+ '...'.format(section, config_file,
+ sentry_unit.info['unit_name']))
+ config = self._get_config(sentry_unit, config_file)
+
+ if section != 'DEFAULT' and not config.has_section(section):
+ return "section [{}] does not exist".format(section)
+
+ for k in expected.keys():
+ if not config.has_option(section, k):
+ return "section [{}] is missing option {}".format(section, k)
+
+ actual = config.get(section, k)
+ v = expected[k]
+ if (isinstance(v, six.string_types) or
+ isinstance(v, bool) or
+ isinstance(v, six.integer_types)):
+ # handle explicit values
+ if actual != v:
+ return "section [{}] {}:{} != expected {}:{}".format(
+ section, k, actual, k, expected[k])
+ # handle function pointers, such as not_null or valid_ip
+ elif not v(actual):
+ return "section [{}] {}:{} != expected {}:{}".format(
+ section, k, actual, k, expected[k])
+ return None
+
+ def _validate_dict_data(self, expected, actual):
+ """Validate dictionary data.
+
+ Compare expected dictionary data vs actual dictionary data.
+ The values in the 'expected' dictionary can be strings, bools, ints,
+ longs, or can be a function that evaluates a variable and returns a
+ bool.
+ """
+ self.log.debug('actual: {}'.format(repr(actual)))
+ self.log.debug('expected: {}'.format(repr(expected)))
+
+ for k, v in six.iteritems(expected):
+ if k in actual:
+ if (isinstance(v, six.string_types) or
+ isinstance(v, bool) or
+ isinstance(v, six.integer_types)):
+ # handle explicit values
+ if v != actual[k]:
+ return "{}:{}".format(k, actual[k])
+ # handle function pointers, such as not_null or valid_ip
+ elif not v(actual[k]):
+ return "{}:{}".format(k, actual[k])
+ else:
+ return "key '{}' does not exist".format(k)
+ return None
+
+ def validate_relation_data(self, sentry_unit, relation, expected):
+ """Validate actual relation data based on expected relation data."""
+ actual = sentry_unit.relation(relation[0], relation[1])
+ return self._validate_dict_data(expected, actual)
+
+ def _validate_list_data(self, expected, actual):
+ """Compare expected list vs actual list data."""
+ for e in expected:
+ if e not in actual:
+ return "expected item {} not found in actual list".format(e)
+ return None
+
+ def not_null(self, string):
+ if string is not None:
+ return True
+ else:
+ return False
+
+ def _get_file_mtime(self, sentry_unit, filename):
+ """Get last modification time of file."""
+ return sentry_unit.file_stat(filename)['mtime']
+
+ def _get_dir_mtime(self, sentry_unit, directory):
+ """Get last modification time of directory."""
+ return sentry_unit.directory_stat(directory)['mtime']
+
+ def _get_proc_start_time(self, sentry_unit, service, pgrep_full=None):
+ """Get start time of a process based on the last modification time
+ of the /proc/pid directory.
+
+ :sentry_unit: The sentry unit to check for the service on
+ :service: service name to look for in process table
+ :pgrep_full: [Deprecated] Use full command line search mode with pgrep
+ :returns: epoch time of service process start
+ :param commands: list of bash commands
+ :param sentry_units: list of sentry unit pointers
+ :returns: None if successful; Failure message otherwise
+ """
+ if pgrep_full is not None:
+ # /!\ DEPRECATION WARNING (beisner):
+ # No longer implemented, as pidof is now used instead of pgrep.
+ # https://bugs.launchpad.net/charm-helpers/+bug/1474030
+ self.log.warn('DEPRECATION WARNING: pgrep_full bool is no '
+ 'longer implemented re: lp 1474030.')
+
+ pid_list = self.get_process_id_list(sentry_unit, service)
+ pid = pid_list[0]
+ proc_dir = '/proc/{}'.format(pid)
+ self.log.debug('Pid for {} on {}: {}'.format(
+ service, sentry_unit.info['unit_name'], pid))
+
+ return self._get_dir_mtime(sentry_unit, proc_dir)
+
+ def service_restarted(self, sentry_unit, service, filename,
+ pgrep_full=None, sleep_time=20):
+ """Check if service was restarted.
+
+ Compare a service's start time vs a file's last modification time
+ (such as a config file for that service) to determine if the service
+ has been restarted.
+ """
+ # /!\ DEPRECATION WARNING (beisner):
+ # This method is prone to races in that no before-time is known.
+ # Use validate_service_config_changed instead.
+
+ # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
+ # used instead of pgrep. pgrep_full is still passed through to ensure
+ # deprecation WARNS. lp1474030
+ self.log.warn('DEPRECATION WARNING: use '
+ 'validate_service_config_changed instead of '
+ 'service_restarted due to known races.')
+
+ time.sleep(sleep_time)
+ if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >=
+ self._get_file_mtime(sentry_unit, filename)):
+ return True
+ else:
+ return False
+
+ def service_restarted_since(self, sentry_unit, mtime, service,
+ pgrep_full=None, sleep_time=20,
+ retry_count=30, retry_sleep_time=10):
+ """Check if service was been started after a given time.
+
+ Args:
+ sentry_unit (sentry): The sentry unit to check for the service on
+ mtime (float): The epoch time to check against
+ service (string): service name to look for in process table
+ pgrep_full: [Deprecated] Use full command line search mode with pgrep
+ sleep_time (int): Initial sleep time (s) before looking for file
+ retry_sleep_time (int): Time (s) to sleep between retries
+ retry_count (int): If file is not found, how many times to retry
+
+ Returns:
+ bool: True if service found and its start time it newer than mtime,
+ False if service is older than mtime or if service was
+ not found.
+ """
+ # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
+ # used instead of pgrep. pgrep_full is still passed through to ensure
+ # deprecation WARNS. lp1474030
+
+ unit_name = sentry_unit.info['unit_name']
+ self.log.debug('Checking that %s service restarted since %s on '
+ '%s' % (service, mtime, unit_name))
+ time.sleep(sleep_time)
+ proc_start_time = None
+ tries = 0
+ while tries <= retry_count and not proc_start_time:
+ try:
+ proc_start_time = self._get_proc_start_time(sentry_unit,
+ service,
+ pgrep_full)
+ self.log.debug('Attempt {} to get {} proc start time on {} '
+ 'OK'.format(tries, service, unit_name))
+ except IOError as e:
+ # NOTE(beisner) - race avoidance, proc may not exist yet.
+ # https://bugs.launchpad.net/charm-helpers/+bug/1474030
+ self.log.debug('Attempt {} to get {} proc start time on {} '
+ 'failed\n{}'.format(tries, service,
+ unit_name, e))
+ time.sleep(retry_sleep_time)
+ tries += 1
+
+ if not proc_start_time:
+ self.log.warn('No proc start time found, assuming service did '
+ 'not start')
+ return False
+ if proc_start_time >= mtime:
+ self.log.debug('Proc start time is newer than provided mtime'
+ '(%s >= %s) on %s (OK)' % (proc_start_time,
+ mtime, unit_name))
+ return True
+ else:
+ self.log.warn('Proc start time (%s) is older than provided mtime '
+ '(%s) on %s, service did not '
+ 'restart' % (proc_start_time, mtime, unit_name))
+ return False
+
+ def config_updated_since(self, sentry_unit, filename, mtime,
+ sleep_time=20, retry_count=30,
+ retry_sleep_time=10):
+ """Check if file was modified after a given time.
+
+ Args:
+ sentry_unit (sentry): The sentry unit to check the file mtime on
+ filename (string): The file to check mtime of
+ mtime (float): The epoch time to check against
+ sleep_time (int): Initial sleep time (s) before looking for file
+ retry_sleep_time (int): Time (s) to sleep between retries
+ retry_count (int): If file is not found, how many times to retry
+
+ Returns:
+ bool: True if file was modified more recently than mtime, False if
+ file was modified before mtime, or if file not found.
+ """
+ unit_name = sentry_unit.info['unit_name']
+ self.log.debug('Checking that %s updated since %s on '
+ '%s' % (filename, mtime, unit_name))
+ time.sleep(sleep_time)
+ file_mtime = None
+ tries = 0
+ while tries <= retry_count and not file_mtime:
+ try:
+ file_mtime = self._get_file_mtime(sentry_unit, filename)
+ self.log.debug('Attempt {} to get {} file mtime on {} '
+ 'OK'.format(tries, filename, unit_name))
+ except IOError as e:
+ # NOTE(beisner) - race avoidance, file may not exist yet.
+ # https://bugs.launchpad.net/charm-helpers/+bug/1474030
+ self.log.debug('Attempt {} to get {} file mtime on {} '
+ 'failed\n{}'.format(tries, filename,
+ unit_name, e))
+ time.sleep(retry_sleep_time)
+ tries += 1
+
+ if not file_mtime:
+ self.log.warn('Could not determine file mtime, assuming '
+ 'file does not exist')
+ return False
+
+ if file_mtime >= mtime:
+ self.log.debug('File mtime is newer than provided mtime '
+ '(%s >= %s) on %s (OK)' % (file_mtime,
+ mtime, unit_name))
+ return True
+ else:
+ self.log.warn('File mtime is older than provided mtime'
+ '(%s < on %s) on %s' % (file_mtime,
+ mtime, unit_name))
+ return False
+
+ def validate_service_config_changed(self, sentry_unit, mtime, service,
+ filename, pgrep_full=None,
+ sleep_time=20, retry_count=30,
+ retry_sleep_time=10):
+ """Check service and file were updated after mtime
+
+ Args:
+ sentry_unit (sentry): The sentry unit to check for the service on
+ mtime (float): The epoch time to check against
+ service (string): service name to look for in process table
+ filename (string): The file to check mtime of
+ pgrep_full: [Deprecated] Use full command line search mode with pgrep
+ sleep_time (int): Initial sleep in seconds to pass to test helpers
+ retry_count (int): If service is not found, how many times to retry
+ retry_sleep_time (int): Time in seconds to wait between retries
+
+ Typical Usage:
+ u = OpenStackAmuletUtils(ERROR)
+ ...
+ mtime = u.get_sentry_time(self.cinder_sentry)
+ self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'})
+ if not u.validate_service_config_changed(self.cinder_sentry,
+ mtime,
+ 'cinder-api',
+ '/etc/cinder/cinder.conf')
+ amulet.raise_status(amulet.FAIL, msg='update failed')
+ Returns:
+ bool: True if both service and file where updated/restarted after
+ mtime, False if service is older than mtime or if service was
+ not found or if filename was modified before mtime.
+ """
+
+ # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
+ # used instead of pgrep. pgrep_full is still passed through to ensure
+ # deprecation WARNS. lp1474030
+
+ service_restart = self.service_restarted_since(
+ sentry_unit, mtime,
+ service,
+ pgrep_full=pgrep_full,
+ sleep_time=sleep_time,
+ retry_count=retry_count,
+ retry_sleep_time=retry_sleep_time)
+
+ config_update = self.config_updated_since(
+ sentry_unit,
+ filename,
+ mtime,
+ sleep_time=sleep_time,
+ retry_count=retry_count,
+ retry_sleep_time=retry_sleep_time)
+
+ return service_restart and config_update
+
+ def get_sentry_time(self, sentry_unit):
+ """Return current epoch time on a sentry"""
+ cmd = "date +'%s'"
+ return float(sentry_unit.run(cmd)[0])
+
+ def relation_error(self, name, data):
+ return 'unexpected relation data in {} - {}'.format(name, data)
+
+ def endpoint_error(self, name, data):
+ return 'unexpected endpoint data in {} - {}'.format(name, data)
+
+ def get_ubuntu_releases(self):
+ """Return a list of all Ubuntu releases in order of release."""
+ _d = distro_info.UbuntuDistroInfo()
+ _release_list = _d.all
+ return _release_list
+
+ def file_to_url(self, file_rel_path):
+ """Convert a relative file path to a file URL."""
+ _abs_path = os.path.abspath(file_rel_path)
+ return urlparse.urlparse(_abs_path, scheme='file').geturl()
+
+ def check_commands_on_units(self, commands, sentry_units):
+ """Check that all commands in a list exit zero on all
+ sentry units in a list.
+
+ :param commands: list of bash commands
+ :param sentry_units: list of sentry unit pointers
+ :returns: None if successful; Failure message otherwise
+ """
+ self.log.debug('Checking exit codes for {} commands on {} '
+ 'sentry units...'.format(len(commands),
+ len(sentry_units)))
+ for sentry_unit in sentry_units:
+ for cmd in commands:
+ output, code = sentry_unit.run(cmd)
+ if code == 0:
+ self.log.debug('{} `{}` returned {} '
+ '(OK)'.format(sentry_unit.info['unit_name'],
+ cmd, code))
+ else:
+ return ('{} `{}` returned {} '
+ '{}'.format(sentry_unit.info['unit_name'],
+ cmd, code, output))
+ return None
+
+ def get_process_id_list(self, sentry_unit, process_name,
+ expect_success=True):
+ """Get a list of process ID(s) from a single sentry juju unit
+ for a single process name.
+
+ :param sentry_unit: Amulet sentry instance (juju unit)
+ :param process_name: Process name
+ :param expect_success: If False, expect the PID to be missing,
+ raise if it is present.
+ :returns: List of process IDs
+ """
+ cmd = 'pidof -x "{}"'.format(process_name)
+ if not expect_success:
+ cmd += " || exit 0 && exit 1"
+ output, code = sentry_unit.run(cmd)
+ if code != 0:
+ msg = ('{} `{}` returned {} '
+ '{}'.format(sentry_unit.info['unit_name'],
+ cmd, code, output))
+ amulet.raise_status(amulet.FAIL, msg=msg)
+ return str(output).split()
+
+ def get_unit_process_ids(self, unit_processes, expect_success=True):
+ """Construct a dict containing unit sentries, process names, and
+ process IDs.
+
+ :param unit_processes: A dictionary of Amulet sentry instance
+ to list of process names.
+ :param expect_success: if False expect the processes to not be
+ running, raise if they are.
+ :returns: Dictionary of Amulet sentry instance to dictionary
+ of process names to PIDs.
+ """
+ pid_dict = {}
+ for sentry_unit, process_list in six.iteritems(unit_processes):
+ pid_dict[sentry_unit] = {}
+ for process in process_list:
+ pids = self.get_process_id_list(
+ sentry_unit, process, expect_success=expect_success)
+ pid_dict[sentry_unit].update({process: pids})
+ return pid_dict
+
+ def validate_unit_process_ids(self, expected, actual):
+ """Validate process id quantities for services on units."""
+ self.log.debug('Checking units for running processes...')
+ self.log.debug('Expected PIDs: {}'.format(expected))
+ self.log.debug('Actual PIDs: {}'.format(actual))
+
+ if len(actual) != len(expected):
+ return ('Unit count mismatch. expected, actual: {}, '
+ '{} '.format(len(expected), len(actual)))
+
+ for (e_sentry, e_proc_names) in six.iteritems(expected):
+ e_sentry_name = e_sentry.info['unit_name']
+ if e_sentry in actual.keys():
+ a_proc_names = actual[e_sentry]
+ else:
+ return ('Expected sentry ({}) not found in actual dict data.'
+ '{}'.format(e_sentry_name, e_sentry))
+
+ if len(e_proc_names.keys()) != len(a_proc_names.keys()):
+ return ('Process name count mismatch. expected, actual: {}, '
+ '{}'.format(len(expected), len(actual)))
+
+ for (e_proc_name, e_pids), (a_proc_name, a_pids) in \
+ zip(e_proc_names.items(), a_proc_names.items()):
+ if e_proc_name != a_proc_name:
+ return ('Process name mismatch. expected, actual: {}, '
+ '{}'.format(e_proc_name, a_proc_name))
+
+ a_pids_length = len(a_pids)
+ fail_msg = ('PID count mismatch. {} ({}) expected, actual: '
+ '{}, {} ({})'.format(e_sentry_name, e_proc_name,
+ e_pids, a_pids_length,
+ a_pids))
+
+ # If expected is a list, ensure at least one PID quantity match
+ if isinstance(e_pids, list) and \
+ a_pids_length not in e_pids:
+ return fail_msg
+ # If expected is not bool and not list,
+ # ensure PID quantities match
+ elif not isinstance(e_pids, bool) and \
+ not isinstance(e_pids, list) and \
+ a_pids_length != e_pids:
+ return fail_msg
+ # If expected is bool True, ensure 1 or more PIDs exist
+ elif isinstance(e_pids, bool) and \
+ e_pids is True and a_pids_length < 1:
+ return fail_msg
+ # If expected is bool False, ensure 0 PIDs exist
+ elif isinstance(e_pids, bool) and \
+ e_pids is False and a_pids_length != 0:
+ return fail_msg
+ else:
+ self.log.debug('PID check OK: {} {} {}: '
+ '{}'.format(e_sentry_name, e_proc_name,
+ e_pids, a_pids))
+ return None
+
+ def validate_list_of_identical_dicts(self, list_of_dicts):
+ """Check that all dicts within a list are identical."""
+ hashes = []
+ for _dict in list_of_dicts:
+ hashes.append(hash(frozenset(_dict.items())))
+
+ self.log.debug('Hashes: {}'.format(hashes))
+ if len(set(hashes)) == 1:
+ self.log.debug('Dicts within list are identical')
+ else:
+ return 'Dicts within list are not identical'
+
+ return None
+
+ def validate_sectionless_conf(self, file_contents, expected):
+ """A crude conf parser. Useful to inspect configuration files which
+ do not have section headers (as would be necessary in order to use
+ the configparser). Such as openstack-dashboard or rabbitmq confs."""
+ for line in file_contents.split('\n'):
+ if '=' in line:
+ args = line.split('=')
+ if len(args) <= 1:
+ continue
+ key = args[0].strip()
+ value = args[1].strip()
+ if key in expected.keys():
+ if expected[key] != value:
+ msg = ('Config mismatch. Expected, actual: {}, '
+ '{}'.format(expected[key], value))
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ def get_unit_hostnames(self, units):
+ """Return a dict of juju unit names to hostnames."""
+ host_names = {}
+ for unit in units:
+ host_names[unit.info['unit_name']] = \
+ str(unit.file_contents('/etc/hostname').strip())
+ self.log.debug('Unit host names: {}'.format(host_names))
+ return host_names
+
+ def run_cmd_unit(self, sentry_unit, cmd):
+ """Run a command on a unit, return the output and exit code."""
+ output, code = sentry_unit.run(cmd)
+ if code == 0:
+ self.log.debug('{} `{}` command returned {} '
+ '(OK)'.format(sentry_unit.info['unit_name'],
+ cmd, code))
+ else:
+ msg = ('{} `{}` command returned {} '
+ '{}'.format(sentry_unit.info['unit_name'],
+ cmd, code, output))
+ amulet.raise_status(amulet.FAIL, msg=msg)
+ return str(output), code
+
+ def file_exists_on_unit(self, sentry_unit, file_name):
+ """Check if a file exists on a unit."""
+ try:
+ sentry_unit.file_stat(file_name)
+ return True
+ except IOError:
+ return False
+ except Exception as e:
+ msg = 'Error checking file {}: {}'.format(file_name, e)
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ def file_contents_safe(self, sentry_unit, file_name,
+ max_wait=60, fatal=False):
+ """Get file contents from a sentry unit. Wrap amulet file_contents
+ with retry logic to address races where a file checks as existing,
+ but no longer exists by the time file_contents is called.
+ Return None if file not found. Optionally raise if fatal is True."""
+ unit_name = sentry_unit.info['unit_name']
+ file_contents = False
+ tries = 0
+ while not file_contents and tries < (max_wait / 4):
+ try:
+ file_contents = sentry_unit.file_contents(file_name)
+ except IOError:
+ self.log.debug('Attempt {} to open file {} from {} '
+ 'failed'.format(tries, file_name,
+ unit_name))
+ time.sleep(4)
+ tries += 1
+
+ if file_contents:
+ return file_contents
+ elif not fatal:
+ return None
+ elif fatal:
+ msg = 'Failed to get file contents from unit.'
+ amulet.raise_status(amulet.FAIL, msg)
+
+ def port_knock_tcp(self, host="localhost", port=22, timeout=15):
+ """Open a TCP socket to check for a listening sevice on a host.
+
+ :param host: host name or IP address, default to localhost
+ :param port: TCP port number, default to 22
+ :param timeout: Connect timeout, default to 15 seconds
+ :returns: True if successful, False if connect failed
+ """
+
+ # Resolve host name if possible
+ try:
+ connect_host = socket.gethostbyname(host)
+ host_human = "{} ({})".format(connect_host, host)
+ except socket.error as e:
+ self.log.warn('Unable to resolve address: '
+ '{} ({}) Trying anyway!'.format(host, e))
+ connect_host = host
+ host_human = connect_host
+
+ # Attempt socket connection
+ try:
+ knock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ knock.settimeout(timeout)
+ knock.connect((connect_host, port))
+ knock.close()
+ self.log.debug('Socket connect OK for host '
+ '{} on port {}.'.format(host_human, port))
+ return True
+ except socket.error as e:
+ self.log.debug('Socket connect FAIL for'
+ ' {} port {} ({})'.format(host_human, port, e))
+ return False
+
+ def port_knock_units(self, sentry_units, port=22,
+ timeout=15, expect_success=True):
+ """Open a TCP socket to check for a listening sevice on each
+ listed juju unit.
+
+ :param sentry_units: list of sentry unit pointers
+ :param port: TCP port number, default to 22
+ :param timeout: Connect timeout, default to 15 seconds
+ :expect_success: True by default, set False to invert logic
+ :returns: None if successful, Failure message otherwise
+ """
+ for unit in sentry_units:
+ host = unit.info['public-address']
+ connected = self.port_knock_tcp(host, port, timeout)
+ if not connected and expect_success:
+ return 'Socket connect failed.'
+ elif connected and not expect_success:
+ return 'Socket connected unexpectedly.'
+
+ def get_uuid_epoch_stamp(self):
+ """Returns a stamp string based on uuid4 and epoch time. Useful in
+ generating test messages which need to be unique-ish."""
+ return '[{}-{}]'.format(uuid.uuid4(), time.time())
+
+ # amulet juju action helpers:
+ def run_action(self, unit_sentry, action,
+ _check_output=subprocess.check_output,
+ params=None):
+ """Translate to amulet's built in run_action(). Deprecated.
+
+ Run the named action on a given unit sentry.
+
+ params a dict of parameters to use
+ _check_output parameter is no longer used
+
+ @return action_id.
+ """
+ self.log.warn('charmhelpers.contrib.amulet.utils.run_action has been '
+ 'deprecated for amulet.run_action')
+ return unit_sentry.run_action(action, action_args=params)
+
+ def wait_on_action(self, action_id, _check_output=subprocess.check_output):
+ """Wait for a given action, returning if it completed or not.
+
+ action_id a string action uuid
+ _check_output parameter is no longer used
+ """
+ data = amulet.actions.get_action_output(action_id, full_output=True)
+ return data.get(u"status") == "completed"
+
+ def status_get(self, unit):
+ """Return the current service status of this unit."""
+ raw_status, return_code = unit.run(
+ "status-get --format=json --include-data")
+ if return_code != 0:
+ return ("unknown", "")
+ status = json.loads(raw_status)
+ return (status["status"], status["message"])
diff --git a/tests/charmhelpers/contrib/network/__init__.py b/tests/charmhelpers/contrib/network/__init__.py
new file mode 100644
index 0000000..d7567b8
--- /dev/null
+++ b/tests/charmhelpers/contrib/network/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/tests/charmhelpers/contrib/network/ip.py b/tests/charmhelpers/contrib/network/ip.py
new file mode 100644
index 0000000..a871ce3
--- /dev/null
+++ b/tests/charmhelpers/contrib/network/ip.py
@@ -0,0 +1,593 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+import re
+import subprocess
+import six
+import socket
+
+from functools import partial
+
+from charmhelpers.fetch import apt_install, apt_update
+from charmhelpers.core.hookenv import (
+ config,
+ log,
+ network_get_primary_address,
+ unit_get,
+ WARNING,
+)
+
+from charmhelpers.core.host import (
+ lsb_release,
+ CompareHostReleases,
+)
+
+try:
+ import netifaces
+except ImportError:
+ apt_update(fatal=True)
+ if six.PY2:
+ apt_install('python-netifaces', fatal=True)
+ else:
+ apt_install('python3-netifaces', fatal=True)
+ import netifaces
+
+try:
+ import netaddr
+except ImportError:
+ apt_update(fatal=True)
+ if six.PY2:
+ apt_install('python-netaddr', fatal=True)
+ else:
+ apt_install('python3-netaddr', fatal=True)
+ import netaddr
+
+
+def _validate_cidr(network):
+ try:
+ netaddr.IPNetwork(network)
+ except (netaddr.core.AddrFormatError, ValueError):
+ raise ValueError("Network (%s) is not in CIDR presentation format" %
+ network)
+
+
+def no_ip_found_error_out(network):
+ errmsg = ("No IP address found in network(s): %s" % network)
+ raise ValueError(errmsg)
+
+
+def _get_ipv6_network_from_address(address):
+ """Get an netaddr.IPNetwork for the given IPv6 address
+ :param address: a dict as returned by netifaces.ifaddresses
+ :returns netaddr.IPNetwork: None if the address is a link local or loopback
+ address
+ """
+ if address['addr'].startswith('fe80') or address['addr'] == "::1":
+ return None
+
+ prefix = address['netmask'].split("/")
+ if len(prefix) > 1:
+ netmask = prefix[1]
+ else:
+ netmask = address['netmask']
+ return netaddr.IPNetwork("%s/%s" % (address['addr'],
+ netmask))
+
+
+def get_address_in_network(network, fallback=None, fatal=False):
+ """Get an IPv4 or IPv6 address within the network from the host.
+
+ :param network (str): CIDR presentation format. For example,
+ '192.168.1.0/24'. Supports multiple networks as a space-delimited list.
+ :param fallback (str): If no address is found, return fallback.
+ :param fatal (boolean): If no address is found, fallback is not
+ set and fatal is True then exit(1).
+ """
+ if network is None:
+ if fallback is not None:
+ return fallback
+
+ if fatal:
+ no_ip_found_error_out(network)
+ else:
+ return None
+
+ networks = network.split() or [network]
+ for network in networks:
+ _validate_cidr(network)
+ network = netaddr.IPNetwork(network)
+ for iface in netifaces.interfaces():
+ addresses = netifaces.ifaddresses(iface)
+ if network.version == 4 and netifaces.AF_INET in addresses:
+ for addr in addresses[netifaces.AF_INET]:
+ cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
+ addr['netmask']))
+ if cidr in network:
+ return str(cidr.ip)
+
+ if network.version == 6 and netifaces.AF_INET6 in addresses:
+ for addr in addresses[netifaces.AF_INET6]:
+ cidr = _get_ipv6_network_from_address(addr)
+ if cidr and cidr in network:
+ return str(cidr.ip)
+
+ if fallback is not None:
+ return fallback
+
+ if fatal:
+ no_ip_found_error_out(network)
+
+ return None
+
+
+def is_ipv6(address):
+ """Determine whether provided address is IPv6 or not."""
+ try:
+ address = netaddr.IPAddress(address)
+ except netaddr.AddrFormatError:
+ # probably a hostname - so not an address at all!
+ return False
+
+ return address.version == 6
+
+
+def is_address_in_network(network, address):
+ """
+ Determine whether the provided address is within a network range.
+
+ :param network (str): CIDR presentation format. For example,
+ '192.168.1.0/24'.
+ :param address: An individual IPv4 or IPv6 address without a net
+ mask or subnet prefix. For example, '192.168.1.1'.
+ :returns boolean: Flag indicating whether address is in network.
+ """
+ try:
+ network = netaddr.IPNetwork(network)
+ except (netaddr.core.AddrFormatError, ValueError):
+ raise ValueError("Network (%s) is not in CIDR presentation format" %
+ network)
+
+ try:
+ address = netaddr.IPAddress(address)
+ except (netaddr.core.AddrFormatError, ValueError):
+ raise ValueError("Address (%s) is not in correct presentation format" %
+ address)
+
+ if address in network:
+ return True
+ else:
+ return False
+
+
+def _get_for_address(address, key):
+ """Retrieve an attribute of or the physical interface that
+ the IP address provided could be bound to.
+
+ :param address (str): An individual IPv4 or IPv6 address without a net
+ mask or subnet prefix. For example, '192.168.1.1'.
+ :param key: 'iface' for the physical interface name or an attribute
+ of the configured interface, for example 'netmask'.
+ :returns str: Requested attribute or None if address is not bindable.
+ """
+ address = netaddr.IPAddress(address)
+ for iface in netifaces.interfaces():
+ addresses = netifaces.ifaddresses(iface)
+ if address.version == 4 and netifaces.AF_INET in addresses:
+ addr = addresses[netifaces.AF_INET][0]['addr']
+ netmask = addresses[netifaces.AF_INET][0]['netmask']
+ network = netaddr.IPNetwork("%s/%s" % (addr, netmask))
+ cidr = network.cidr
+ if address in cidr:
+ if key == 'iface':
+ return iface
+ else:
+ return addresses[netifaces.AF_INET][0][key]
+
+ if address.version == 6 and netifaces.AF_INET6 in addresses:
+ for addr in addresses[netifaces.AF_INET6]:
+ network = _get_ipv6_network_from_address(addr)
+ if not network:
+ continue
+
+ cidr = network.cidr
+ if address in cidr:
+ if key == 'iface':
+ return iface
+ elif key == 'netmask' and cidr:
+ return str(cidr).split('/')[1]
+ else:
+ return addr[key]
+ return None
+
+
+get_iface_for_address = partial(_get_for_address, key='iface')
+
+
+get_netmask_for_address = partial(_get_for_address, key='netmask')
+
+
+def resolve_network_cidr(ip_address):
+ '''
+ Resolves the full address cidr of an ip_address based on
+ configured network interfaces
+ '''
+ netmask = get_netmask_for_address(ip_address)
+ return str(netaddr.IPNetwork("%s/%s" % (ip_address, netmask)).cidr)
+
+
+def format_ipv6_addr(address):
+ """If address is IPv6, wrap it in '[]' otherwise return None.
+
+ This is required by most configuration files when specifying IPv6
+ addresses.
+ """
+ if is_ipv6(address):
+ return "[%s]" % address
+
+ return None
+
+
+def is_ipv6_disabled():
+ try:
+ result = subprocess.check_output(
+ ['sysctl', 'net.ipv6.conf.all.disable_ipv6'],
+ stderr=subprocess.STDOUT,
+ universal_newlines=True)
+ except subprocess.CalledProcessError:
+ return True
+
+ return "net.ipv6.conf.all.disable_ipv6 = 1" in result
+
+
+def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False,
+ fatal=True, exc_list=None):
+ """Return the assigned IP address for a given interface, if any.
+
+ :param iface: network interface on which address(es) are expected to
+ be found.
+ :param inet_type: inet address family
+ :param inc_aliases: include alias interfaces in search
+ :param fatal: if True, raise exception if address not found
+ :param exc_list: list of addresses to ignore
+ :return: list of ip addresses
+ """
+ # Extract nic if passed /dev/ethX
+ if '/' in iface:
+ iface = iface.split('/')[-1]
+
+ if not exc_list:
+ exc_list = []
+
+ try:
+ inet_num = getattr(netifaces, inet_type)
+ except AttributeError:
+ raise Exception("Unknown inet type '%s'" % str(inet_type))
+
+ interfaces = netifaces.interfaces()
+ if inc_aliases:
+ ifaces = []
+ for _iface in interfaces:
+ if iface == _iface or _iface.split(':')[0] == iface:
+ ifaces.append(_iface)
+
+ if fatal and not ifaces:
+ raise Exception("Invalid interface '%s'" % iface)
+
+ ifaces.sort()
+ else:
+ if iface not in interfaces:
+ if fatal:
+ raise Exception("Interface '%s' not found " % (iface))
+ else:
+ return []
+
+ else:
+ ifaces = [iface]
+
+ addresses = []
+ for netiface in ifaces:
+ net_info = netifaces.ifaddresses(netiface)
+ if inet_num in net_info:
+ for entry in net_info[inet_num]:
+ if 'addr' in entry and entry['addr'] not in exc_list:
+ addresses.append(entry['addr'])
+
+ if fatal and not addresses:
+ raise Exception("Interface '%s' doesn't have any %s addresses." %
+ (iface, inet_type))
+
+ return sorted(addresses)
+
+
+get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET')
+
+
+def get_iface_from_addr(addr):
+ """Work out on which interface the provided address is configured."""
+ for iface in netifaces.interfaces():
+ addresses = netifaces.ifaddresses(iface)
+ for inet_type in addresses:
+ for _addr in addresses[inet_type]:
+ _addr = _addr['addr']
+ # link local
+ ll_key = re.compile("(.+)%.*")
+ raw = re.match(ll_key, _addr)
+ if raw:
+ _addr = raw.group(1)
+
+ if _addr == addr:
+ log("Address '%s' is configured on iface '%s'" %
+ (addr, iface))
+ return iface
+
+ msg = "Unable to infer net iface on which '%s' is configured" % (addr)
+ raise Exception(msg)
+
+
+def sniff_iface(f):
+ """Ensure decorated function is called with a value for iface.
+
+ If no iface provided, inject net iface inferred from unit private address.
+ """
+ def iface_sniffer(*args, **kwargs):
+ if not kwargs.get('iface', None):
+ kwargs['iface'] = get_iface_from_addr(unit_get('private-address'))
+
+ return f(*args, **kwargs)
+
+ return iface_sniffer
+
+
+@sniff_iface
+def get_ipv6_addr(iface=None, inc_aliases=False, fatal=True, exc_list=None,
+ dynamic_only=True):
+ """Get assigned IPv6 address for a given interface.
+
+ Returns list of addresses found. If no address found, returns empty list.
+
+ If iface is None, we infer the current primary interface by doing a reverse
+ lookup on the unit private-address.
+
+ We currently only support scope global IPv6 addresses i.e. non-temporary
+ addresses. If no global IPv6 address is found, return the first one found
+ in the ipv6 address list.
+
+ :param iface: network interface on which ipv6 address(es) are expected to
+ be found.
+ :param inc_aliases: include alias interfaces in search
+ :param fatal: if True, raise exception if address not found
+ :param exc_list: list of addresses to ignore
+ :param dynamic_only: only recognise dynamic addresses
+ :return: list of ipv6 addresses
+ """
+ addresses = get_iface_addr(iface=iface, inet_type='AF_INET6',
+ inc_aliases=inc_aliases, fatal=fatal,
+ exc_list=exc_list)
+
+ if addresses:
+ global_addrs = []
+ for addr in addresses:
+ key_scope_link_local = re.compile("^fe80::..(.+)%(.+)")
+ m = re.match(key_scope_link_local, addr)
+ if m:
+ eui_64_mac = m.group(1)
+ iface = m.group(2)
+ else:
+ global_addrs.append(addr)
+
+ if global_addrs:
+ # Make sure any found global addresses are not temporary
+ cmd = ['ip', 'addr', 'show', iface]
+ out = subprocess.check_output(cmd).decode('UTF-8')
+ if dynamic_only:
+ key = re.compile("inet6 (.+)/[0-9]+ scope global.* dynamic.*")
+ else:
+ key = re.compile("inet6 (.+)/[0-9]+ scope global.*")
+
+ addrs = []
+ for line in out.split('\n'):
+ line = line.strip()
+ m = re.match(key, line)
+ if m and 'temporary' not in line:
+ # Return the first valid address we find
+ for addr in global_addrs:
+ if m.group(1) == addr:
+ if not dynamic_only or \
+ m.group(1).endswith(eui_64_mac):
+ addrs.append(addr)
+
+ if addrs:
+ return addrs
+
+ if fatal:
+ raise Exception("Interface '%s' does not have a scope global "
+ "non-temporary ipv6 address." % iface)
+
+ return []
+
+
+def get_bridges(vnic_dir='/sys/devices/virtual/net'):
+ """Return a list of bridges on the system."""
+ b_regex = "%s/*/bridge" % vnic_dir
+ return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_regex)]
+
+
+def get_bridge_nics(bridge, vnic_dir='/sys/devices/virtual/net'):
+ """Return a list of nics comprising a given bridge on the system."""
+ brif_regex = "%s/%s/brif/*" % (vnic_dir, bridge)
+ return [x.split('/')[-1] for x in glob.glob(brif_regex)]
+
+
+def is_bridge_member(nic):
+ """Check if a given nic is a member of a bridge."""
+ for bridge in get_bridges():
+ if nic in get_bridge_nics(bridge):
+ return True
+
+ return False
+
+
+def is_ip(address):
+ """
+ Returns True if address is a valid IP address.
+ """
+ try:
+ # Test to see if already an IPv4/IPv6 address
+ address = netaddr.IPAddress(address)
+ return True
+ except (netaddr.AddrFormatError, ValueError):
+ return False
+
+
+def ns_query(address):
+ try:
+ import dns.resolver
+ except ImportError:
+ if six.PY2:
+ apt_install('python-dnspython', fatal=True)
+ else:
+ apt_install('python3-dnspython', fatal=True)
+ import dns.resolver
+
+ if isinstance(address, dns.name.Name):
+ rtype = 'PTR'
+ elif isinstance(address, six.string_types):
+ rtype = 'A'
+ else:
+ return None
+
+ try:
+ answers = dns.resolver.query(address, rtype)
+ except dns.resolver.NXDOMAIN:
+ return None
+
+ if answers:
+ return str(answers[0])
+ return None
+
+
+def get_host_ip(hostname, fallback=None):
+ """
+ Resolves the IP for a given hostname, or returns
+ the input if it is already an IP.
+ """
+ if is_ip(hostname):
+ return hostname
+
+ ip_addr = ns_query(hostname)
+ if not ip_addr:
+ try:
+ ip_addr = socket.gethostbyname(hostname)
+ except Exception:
+ log("Failed to resolve hostname '%s'" % (hostname),
+ level=WARNING)
+ return fallback
+ return ip_addr
+
+
+def get_hostname(address, fqdn=True):
+ """
+ Resolves hostname for given IP, or returns the input
+ if it is already a hostname.
+ """
+ if is_ip(address):
+ try:
+ import dns.reversename
+ except ImportError:
+ if six.PY2:
+ apt_install("python-dnspython", fatal=True)
+ else:
+ apt_install("python3-dnspython", fatal=True)
+ import dns.reversename
+
+ rev = dns.reversename.from_address(address)
+ result = ns_query(rev)
+
+ if not result:
+ try:
+ result = socket.gethostbyaddr(address)[0]
+ except Exception:
+ return None
+ else:
+ result = address
+
+ if fqdn:
+ # strip trailing .
+ if result.endswith('.'):
+ return result[:-1]
+ else:
+ return result
+ else:
+ return result.split('.')[0]
+
+
+def port_has_listener(address, port):
+ """
+ Returns True if the address:port is open and being listened to,
+ else False.
+
+ @param address: an IP address or hostname
+ @param port: integer port
+
+ Note calls 'zc' via a subprocess shell
+ """
+ cmd = ['nc', '-z', address, str(port)]
+ result = subprocess.call(cmd)
+ return not(bool(result))
+
+
+def assert_charm_supports_ipv6():
+ """Check whether we are able to support charms ipv6."""
+ release = lsb_release()['DISTRIB_CODENAME'].lower()
+ if CompareHostReleases(release) < "trusty":
+ raise Exception("IPv6 is not supported in the charms for Ubuntu "
+ "versions less than Trusty 14.04")
+
+
+def get_relation_ip(interface, cidr_network=None):
+ """Return this unit's IP for the given interface.
+
+ Allow for an arbitrary interface to use with network-get to select an IP.
+ Handle all address selection options including passed cidr network and
+ IPv6.
+
+ Usage: get_relation_ip('amqp', cidr_network='10.0.0.0/8')
+
+ @param interface: string name of the relation.
+ @param cidr_network: string CIDR Network to select an address from.
+ @raises Exception if prefer-ipv6 is configured but IPv6 unsupported.
+ @returns IPv6 or IPv4 address
+ """
+ # Select the interface address first
+ # For possible use as a fallback bellow with get_address_in_network
+ try:
+ # Get the interface specific IP
+ address = network_get_primary_address(interface)
+ except NotImplementedError:
+ # If network-get is not available
+ address = get_host_ip(unit_get('private-address'))
+
+ if config('prefer-ipv6'):
+ # Currently IPv6 has priority, eventually we want IPv6 to just be
+ # another network space.
+ assert_charm_supports_ipv6()
+ return get_ipv6_addr()[0]
+ elif cidr_network:
+ # If a specific CIDR network is passed get the address from that
+ # network.
+ return get_address_in_network(cidr_network, address)
+
+ # Return the interface address
+ return address
diff --git a/tests/charmhelpers/contrib/openstack/__init__.py b/tests/charmhelpers/contrib/openstack/__init__.py
new file mode 100644
index 0000000..d7567b8
--- /dev/null
+++ b/tests/charmhelpers/contrib/openstack/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/tests/charmhelpers/contrib/openstack/amulet/__init__.py b/tests/charmhelpers/contrib/openstack/amulet/__init__.py
new file mode 100644
index 0000000..d7567b8
--- /dev/null
+++ b/tests/charmhelpers/contrib/openstack/amulet/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
new file mode 100644
index 0000000..5afbbd8
--- /dev/null
+++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -0,0 +1,354 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import re
+import sys
+import six
+from collections import OrderedDict
+from charmhelpers.contrib.amulet.deployment import (
+ AmuletDeployment
+)
+
+DEBUG = logging.DEBUG
+ERROR = logging.ERROR
+
+
+class OpenStackAmuletDeployment(AmuletDeployment):
+ """OpenStack amulet deployment.
+
+ This class inherits from AmuletDeployment and has additional support
+ that is specifically for use by OpenStack charms.
+ """
+
+ def __init__(self, series=None, openstack=None, source=None,
+ stable=True, log_level=DEBUG):
+ """Initialize the deployment environment."""
+ super(OpenStackAmuletDeployment, self).__init__(series)
+ self.log = self.get_logger(level=log_level)
+ self.log.info('OpenStackAmuletDeployment: init')
+ self.openstack = openstack
+ self.source = source
+ self.stable = stable
+
+ def get_logger(self, name="deployment-logger", level=logging.DEBUG):
+ """Get a logger object that will log to stdout."""
+ log = logging
+ logger = log.getLogger(name)
+ fmt = log.Formatter("%(asctime)s %(funcName)s "
+ "%(levelname)s: %(message)s")
+
+ handler = log.StreamHandler(stream=sys.stdout)
+ handler.setLevel(level)
+ handler.setFormatter(fmt)
+
+ logger.addHandler(handler)
+ logger.setLevel(level)
+
+ return logger
+
+ def _determine_branch_locations(self, other_services):
+ """Determine the branch locations for the other services.
+
+ Determine if the local branch being tested is derived from its
+ stable or next (dev) branch, and based on this, use the corresonding
+ stable or next branches for the other_services."""
+
+ self.log.info('OpenStackAmuletDeployment: determine branch locations')
+
+ # Charms outside the ~openstack-charmers
+ base_charms = {
+ 'mysql': ['trusty'],
+ 'mongodb': ['trusty'],
+ 'nrpe': ['trusty', 'xenial'],
+ }
+
+ for svc in other_services:
+ # If a location has been explicitly set, use it
+ if svc.get('location'):
+ continue
+ if svc['name'] in base_charms:
+ # NOTE: not all charms have support for all series we
+ # want/need to test against, so fix to most recent
+ # that each base charm supports
+ target_series = self.series
+ if self.series not in base_charms[svc['name']]:
+ target_series = base_charms[svc['name']][-1]
+ svc['location'] = 'cs:{}/{}'.format(target_series,
+ svc['name'])
+ elif self.stable:
+ svc['location'] = 'cs:{}/{}'.format(self.series,
+ svc['name'])
+ else:
+ svc['location'] = 'cs:~openstack-charmers-next/{}/{}'.format(
+ self.series,
+ svc['name']
+ )
+
+ return other_services
+
+ def _add_services(self, this_service, other_services, use_source=None,
+ no_origin=None):
+ """Add services to the deployment and optionally set
+ openstack-origin/source.
+
+ :param this_service dict: Service dictionary describing the service
+ whose amulet tests are being run
+ :param other_services dict: List of service dictionaries describing
+ the services needed to support the target
+ service
+ :param use_source list: List of services which use the 'source' config
+ option rather than 'openstack-origin'
+ :param no_origin list: List of services which do not support setting
+ the Cloud Archive.
+ Service Dict:
+ {
+ 'name': str charm-name,
+ 'units': int number of units,
+ 'constraints': dict of juju constraints,
+ 'location': str location of charm,
+ }
+ eg
+ this_service = {
+ 'name': 'openvswitch-odl',
+ 'constraints': {'mem': '8G'},
+ }
+ other_services = [
+ {
+ 'name': 'nova-compute',
+ 'units': 2,
+ 'constraints': {'mem': '4G'},
+ 'location': cs:~bob/xenial/nova-compute
+ },
+ {
+ 'name': 'mysql',
+ 'constraints': {'mem': '2G'},
+ },
+ {'neutron-api-odl'}]
+ use_source = ['mysql']
+ no_origin = ['neutron-api-odl']
+ """
+ self.log.info('OpenStackAmuletDeployment: adding services')
+
+ other_services = self._determine_branch_locations(other_services)
+
+ super(OpenStackAmuletDeployment, self)._add_services(this_service,
+ other_services)
+
+ services = other_services
+ services.append(this_service)
+
+ use_source = use_source or []
+ no_origin = no_origin or []
+
+ # Charms which should use the source config option
+ use_source = list(set(
+ use_source + ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
+ 'ceph-osd', 'ceph-radosgw', 'ceph-mon',
+ 'ceph-proxy', 'percona-cluster', 'lxd']))
+
+ # Charms which can not use openstack-origin, ie. many subordinates
+ no_origin = list(set(
+ no_origin + ['cinder-ceph', 'hacluster', 'neutron-openvswitch',
+ 'nrpe', 'openvswitch-odl', 'neutron-api-odl',
+ 'odl-controller', 'cinder-backup', 'nexentaedge-data',
+ 'nexentaedge-iscsi-gw', 'nexentaedge-swift-gw',
+ 'cinder-nexentaedge', 'nexentaedge-mgmt']))
+
+ if self.openstack:
+ for svc in services:
+ if svc['name'] not in use_source + no_origin:
+ config = {'openstack-origin': self.openstack}
+ self.d.configure(svc['name'], config)
+
+ if self.source:
+ for svc in services:
+ if svc['name'] in use_source and svc['name'] not in no_origin:
+ config = {'source': self.source}
+ self.d.configure(svc['name'], config)
+
+ def _configure_services(self, configs):
+ """Configure all of the services."""
+ self.log.info('OpenStackAmuletDeployment: configure services')
+ for service, config in six.iteritems(configs):
+ self.d.configure(service, config)
+
+ def _auto_wait_for_status(self, message=None, exclude_services=None,
+ include_only=None, timeout=None):
+ """Wait for all units to have a specific extended status, except
+ for any defined as excluded. Unless specified via message, any
+ status containing any case of 'ready' will be considered a match.
+
+ Examples of message usage:
+
+ Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
+ message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
+
+ Wait for all units to reach this status (exact match):
+ message = re.compile('^Unit is ready and clustered$')
+
+ Wait for all units to reach any one of these (exact match):
+ message = re.compile('Unit is ready|OK|Ready')
+
+ Wait for at least one unit to reach this status (exact match):
+ message = {'ready'}
+
+ See Amulet's sentry.wait_for_messages() for message usage detail.
+ https://github.com/juju/amulet/blob/master/amulet/sentry.py
+
+ :param message: Expected status match
+ :param exclude_services: List of juju service names to ignore,
+ not to be used in conjuction with include_only.
+ :param include_only: List of juju service names to exclusively check,
+ not to be used in conjuction with exclude_services.
+ :param timeout: Maximum time in seconds to wait for status match
+ :returns: None. Raises if timeout is hit.
+ """
+ if not timeout:
+ timeout = int(os.environ.get('AMULET_SETUP_TIMEOUT', 1800))
+ self.log.info('Waiting for extended status on units for {}s...'
+ ''.format(timeout))
+
+ all_services = self.d.services.keys()
+
+ if exclude_services and include_only:
+ raise ValueError('exclude_services can not be used '
+ 'with include_only')
+
+ if message:
+ if isinstance(message, re._pattern_type):
+ match = message.pattern
+ else:
+ match = message
+
+ self.log.debug('Custom extended status wait match: '
+ '{}'.format(match))
+ else:
+ self.log.debug('Default extended status wait match: contains '
+ 'READY (case-insensitive)')
+ message = re.compile('.*ready.*', re.IGNORECASE)
+
+ if exclude_services:
+ self.log.debug('Excluding services from extended status match: '
+ '{}'.format(exclude_services))
+ else:
+ exclude_services = []
+
+ if include_only:
+ services = include_only
+ else:
+ services = list(set(all_services) - set(exclude_services))
+
+ self.log.debug('Waiting up to {}s for extended status on services: '
+ '{}'.format(timeout, services))
+ service_messages = {service: message for service in services}
+
+ # Check for idleness
+ self.d.sentry.wait(timeout=timeout)
+ # Check for error states and bail early
+ self.d.sentry.wait_for_status(self.d.juju_env, services, timeout=timeout)
+ # Check for ready messages
+ self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+
+ self.log.info('OK')
+
+ def _get_openstack_release(self):
+ """Get openstack release.
+
+ Return an integer representing the enum value of the openstack
+ release.
+ """
+ # Must be ordered by OpenStack release (not by Ubuntu release):
+ (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty,
+ self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton,
+ self.yakkety_newton, self.xenial_ocata, self.zesty_ocata,
+ self.xenial_pike, self.artful_pike, self.xenial_queens,
+ self.bionic_queens,) = range(13)
+
+ releases = {
+ ('trusty', None): self.trusty_icehouse,
+ ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
+ ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
+ ('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka,
+ ('xenial', None): self.xenial_mitaka,
+ ('xenial', 'cloud:xenial-newton'): self.xenial_newton,
+ ('xenial', 'cloud:xenial-ocata'): self.xenial_ocata,
+ ('xenial', 'cloud:xenial-pike'): self.xenial_pike,
+ ('xenial', 'cloud:xenial-queens'): self.xenial_queens,
+ ('yakkety', None): self.yakkety_newton,
+ ('zesty', None): self.zesty_ocata,
+ ('artful', None): self.artful_pike,
+ ('bionic', None): self.bionic_queens,
+ }
+ return releases[(self.series, self.openstack)]
+
+ def _get_openstack_release_string(self):
+ """Get openstack release string.
+
+ Return a string representing the openstack release.
+ """
+ releases = OrderedDict([
+ ('trusty', 'icehouse'),
+ ('xenial', 'mitaka'),
+ ('yakkety', 'newton'),
+ ('zesty', 'ocata'),
+ ('artful', 'pike'),
+ ('bionic', 'queens'),
+ ])
+ if self.openstack:
+ os_origin = self.openstack.split(':')[1]
+ return os_origin.split('%s-' % self.series)[1].split('/')[0]
+ else:
+ return releases[self.series]
+
+ def get_ceph_expected_pools(self, radosgw=False):
+ """Return a list of expected ceph pools in a ceph + cinder + glance
+ test scenario, based on OpenStack release and whether ceph radosgw
+ is flagged as present or not."""
+
+ if self._get_openstack_release() == self.trusty_icehouse:
+ # Icehouse
+ pools = [
+ 'data',
+ 'metadata',
+ 'rbd',
+ 'cinder-ceph',
+ 'glance'
+ ]
+ elif (self.trusty_kilo <= self._get_openstack_release() <=
+ self.zesty_ocata):
+ # Kilo through Ocata
+ pools = [
+ 'rbd',
+ 'cinder-ceph',
+ 'glance'
+ ]
+ else:
+ # Pike and later
+ pools = [
+ 'cinder-ceph',
+ 'glance'
+ ]
+
+ if radosgw:
+ pools.extend([
+ '.rgw.root',
+ '.rgw.control',
+ '.rgw',
+ '.rgw.gc',
+ '.users.uid'
+ ])
+
+ return pools
diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py
new file mode 100644
index 0000000..b71b2b1
--- /dev/null
+++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py
@@ -0,0 +1,1335 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import amulet
+import json
+import logging
+import os
+import re
+import six
+import time
+import urllib
+import urlparse
+
+import cinderclient.v1.client as cinder_client
+import cinderclient.v2.client as cinder_clientv2
+import glanceclient.v1.client as glance_client
+import heatclient.v1.client as heat_client
+from keystoneclient.v2_0 import client as keystone_client
+from keystoneauth1.identity import (
+ v3,
+ v2,
+)
+from keystoneauth1 import session as keystone_session
+from keystoneclient.v3 import client as keystone_client_v3
+from novaclient import exceptions
+
+import novaclient.client as nova_client
+import novaclient
+import pika
+import swiftclient
+
+from charmhelpers.contrib.amulet.utils import (
+ AmuletUtils
+)
+from charmhelpers.core.host import CompareHostReleases
+
+DEBUG = logging.DEBUG
+ERROR = logging.ERROR
+
+NOVA_CLIENT_VERSION = "2"
+
+
+class OpenStackAmuletUtils(AmuletUtils):
+ """OpenStack amulet utilities.
+
+ This class inherits from AmuletUtils and has additional support
+ that is specifically for use by OpenStack charm tests.
+ """
+
+ def __init__(self, log_level=ERROR):
+ """Initialize the deployment environment."""
+ super(OpenStackAmuletUtils, self).__init__(log_level)
+
+ def validate_endpoint_data(self, endpoints, admin_port, internal_port,
+ public_port, expected):
+ """Validate endpoint data.
+
+ Validate actual endpoint data vs expected endpoint data. The ports
+ are used to find the matching endpoint.
+ """
+ self.log.debug('Validating endpoint data...')
+ self.log.debug('actual: {}'.format(repr(endpoints)))
+ found = False
+ for ep in endpoints:
+ self.log.debug('endpoint: {}'.format(repr(ep)))
+ if (admin_port in ep.adminurl and
+ internal_port in ep.internalurl and
+ public_port in ep.publicurl):
+ found = True
+ actual = {'id': ep.id,
+ 'region': ep.region,
+ 'adminurl': ep.adminurl,
+ 'internalurl': ep.internalurl,
+ 'publicurl': ep.publicurl,
+ 'service_id': ep.service_id}
+ ret = self._validate_dict_data(expected, actual)
+ if ret:
+ return 'unexpected endpoint data - {}'.format(ret)
+
+ if not found:
+ return 'endpoint not found'
+
+ def validate_v3_endpoint_data(self, endpoints, admin_port, internal_port,
+ public_port, expected):
+ """Validate keystone v3 endpoint data.
+
+ Validate the v3 endpoint data which has changed from v2. The
+ ports are used to find the matching endpoint.
+
+ The new v3 endpoint data looks like:
+
+ ['},
+ region=RegionOne,
+ region_id=RegionOne,
+ service_id=17f842a0dc084b928e476fafe67e4095,
+ url=http://10.5.6.5:9312>,
+ '},
+ region=RegionOne,
+ region_id=RegionOne,
+ service_id=72fc8736fb41435e8b3584205bb2cfa3,
+ url=http://10.5.6.6:35357/v3>,
+ ... ]
+ """
+ self.log.debug('Validating v3 endpoint data...')
+ self.log.debug('actual: {}'.format(repr(endpoints)))
+ found = []
+ for ep in endpoints:
+ self.log.debug('endpoint: {}'.format(repr(ep)))
+ if ((admin_port in ep.url and ep.interface == 'admin') or
+ (internal_port in ep.url and ep.interface == 'internal') or
+ (public_port in ep.url and ep.interface == 'public')):
+ found.append(ep.interface)
+ # note we ignore the links member.
+ actual = {'id': ep.id,
+ 'region': ep.region,
+ 'region_id': ep.region_id,
+ 'interface': self.not_null,
+ 'url': ep.url,
+ 'service_id': ep.service_id, }
+ ret = self._validate_dict_data(expected, actual)
+ if ret:
+ return 'unexpected endpoint data - {}'.format(ret)
+
+ if len(found) != 3:
+ return 'Unexpected number of endpoints found'
+
+ def validate_svc_catalog_endpoint_data(self, expected, actual):
+ """Validate service catalog endpoint data.
+
+ Validate a list of actual service catalog endpoints vs a list of
+ expected service catalog endpoints.
+ """
+ self.log.debug('Validating service catalog endpoint data...')
+ self.log.debug('actual: {}'.format(repr(actual)))
+ for k, v in six.iteritems(expected):
+ if k in actual:
+ ret = self._validate_dict_data(expected[k][0], actual[k][0])
+ if ret:
+ return self.endpoint_error(k, ret)
+ else:
+ return "endpoint {} does not exist".format(k)
+ return ret
+
+ def validate_v3_svc_catalog_endpoint_data(self, expected, actual):
+ """Validate the keystone v3 catalog endpoint data.
+
+ Validate a list of dictinaries that make up the keystone v3 service
+ catalogue.
+
+ It is in the form of:
+
+
+ {u'identity': [{u'id': u'48346b01c6804b298cdd7349aadb732e',
+ u'interface': u'admin',
+ u'region': u'RegionOne',
+ u'region_id': u'RegionOne',
+ u'url': u'http://10.5.5.224:35357/v3'},
+ {u'id': u'8414f7352a4b47a69fddd9dbd2aef5cf',
+ u'interface': u'public',
+ u'region': u'RegionOne',
+ u'region_id': u'RegionOne',
+ u'url': u'http://10.5.5.224:5000/v3'},
+ {u'id': u'd5ca31440cc24ee1bf625e2996fb6a5b',
+ u'interface': u'internal',
+ u'region': u'RegionOne',
+ u'region_id': u'RegionOne',
+ u'url': u'http://10.5.5.224:5000/v3'}],
+ u'key-manager': [{u'id': u'68ebc17df0b045fcb8a8a433ebea9e62',
+ u'interface': u'public',
+ u'region': u'RegionOne',
+ u'region_id': u'RegionOne',
+ u'url': u'http://10.5.5.223:9311'},
+ {u'id': u'9cdfe2a893c34afd8f504eb218cd2f9d',
+ u'interface': u'internal',
+ u'region': u'RegionOne',
+ u'region_id': u'RegionOne',
+ u'url': u'http://10.5.5.223:9311'},
+ {u'id': u'f629388955bc407f8b11d8b7ca168086',
+ u'interface': u'admin',
+ u'region': u'RegionOne',
+ u'region_id': u'RegionOne',
+ u'url': u'http://10.5.5.223:9312'}]}
+
+ Note, that an added complication is that the order of admin, public,
+ internal against 'interface' in each region.
+
+ Thus, the function sorts the expected and actual lists using the
+ interface key as a sort key, prior to the comparison.
+ """
+ self.log.debug('Validating v3 service catalog endpoint data...')
+ self.log.debug('actual: {}'.format(repr(actual)))
+ for k, v in six.iteritems(expected):
+ if k in actual:
+ l_expected = sorted(v, key=lambda x: x['interface'])
+ l_actual = sorted(actual[k], key=lambda x: x['interface'])
+ if len(l_actual) != len(l_expected):
+ return ("endpoint {} has differing number of interfaces "
+ " - expected({}), actual({})"
+ .format(k, len(l_expected), len(l_actual)))
+ for i_expected, i_actual in zip(l_expected, l_actual):
+ self.log.debug("checking interface {}"
+ .format(i_expected['interface']))
+ ret = self._validate_dict_data(i_expected, i_actual)
+ if ret:
+ return self.endpoint_error(k, ret)
+ else:
+ return "endpoint {} does not exist".format(k)
+ return ret
+
+ def validate_tenant_data(self, expected, actual):
+ """Validate tenant data.
+
+ Validate a list of actual tenant data vs list of expected tenant
+ data.
+ """
+ self.log.debug('Validating tenant data...')
+ self.log.debug('actual: {}'.format(repr(actual)))
+ for e in expected:
+ found = False
+ for act in actual:
+ a = {'enabled': act.enabled, 'description': act.description,
+ 'name': act.name, 'id': act.id}
+ if e['name'] == a['name']:
+ found = True
+ ret = self._validate_dict_data(e, a)
+ if ret:
+ return "unexpected tenant data - {}".format(ret)
+ if not found:
+ return "tenant {} does not exist".format(e['name'])
+ return ret
+
+ def validate_role_data(self, expected, actual):
+ """Validate role data.
+
+ Validate a list of actual role data vs a list of expected role
+ data.
+ """
+ self.log.debug('Validating role data...')
+ self.log.debug('actual: {}'.format(repr(actual)))
+ for e in expected:
+ found = False
+ for act in actual:
+ a = {'name': act.name, 'id': act.id}
+ if e['name'] == a['name']:
+ found = True
+ ret = self._validate_dict_data(e, a)
+ if ret:
+ return "unexpected role data - {}".format(ret)
+ if not found:
+ return "role {} does not exist".format(e['name'])
+ return ret
+
+ def validate_user_data(self, expected, actual, api_version=None):
+ """Validate user data.
+
+ Validate a list of actual user data vs a list of expected user
+ data.
+ """
+ self.log.debug('Validating user data...')
+ self.log.debug('actual: {}'.format(repr(actual)))
+ for e in expected:
+ found = False
+ for act in actual:
+ if e['name'] == act.name:
+ a = {'enabled': act.enabled, 'name': act.name,
+ 'email': act.email, 'id': act.id}
+ if api_version == 3:
+ a['default_project_id'] = getattr(act,
+ 'default_project_id',
+ 'none')
+ else:
+ a['tenantId'] = act.tenantId
+ found = True
+ ret = self._validate_dict_data(e, a)
+ if ret:
+ return "unexpected user data - {}".format(ret)
+ if not found:
+ return "user {} does not exist".format(e['name'])
+ return ret
+
+ def validate_flavor_data(self, expected, actual):
+ """Validate flavor data.
+
+ Validate a list of actual flavors vs a list of expected flavors.
+ """
+ self.log.debug('Validating flavor data...')
+ self.log.debug('actual: {}'.format(repr(actual)))
+ act = [a.name for a in actual]
+ return self._validate_list_data(expected, act)
+
+ def tenant_exists(self, keystone, tenant):
+ """Return True if tenant exists."""
+ self.log.debug('Checking if tenant exists ({})...'.format(tenant))
+ return tenant in [t.name for t in keystone.tenants.list()]
+
+ def keystone_wait_for_propagation(self, sentry_relation_pairs,
+ api_version):
+ """Iterate over list of sentry and relation tuples and verify that
+ api_version has the expected value.
+
+ :param sentry_relation_pairs: list of sentry, relation name tuples used
+ for monitoring propagation of relation
+ data
+ :param api_version: api_version to expect in relation data
+ :returns: None if successful. Raise on error.
+ """
+ for (sentry, relation_name) in sentry_relation_pairs:
+ rel = sentry.relation('identity-service',
+ relation_name)
+ self.log.debug('keystone relation data: {}'.format(rel))
+ if rel.get('api_version') != str(api_version):
+ raise Exception("api_version not propagated through relation"
+ " data yet ('{}' != '{}')."
+ "".format(rel['api_version'], api_version))
+
+ def keystone_configure_api_version(self, sentry_relation_pairs, deployment,
+ api_version):
+ """Configure preferred-api-version of keystone in deployment and
+ monitor provided list of relation objects for propagation
+ before returning to caller.
+
+ :param sentry_relation_pairs: list of sentry, relation tuples used for
+ monitoring propagation of relation data
+ :param deployment: deployment to configure
+ :param api_version: value preferred-api-version will be set to
+ :returns: None if successful. Raise on error.
+ """
+ self.log.debug("Setting keystone preferred-api-version: '{}'"
+ "".format(api_version))
+
+ config = {'preferred-api-version': api_version}
+ deployment.d.configure('keystone', config)
+ deployment._auto_wait_for_status()
+ self.keystone_wait_for_propagation(sentry_relation_pairs, api_version)
+
+ def authenticate_cinder_admin(self, keystone_sentry, username,
+ password, tenant, api_version=2):
+ """Authenticates admin user with cinder."""
+ # NOTE(beisner): cinder python client doesn't accept tokens.
+ keystone_ip = keystone_sentry.info['public-address']
+ ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8'))
+ _clients = {
+ 1: cinder_client.Client,
+ 2: cinder_clientv2.Client}
+ return _clients[api_version](username, password, tenant, ept)
+
+ def authenticate_keystone(self, keystone_ip, username, password,
+ api_version=False, admin_port=False,
+ user_domain_name=None, domain_name=None,
+ project_domain_name=None, project_name=None):
+ """Authenticate with Keystone"""
+ self.log.debug('Authenticating with keystone...')
+ port = 5000
+ if admin_port:
+ port = 35357
+ base_ep = "http://{}:{}".format(keystone_ip.strip().decode('utf-8'),
+ port)
+ if not api_version or api_version == 2:
+ ep = base_ep + "/v2.0"
+ auth = v2.Password(
+ username=username,
+ password=password,
+ tenant_name=project_name,
+ auth_url=ep
+ )
+ sess = keystone_session.Session(auth=auth)
+ client = keystone_client.Client(session=sess)
+ # This populates the client.service_catalog
+ client.auth_ref = auth.get_access(sess)
+ return client
+ else:
+ ep = base_ep + "/v3"
+ auth = v3.Password(
+ user_domain_name=user_domain_name,
+ username=username,
+ password=password,
+ domain_name=domain_name,
+ project_domain_name=project_domain_name,
+ project_name=project_name,
+ auth_url=ep
+ )
+ sess = keystone_session.Session(auth=auth)
+ client = keystone_client_v3.Client(session=sess)
+ # This populates the client.service_catalog
+ client.auth_ref = auth.get_access(sess)
+ return client
+
+ def authenticate_keystone_admin(self, keystone_sentry, user, password,
+ tenant=None, api_version=None,
+ keystone_ip=None, user_domain_name=None,
+ project_domain_name=None,
+ project_name=None):
+ """Authenticates admin user with the keystone admin endpoint."""
+ self.log.debug('Authenticating keystone admin...')
+ if not keystone_ip:
+ keystone_ip = keystone_sentry.info['public-address']
+
+ # To support backward compatibility usage of this function
+ if not project_name:
+ project_name = tenant
+ if api_version == 3 and not user_domain_name:
+ user_domain_name = 'admin_domain'
+ if api_version == 3 and not project_domain_name:
+ project_domain_name = 'admin_domain'
+ if api_version == 3 and not project_name:
+ project_name = 'admin'
+
+ return self.authenticate_keystone(
+ keystone_ip, user, password,
+ api_version=api_version,
+ user_domain_name=user_domain_name,
+ project_domain_name=project_domain_name,
+ project_name=project_name,
+ admin_port=True)
+
+ def authenticate_keystone_user(self, keystone, user, password, tenant):
+ """Authenticates a regular user with the keystone public endpoint."""
+ self.log.debug('Authenticating keystone user ({})...'.format(user))
+ ep = keystone.service_catalog.url_for(service_type='identity',
+ interface='publicURL')
+ keystone_ip = urlparse.urlparse(ep).hostname
+
+ return self.authenticate_keystone(keystone_ip, user, password,
+ project_name=tenant)
+
+ def authenticate_glance_admin(self, keystone):
+ """Authenticates admin user with glance."""
+ self.log.debug('Authenticating glance admin...')
+ ep = keystone.service_catalog.url_for(service_type='image',
+ interface='adminURL')
+ if keystone.session:
+ return glance_client.Client(ep, session=keystone.session)
+ else:
+ return glance_client.Client(ep, token=keystone.auth_token)
+
+ def authenticate_heat_admin(self, keystone):
+ """Authenticates the admin user with heat."""
+ self.log.debug('Authenticating heat admin...')
+ ep = keystone.service_catalog.url_for(service_type='orchestration',
+ interface='publicURL')
+ if keystone.session:
+ return heat_client.Client(endpoint=ep, session=keystone.session)
+ else:
+ return heat_client.Client(endpoint=ep, token=keystone.auth_token)
+
+ def authenticate_nova_user(self, keystone, user, password, tenant):
+ """Authenticates a regular user with nova-api."""
+ self.log.debug('Authenticating nova user ({})...'.format(user))
+ ep = keystone.service_catalog.url_for(service_type='identity',
+ interface='publicURL')
+ if keystone.session:
+ return nova_client.Client(NOVA_CLIENT_VERSION,
+ session=keystone.session,
+ auth_url=ep)
+ elif novaclient.__version__[0] >= "7":
+ return nova_client.Client(NOVA_CLIENT_VERSION,
+ username=user, password=password,
+ project_name=tenant, auth_url=ep)
+ else:
+ return nova_client.Client(NOVA_CLIENT_VERSION,
+ username=user, api_key=password,
+ project_id=tenant, auth_url=ep)
+
+ def authenticate_swift_user(self, keystone, user, password, tenant):
+ """Authenticates a regular user with swift api."""
+ self.log.debug('Authenticating swift user ({})...'.format(user))
+ ep = keystone.service_catalog.url_for(service_type='identity',
+ interface='publicURL')
+ if keystone.session:
+ return swiftclient.Connection(session=keystone.session)
+ else:
+ return swiftclient.Connection(authurl=ep,
+ user=user,
+ key=password,
+ tenant_name=tenant,
+ auth_version='2.0')
+
+ def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto",
+ ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True):
+ """Create the specified flavor."""
+ try:
+ nova.flavors.find(name=name)
+ except (exceptions.NotFound, exceptions.NoUniqueMatch):
+ self.log.debug('Creating flavor ({})'.format(name))
+ nova.flavors.create(name, ram, vcpus, disk, flavorid,
+ ephemeral, swap, rxtx_factor, is_public)
+
+ def create_cirros_image(self, glance, image_name):
+ """Download the latest cirros image and upload it to glance,
+ validate and return a resource pointer.
+
+ :param glance: pointer to authenticated glance connection
+ :param image_name: display name for new image
+ :returns: glance image pointer
+ """
+ self.log.debug('Creating glance cirros image '
+ '({})...'.format(image_name))
+
+ # Download cirros image
+ http_proxy = os.getenv('AMULET_HTTP_PROXY')
+ self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
+ if http_proxy:
+ proxies = {'http': http_proxy}
+ opener = urllib.FancyURLopener(proxies)
+ else:
+ opener = urllib.FancyURLopener()
+
+ f = opener.open('http://download.cirros-cloud.net/version/released')
+ version = f.read().strip()
+ cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
+ local_path = os.path.join('tests', cirros_img)
+
+ if not os.path.exists(local_path):
+ cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
+ version, cirros_img)
+ opener.retrieve(cirros_url, local_path)
+ f.close()
+
+ # Create glance image
+ with open(local_path) as f:
+ image = glance.images.create(name=image_name, is_public=True,
+ disk_format='qcow2',
+ container_format='bare', data=f)
+
+ # Wait for image to reach active status
+ img_id = image.id
+ ret = self.resource_reaches_status(glance.images, img_id,
+ expected_stat='active',
+ msg='Image status wait')
+ if not ret:
+ msg = 'Glance image failed to reach expected state.'
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ # Re-validate new image
+ self.log.debug('Validating image attributes...')
+ val_img_name = glance.images.get(img_id).name
+ val_img_stat = glance.images.get(img_id).status
+ val_img_pub = glance.images.get(img_id).is_public
+ val_img_cfmt = glance.images.get(img_id).container_format
+ val_img_dfmt = glance.images.get(img_id).disk_format
+ msg_attr = ('Image attributes - name:{} public:{} id:{} stat:{} '
+ 'container fmt:{} disk fmt:{}'.format(
+ val_img_name, val_img_pub, img_id,
+ val_img_stat, val_img_cfmt, val_img_dfmt))
+
+ if val_img_name == image_name and val_img_stat == 'active' \
+ and val_img_pub is True and val_img_cfmt == 'bare' \
+ and val_img_dfmt == 'qcow2':
+ self.log.debug(msg_attr)
+ else:
+ msg = ('Volume validation failed, {}'.format(msg_attr))
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ return image
+
+ def delete_image(self, glance, image):
+ """Delete the specified image."""
+
+ # /!\ DEPRECATION WARNING
+ self.log.warn('/!\\ DEPRECATION WARNING: use '
+ 'delete_resource instead of delete_image.')
+ self.log.debug('Deleting glance image ({})...'.format(image))
+ return self.delete_resource(glance.images, image, msg='glance image')
+
+ def create_instance(self, nova, image_name, instance_name, flavor):
+ """Create the specified instance."""
+ self.log.debug('Creating instance '
+ '({}|{}|{})'.format(instance_name, image_name, flavor))
+ image = nova.glance.find_image(image_name)
+ flavor = nova.flavors.find(name=flavor)
+ instance = nova.servers.create(name=instance_name, image=image,
+ flavor=flavor)
+
+ count = 1
+ status = instance.status
+ while status != 'ACTIVE' and count < 60:
+ time.sleep(3)
+ instance = nova.servers.get(instance.id)
+ status = instance.status
+ self.log.debug('instance status: {}'.format(status))
+ count += 1
+
+ if status != 'ACTIVE':
+ self.log.error('instance creation timed out')
+ return None
+
+ return instance
+
+ def delete_instance(self, nova, instance):
+ """Delete the specified instance."""
+
+ # /!\ DEPRECATION WARNING
+ self.log.warn('/!\\ DEPRECATION WARNING: use '
+ 'delete_resource instead of delete_instance.')
+ self.log.debug('Deleting instance ({})...'.format(instance))
+ return self.delete_resource(nova.servers, instance,
+ msg='nova instance')
+
+ def create_or_get_keypair(self, nova, keypair_name="testkey"):
+ """Create a new keypair, or return pointer if it already exists."""
+ try:
+ _keypair = nova.keypairs.get(keypair_name)
+ self.log.debug('Keypair ({}) already exists, '
+ 'using it.'.format(keypair_name))
+ return _keypair
+ except Exception:
+ self.log.debug('Keypair ({}) does not exist, '
+ 'creating it.'.format(keypair_name))
+
+ _keypair = nova.keypairs.create(name=keypair_name)
+ return _keypair
+
+ def _get_cinder_obj_name(self, cinder_object):
+ """Retrieve name of cinder object.
+
+ :param cinder_object: cinder snapshot or volume object
+ :returns: str cinder object name
+ """
+ # v1 objects store name in 'display_name' attr but v2+ use 'name'
+ try:
+ return cinder_object.display_name
+ except AttributeError:
+ return cinder_object.name
+
+ def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1,
+ img_id=None, src_vol_id=None, snap_id=None):
+ """Create cinder volume, optionally from a glance image, OR
+ optionally as a clone of an existing volume, OR optionally
+ from a snapshot. Wait for the new volume status to reach
+ the expected status, validate and return a resource pointer.
+
+ :param vol_name: cinder volume display name
+ :param vol_size: size in gigabytes
+ :param img_id: optional glance image id
+ :param src_vol_id: optional source volume id to clone
+ :param snap_id: optional snapshot id to use
+ :returns: cinder volume pointer
+ """
+ # Handle parameter input and avoid impossible combinations
+ if img_id and not src_vol_id and not snap_id:
+ # Create volume from image
+ self.log.debug('Creating cinder volume from glance image...')
+ bootable = 'true'
+ elif src_vol_id and not img_id and not snap_id:
+ # Clone an existing volume
+ self.log.debug('Cloning cinder volume...')
+ bootable = cinder.volumes.get(src_vol_id).bootable
+ elif snap_id and not src_vol_id and not img_id:
+ # Create volume from snapshot
+ self.log.debug('Creating cinder volume from snapshot...')
+ snap = cinder.volume_snapshots.find(id=snap_id)
+ vol_size = snap.size
+ snap_vol_id = cinder.volume_snapshots.get(snap_id).volume_id
+ bootable = cinder.volumes.get(snap_vol_id).bootable
+ elif not img_id and not src_vol_id and not snap_id:
+ # Create volume
+ self.log.debug('Creating cinder volume...')
+ bootable = 'false'
+ else:
+ # Impossible combination of parameters
+ msg = ('Invalid method use - name:{} size:{} img_id:{} '
+ 'src_vol_id:{} snap_id:{}'.format(vol_name, vol_size,
+ img_id, src_vol_id,
+ snap_id))
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ # Create new volume
+ try:
+ vol_new = cinder.volumes.create(display_name=vol_name,
+ imageRef=img_id,
+ size=vol_size,
+ source_volid=src_vol_id,
+ snapshot_id=snap_id)
+ vol_id = vol_new.id
+ except TypeError:
+ vol_new = cinder.volumes.create(name=vol_name,
+ imageRef=img_id,
+ size=vol_size,
+ source_volid=src_vol_id,
+ snapshot_id=snap_id)
+ vol_id = vol_new.id
+ except Exception as e:
+ msg = 'Failed to create volume: {}'.format(e)
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ # Wait for volume to reach available status
+ ret = self.resource_reaches_status(cinder.volumes, vol_id,
+ expected_stat="available",
+ msg="Volume status wait")
+ if not ret:
+ msg = 'Cinder volume failed to reach expected state.'
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ # Re-validate new volume
+ self.log.debug('Validating volume attributes...')
+ val_vol_name = self._get_cinder_obj_name(cinder.volumes.get(vol_id))
+ val_vol_boot = cinder.volumes.get(vol_id).bootable
+ val_vol_stat = cinder.volumes.get(vol_id).status
+ val_vol_size = cinder.volumes.get(vol_id).size
+ msg_attr = ('Volume attributes - name:{} id:{} stat:{} boot:'
+ '{} size:{}'.format(val_vol_name, vol_id,
+ val_vol_stat, val_vol_boot,
+ val_vol_size))
+
+ if val_vol_boot == bootable and val_vol_stat == 'available' \
+ and val_vol_name == vol_name and val_vol_size == vol_size:
+ self.log.debug(msg_attr)
+ else:
+ msg = ('Volume validation failed, {}'.format(msg_attr))
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ return vol_new
+
+ def delete_resource(self, resource, resource_id,
+ msg="resource", max_wait=120):
+ """Delete one openstack resource, such as one instance, keypair,
+ image, volume, stack, etc., and confirm deletion within max wait time.
+
+ :param resource: pointer to os resource type, ex:glance_client.images
+ :param resource_id: unique name or id for the openstack resource
+ :param msg: text to identify purpose in logging
+ :param max_wait: maximum wait time in seconds
+ :returns: True if successful, otherwise False
+ """
+ self.log.debug('Deleting OpenStack resource '
+ '{} ({})'.format(resource_id, msg))
+ num_before = len(list(resource.list()))
+ resource.delete(resource_id)
+
+ tries = 0
+ num_after = len(list(resource.list()))
+ while num_after != (num_before - 1) and tries < (max_wait / 4):
+ self.log.debug('{} delete check: '
+ '{} [{}:{}] {}'.format(msg, tries,
+ num_before,
+ num_after,
+ resource_id))
+ time.sleep(4)
+ num_after = len(list(resource.list()))
+ tries += 1
+
+ self.log.debug('{}: expected, actual count = {}, '
+ '{}'.format(msg, num_before - 1, num_after))
+
+ if num_after == (num_before - 1):
+ return True
+ else:
+ self.log.error('{} delete timed out'.format(msg))
+ return False
+
+ def resource_reaches_status(self, resource, resource_id,
+ expected_stat='available',
+ msg='resource', max_wait=120):
+ """Wait for an openstack resources status to reach an
+ expected status within a specified time. Useful to confirm that
+ nova instances, cinder vols, snapshots, glance images, heat stacks
+ and other resources eventually reach the expected status.
+
+ :param resource: pointer to os resource type, ex: heat_client.stacks
+ :param resource_id: unique id for the openstack resource
+ :param expected_stat: status to expect resource to reach
+ :param msg: text to identify purpose in logging
+ :param max_wait: maximum wait time in seconds
+ :returns: True if successful, False if status is not reached
+ """
+
+ tries = 0
+ resource_stat = resource.get(resource_id).status
+ while resource_stat != expected_stat and tries < (max_wait / 4):
+ self.log.debug('{} status check: '
+ '{} [{}:{}] {}'.format(msg, tries,
+ resource_stat,
+ expected_stat,
+ resource_id))
+ time.sleep(4)
+ resource_stat = resource.get(resource_id).status
+ tries += 1
+
+ self.log.debug('{}: expected, actual status = {}, '
+ '{}'.format(msg, resource_stat, expected_stat))
+
+ if resource_stat == expected_stat:
+ return True
+ else:
+ self.log.debug('{} never reached expected status: '
+ '{}'.format(resource_id, expected_stat))
+ return False
+
+ def get_ceph_osd_id_cmd(self, index):
+ """Produce a shell command that will return a ceph-osd id."""
+ return ("`initctl list | grep 'ceph-osd ' | "
+ "awk 'NR=={} {{ print $2 }}' | "
+ "grep -o '[0-9]*'`".format(index + 1))
+
+ def get_ceph_pools(self, sentry_unit):
+ """Return a dict of ceph pools from a single ceph unit, with
+ pool name as keys, pool id as vals."""
+ pools = {}
+ cmd = 'sudo ceph osd lspools'
+ output, code = sentry_unit.run(cmd)
+ if code != 0:
+ msg = ('{} `{}` returned {} '
+ '{}'.format(sentry_unit.info['unit_name'],
+ cmd, code, output))
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance,
+ for pool in str(output).split(','):
+ pool_id_name = pool.split(' ')
+ if len(pool_id_name) == 2:
+ pool_id = pool_id_name[0]
+ pool_name = pool_id_name[1]
+ pools[pool_name] = int(pool_id)
+
+ self.log.debug('Pools on {}: {}'.format(sentry_unit.info['unit_name'],
+ pools))
+ return pools
+
+ def get_ceph_df(self, sentry_unit):
+ """Return dict of ceph df json output, including ceph pool state.
+
+ :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+ :returns: Dict of ceph df output
+ """
+ cmd = 'sudo ceph df --format=json'
+ output, code = sentry_unit.run(cmd)
+ if code != 0:
+ msg = ('{} `{}` returned {} '
+ '{}'.format(sentry_unit.info['unit_name'],
+ cmd, code, output))
+ amulet.raise_status(amulet.FAIL, msg=msg)
+ return json.loads(output)
+
+ def get_ceph_pool_sample(self, sentry_unit, pool_id=0):
+ """Take a sample of attributes of a ceph pool, returning ceph
+ pool name, object count and disk space used for the specified
+ pool ID number.
+
+ :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+ :param pool_id: Ceph pool ID
+ :returns: List of pool name, object count, kb disk space used
+ """
+ df = self.get_ceph_df(sentry_unit)
+ pool_name = df['pools'][pool_id]['name']
+ obj_count = df['pools'][pool_id]['stats']['objects']
+ kb_used = df['pools'][pool_id]['stats']['kb_used']
+ self.log.debug('Ceph {} pool (ID {}): {} objects, '
+ '{} kb used'.format(pool_name, pool_id,
+ obj_count, kb_used))
+ return pool_name, obj_count, kb_used
+
+ def validate_ceph_pool_samples(self, samples, sample_type="resource pool"):
+ """Validate ceph pool samples taken over time, such as pool
+ object counts or pool kb used, before adding, after adding, and
+ after deleting items which affect those pool attributes. The
+ 2nd element is expected to be greater than the 1st; 3rd is expected
+ to be less than the 2nd.
+
+ :param samples: List containing 3 data samples
+ :param sample_type: String for logging and usage context
+ :returns: None if successful, Failure message otherwise
+ """
+ original, created, deleted = range(3)
+ if samples[created] <= samples[original] or \
+ samples[deleted] >= samples[created]:
+ return ('Ceph {} samples ({}) '
+ 'unexpected.'.format(sample_type, samples))
+ else:
+ self.log.debug('Ceph {} samples (OK): '
+ '{}'.format(sample_type, samples))
+ return None
+
+ # rabbitmq/amqp specific helpers:
+
+ def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
+ """Wait for rmq units extended status to show cluster readiness,
+ after an optional initial sleep period. Initial sleep is likely
+ necessary to be effective following a config change, as status
+ message may not instantly update to non-ready."""
+
+ if init_sleep:
+ time.sleep(init_sleep)
+
+ message = re.compile('^Unit is ready and clustered$')
+ deployment._auto_wait_for_status(message=message,
+ timeout=timeout,
+ include_only=['rabbitmq-server'])
+
+ def add_rmq_test_user(self, sentry_units,
+ username="testuser1", password="changeme"):
+ """Add a test user via the first rmq juju unit, check connection as
+ the new user against all sentry units.
+
+ :param sentry_units: list of sentry unit pointers
+ :param username: amqp user name, default to testuser1
+ :param password: amqp user password
+ :returns: None if successful. Raise on error.
+ """
+ self.log.debug('Adding rmq user ({})...'.format(username))
+
+ # Check that user does not already exist
+ cmd_user_list = 'rabbitmqctl list_users'
+ output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
+ if username in output:
+ self.log.warning('User ({}) already exists, returning '
+ 'gracefully.'.format(username))
+ return
+
+ perms = '".*" ".*" ".*"'
+ cmds = ['rabbitmqctl add_user {} {}'.format(username, password),
+ 'rabbitmqctl set_permissions {} {}'.format(username, perms)]
+
+ # Add user via first unit
+ for cmd in cmds:
+ output, _ = self.run_cmd_unit(sentry_units[0], cmd)
+
+ # Check connection against the other sentry_units
+ self.log.debug('Checking user connect against units...')
+ for sentry_unit in sentry_units:
+ connection = self.connect_amqp_by_unit(sentry_unit, ssl=False,
+ username=username,
+ password=password)
+ connection.close()
+
+ def delete_rmq_test_user(self, sentry_units, username="testuser1"):
+ """Delete a rabbitmq user via the first rmq juju unit.
+
+ :param sentry_units: list of sentry unit pointers
+ :param username: amqp user name, default to testuser1
+ :param password: amqp user password
+ :returns: None if successful or no such user.
+ """
+ self.log.debug('Deleting rmq user ({})...'.format(username))
+
+ # Check that the user exists
+ cmd_user_list = 'rabbitmqctl list_users'
+ output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
+
+ if username not in output:
+ self.log.warning('User ({}) does not exist, returning '
+ 'gracefully.'.format(username))
+ return
+
+ # Delete the user
+ cmd_user_del = 'rabbitmqctl delete_user {}'.format(username)
+ output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_del)
+
+ def get_rmq_cluster_status(self, sentry_unit):
+ """Execute rabbitmq cluster status command on a unit and return
+ the full output.
+
+ :param unit: sentry unit
+ :returns: String containing console output of cluster status command
+ """
+ cmd = 'rabbitmqctl cluster_status'
+ output, _ = self.run_cmd_unit(sentry_unit, cmd)
+ self.log.debug('{} cluster_status:\n{}'.format(
+ sentry_unit.info['unit_name'], output))
+ return str(output)
+
+ def get_rmq_cluster_running_nodes(self, sentry_unit):
+ """Parse rabbitmqctl cluster_status output string, return list of
+ running rabbitmq cluster nodes.
+
+ :param unit: sentry unit
+ :returns: List containing node names of running nodes
+ """
+ # NOTE(beisner): rabbitmqctl cluster_status output is not
+ # json-parsable, do string chop foo, then json.loads that.
+ str_stat = self.get_rmq_cluster_status(sentry_unit)
+ if 'running_nodes' in str_stat:
+ pos_start = str_stat.find("{running_nodes,") + 15
+ pos_end = str_stat.find("]},", pos_start) + 1
+ str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"')
+ run_nodes = json.loads(str_run_nodes)
+ return run_nodes
+ else:
+ return []
+
+ def validate_rmq_cluster_running_nodes(self, sentry_units):
+ """Check that all rmq unit hostnames are represented in the
+ cluster_status output of all units.
+
+ :param host_names: dict of juju unit names to host names
+ :param units: list of sentry unit pointers (all rmq units)
+ :returns: None if successful, otherwise return error message
+ """
+ host_names = self.get_unit_hostnames(sentry_units)
+ errors = []
+
+ # Query every unit for cluster_status running nodes
+ for query_unit in sentry_units:
+ query_unit_name = query_unit.info['unit_name']
+ running_nodes = self.get_rmq_cluster_running_nodes(query_unit)
+
+ # Confirm that every unit is represented in the queried unit's
+ # cluster_status running nodes output.
+ for validate_unit in sentry_units:
+ val_host_name = host_names[validate_unit.info['unit_name']]
+ val_node_name = 'rabbit@{}'.format(val_host_name)
+
+ if val_node_name not in running_nodes:
+ errors.append('Cluster member check failed on {}: {} not '
+ 'in {}\n'.format(query_unit_name,
+ val_node_name,
+ running_nodes))
+ if errors:
+ return ''.join(errors)
+
+ def rmq_ssl_is_enabled_on_unit(self, sentry_unit, port=None):
+ """Check a single juju rmq unit for ssl and port in the config file."""
+ host = sentry_unit.info['public-address']
+ unit_name = sentry_unit.info['unit_name']
+
+ conf_file = '/etc/rabbitmq/rabbitmq.config'
+ conf_contents = str(self.file_contents_safe(sentry_unit,
+ conf_file, max_wait=16))
+ # Checks
+ conf_ssl = 'ssl' in conf_contents
+ conf_port = str(port) in conf_contents
+
+ # Port explicitly checked in config
+ if port and conf_port and conf_ssl:
+ self.log.debug('SSL is enabled @{}:{} '
+ '({})'.format(host, port, unit_name))
+ return True
+ elif port and not conf_port and conf_ssl:
+ self.log.debug('SSL is enabled @{} but not on port {} '
+ '({})'.format(host, port, unit_name))
+ return False
+ # Port not checked (useful when checking that ssl is disabled)
+ elif not port and conf_ssl:
+ self.log.debug('SSL is enabled @{}:{} '
+ '({})'.format(host, port, unit_name))
+ return True
+ elif not conf_ssl:
+ self.log.debug('SSL not enabled @{}:{} '
+ '({})'.format(host, port, unit_name))
+ return False
+ else:
+ msg = ('Unknown condition when checking SSL status @{}:{} '
+ '({})'.format(host, port, unit_name))
+ amulet.raise_status(amulet.FAIL, msg)
+
+ def validate_rmq_ssl_enabled_units(self, sentry_units, port=None):
+ """Check that ssl is enabled on rmq juju sentry units.
+
+ :param sentry_units: list of all rmq sentry units
+ :param port: optional ssl port override to validate
+ :returns: None if successful, otherwise return error message
+ """
+ for sentry_unit in sentry_units:
+ if not self.rmq_ssl_is_enabled_on_unit(sentry_unit, port=port):
+ return ('Unexpected condition: ssl is disabled on unit '
+ '({})'.format(sentry_unit.info['unit_name']))
+ return None
+
+ def validate_rmq_ssl_disabled_units(self, sentry_units):
+ """Check that ssl is enabled on listed rmq juju sentry units.
+
+ :param sentry_units: list of all rmq sentry units
+ :returns: True if successful. Raise on error.
+ """
+ for sentry_unit in sentry_units:
+ if self.rmq_ssl_is_enabled_on_unit(sentry_unit):
+ return ('Unexpected condition: ssl is enabled on unit '
+ '({})'.format(sentry_unit.info['unit_name']))
+ return None
+
+ def configure_rmq_ssl_on(self, sentry_units, deployment,
+ port=None, max_wait=60):
+ """Turn ssl charm config option on, with optional non-default
+ ssl port specification. Confirm that it is enabled on every
+ unit.
+
+ :param sentry_units: list of sentry units
+ :param deployment: amulet deployment object pointer
+ :param port: amqp port, use defaults if None
+ :param max_wait: maximum time to wait in seconds to confirm
+ :returns: None if successful. Raise on error.
+ """
+ self.log.debug('Setting ssl charm config option: on')
+
+ # Enable RMQ SSL
+ config = {'ssl': 'on'}
+ if port:
+ config['ssl_port'] = port
+
+ deployment.d.configure('rabbitmq-server', config)
+
+ # Wait for unit status
+ self.rmq_wait_for_cluster(deployment)
+
+ # Confirm
+ tries = 0
+ ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
+ while ret and tries < (max_wait / 4):
+ time.sleep(4)
+ self.log.debug('Attempt {}: {}'.format(tries, ret))
+ ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
+ tries += 1
+
+ if ret:
+ amulet.raise_status(amulet.FAIL, ret)
+
+ def configure_rmq_ssl_off(self, sentry_units, deployment, max_wait=60):
+ """Turn ssl charm config option off, confirm that it is disabled
+ on every unit.
+
+ :param sentry_units: list of sentry units
+ :param deployment: amulet deployment object pointer
+ :param max_wait: maximum time to wait in seconds to confirm
+ :returns: None if successful. Raise on error.
+ """
+ self.log.debug('Setting ssl charm config option: off')
+
+ # Disable RMQ SSL
+ config = {'ssl': 'off'}
+ deployment.d.configure('rabbitmq-server', config)
+
+ # Wait for unit status
+ self.rmq_wait_for_cluster(deployment)
+
+ # Confirm
+ tries = 0
+ ret = self.validate_rmq_ssl_disabled_units(sentry_units)
+ while ret and tries < (max_wait / 4):
+ time.sleep(4)
+ self.log.debug('Attempt {}: {}'.format(tries, ret))
+ ret = self.validate_rmq_ssl_disabled_units(sentry_units)
+ tries += 1
+
+ if ret:
+ amulet.raise_status(amulet.FAIL, ret)
+
+ def connect_amqp_by_unit(self, sentry_unit, ssl=False,
+ port=None, fatal=True,
+ username="testuser1", password="changeme"):
+ """Establish and return a pika amqp connection to the rabbitmq service
+ running on a rmq juju unit.
+
+ :param sentry_unit: sentry unit pointer
+ :param ssl: boolean, default to False
+ :param port: amqp port, use defaults if None
+ :param fatal: boolean, default to True (raises on connect error)
+ :param username: amqp user name, default to testuser1
+ :param password: amqp user password
+ :returns: pika amqp connection pointer or None if failed and non-fatal
+ """
+ host = sentry_unit.info['public-address']
+ unit_name = sentry_unit.info['unit_name']
+
+ # Default port logic if port is not specified
+ if ssl and not port:
+ port = 5671
+ elif not ssl and not port:
+ port = 5672
+
+ self.log.debug('Connecting to amqp on {}:{} ({}) as '
+ '{}...'.format(host, port, unit_name, username))
+
+ try:
+ credentials = pika.PlainCredentials(username, password)
+ parameters = pika.ConnectionParameters(host=host, port=port,
+ credentials=credentials,
+ ssl=ssl,
+ connection_attempts=3,
+ retry_delay=5,
+ socket_timeout=1)
+ connection = pika.BlockingConnection(parameters)
+ assert connection.is_open is True
+ assert connection.is_closing is False
+ self.log.debug('Connect OK')
+ return connection
+ except Exception as e:
+ msg = ('amqp connection failed to {}:{} as '
+ '{} ({})'.format(host, port, username, str(e)))
+ if fatal:
+ amulet.raise_status(amulet.FAIL, msg)
+ else:
+ self.log.warn(msg)
+ return None
+
+ def publish_amqp_message_by_unit(self, sentry_unit, message,
+ queue="test", ssl=False,
+ username="testuser1",
+ password="changeme",
+ port=None):
+ """Publish an amqp message to a rmq juju unit.
+
+ :param sentry_unit: sentry unit pointer
+ :param message: amqp message string
+ :param queue: message queue, default to test
+ :param username: amqp user name, default to testuser1
+ :param password: amqp user password
+ :param ssl: boolean, default to False
+ :param port: amqp port, use defaults if None
+ :returns: None. Raises exception if publish failed.
+ """
+ self.log.debug('Publishing message to {} queue:\n{}'.format(queue,
+ message))
+ connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
+ port=port,
+ username=username,
+ password=password)
+
+ # NOTE(beisner): extra debug here re: pika hang potential:
+ # https://github.com/pika/pika/issues/297
+ # https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw
+ self.log.debug('Defining channel...')
+ channel = connection.channel()
+ self.log.debug('Declaring queue...')
+ channel.queue_declare(queue=queue, auto_delete=False, durable=True)
+ self.log.debug('Publishing message...')
+ channel.basic_publish(exchange='', routing_key=queue, body=message)
+ self.log.debug('Closing channel...')
+ channel.close()
+ self.log.debug('Closing connection...')
+ connection.close()
+
+ def get_amqp_message_by_unit(self, sentry_unit, queue="test",
+ username="testuser1",
+ password="changeme",
+ ssl=False, port=None):
+ """Get an amqp message from a rmq juju unit.
+
+ :param sentry_unit: sentry unit pointer
+ :param queue: message queue, default to test
+ :param username: amqp user name, default to testuser1
+ :param password: amqp user password
+ :param ssl: boolean, default to False
+ :param port: amqp port, use defaults if None
+ :returns: amqp message body as string. Raise if get fails.
+ """
+ connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
+ port=port,
+ username=username,
+ password=password)
+ channel = connection.channel()
+ method_frame, _, body = channel.basic_get(queue)
+
+ if method_frame:
+ self.log.debug('Retreived message from {} queue:\n{}'.format(queue,
+ body))
+ channel.basic_ack(method_frame.delivery_tag)
+ channel.close()
+ connection.close()
+ return body
+ else:
+ msg = 'No message retrieved.'
+ amulet.raise_status(amulet.FAIL, msg)
+
+ def validate_memcache(self, sentry_unit, conf, os_release,
+ earliest_release=5, section='keystone_authtoken',
+ check_kvs=None):
+ """Check Memcache is running and is configured to be used
+
+ Example call from Amulet test:
+
+ def test_110_memcache(self):
+ u.validate_memcache(self.neutron_api_sentry,
+ '/etc/neutron/neutron.conf',
+ self._get_openstack_release())
+
+ :param sentry_unit: sentry unit
+ :param conf: OpenStack config file to check memcache settings
+ :param os_release: Current OpenStack release int code
+ :param earliest_release: Earliest Openstack release to check int code
+ :param section: OpenStack config file section to check
+ :param check_kvs: Dict of settings to check in config file
+ :returns: None
+ """
+ if os_release < earliest_release:
+ self.log.debug('Skipping memcache checks for deployment. {} <'
+ 'mitaka'.format(os_release))
+ return
+ _kvs = check_kvs or {'memcached_servers': 'inet6:[::1]:11211'}
+ self.log.debug('Checking memcached is running')
+ ret = self.validate_services_by_name({sentry_unit: ['memcached']})
+ if ret:
+ amulet.raise_status(amulet.FAIL, msg='Memcache running check'
+ 'failed {}'.format(ret))
+ else:
+ self.log.debug('OK')
+ self.log.debug('Checking memcache url is configured in {}'.format(
+ conf))
+ if self.validate_config_data(sentry_unit, conf, section, _kvs):
+ message = "Memcache config error in: {}".format(conf)
+ amulet.raise_status(amulet.FAIL, msg=message)
+ else:
+ self.log.debug('OK')
+ self.log.debug('Checking memcache configuration in '
+ '/etc/memcached.conf')
+ contents = self.file_contents_safe(sentry_unit, '/etc/memcached.conf',
+ fatal=True)
+ ubuntu_release, _ = self.run_cmd_unit(sentry_unit, 'lsb_release -cs')
+ if CompareHostReleases(ubuntu_release) <= 'trusty':
+ memcache_listen_addr = 'ip6-localhost'
+ else:
+ memcache_listen_addr = '::1'
+ expected = {
+ '-p': '11211',
+ '-l': memcache_listen_addr}
+ found = []
+ for key, value in expected.items():
+ for line in contents.split('\n'):
+ if line.startswith(key):
+ self.log.debug('Checking {} is set to {}'.format(
+ key,
+ value))
+ assert value == line.split()[-1]
+ self.log.debug(line.split()[-1])
+ found.append(key)
+ if sorted(found) == sorted(expected.keys()):
+ self.log.debug('OK')
+ else:
+ message = "Memcache config error in: /etc/memcached.conf"
+ amulet.raise_status(amulet.FAIL, msg=message)
diff --git a/tests/charmhelpers/contrib/openstack/exceptions.py b/tests/charmhelpers/contrib/openstack/exceptions.py
new file mode 100644
index 0000000..f85ae4f
--- /dev/null
+++ b/tests/charmhelpers/contrib/openstack/exceptions.py
@@ -0,0 +1,21 @@
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class OSContextError(Exception):
+ """Raised when an error occurs during context generation.
+
+ This exception is principally used in contrib.openstack.context
+ """
+ pass
diff --git a/tests/charmhelpers/contrib/openstack/utils.py b/tests/charmhelpers/contrib/openstack/utils.py
new file mode 100644
index 0000000..8a541d4
--- /dev/null
+++ b/tests/charmhelpers/contrib/openstack/utils.py
@@ -0,0 +1,2123 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Common python helper functions used for OpenStack charms.
+from collections import OrderedDict
+from functools import wraps
+
+import subprocess
+import json
+import os
+import sys
+import re
+import itertools
+import functools
+import shutil
+
+import six
+import traceback
+import uuid
+import yaml
+
+from charmhelpers import deprecate
+
+from charmhelpers.contrib.network import ip
+
+from charmhelpers.core import unitdata
+
+from charmhelpers.core.hookenv import (
+ action_fail,
+ action_set,
+ config,
+ log as juju_log,
+ charm_dir,
+ INFO,
+ ERROR,
+ related_units,
+ relation_ids,
+ relation_set,
+ service_name,
+ status_set,
+ hook_name,
+ application_version_set,
+ cached,
+)
+
+from charmhelpers.core.strutils import BasicStringComparator
+
+from charmhelpers.contrib.storage.linux.lvm import (
+ deactivate_lvm_volume_group,
+ is_lvm_physical_volume,
+ remove_lvm_physical_volume,
+)
+
+from charmhelpers.contrib.network.ip import (
+ get_ipv6_addr,
+ is_ipv6,
+ port_has_listener,
+)
+
+from charmhelpers.contrib.python.packages import (
+ pip_create_virtualenv,
+ pip_install,
+)
+
+from charmhelpers.core.host import (
+ lsb_release,
+ mounts,
+ umount,
+ service_running,
+ service_pause,
+ service_resume,
+ restart_on_change_helper,
+)
+from charmhelpers.fetch import (
+ apt_cache,
+ install_remote,
+ import_key as fetch_import_key,
+ add_source as fetch_add_source,
+ SourceConfigError,
+ GPGKeyError,
+ get_upstream_version
+)
+
+from charmhelpers.fetch.snap import (
+ snap_install,
+ snap_refresh,
+ valid_snap_channel,
+)
+
+from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
+from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
+from charmhelpers.contrib.openstack.exceptions import OSContextError
+
+CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu"
+CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
+
+DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed '
+ 'restricted main multiverse universe')
+
+OPENSTACK_RELEASES = (
+ 'diablo',
+ 'essex',
+ 'folsom',
+ 'grizzly',
+ 'havana',
+ 'icehouse',
+ 'juno',
+ 'kilo',
+ 'liberty',
+ 'mitaka',
+ 'newton',
+ 'ocata',
+ 'pike',
+ 'queens',
+ 'rocky',
+)
+
+UBUNTU_OPENSTACK_RELEASE = OrderedDict([
+ ('oneiric', 'diablo'),
+ ('precise', 'essex'),
+ ('quantal', 'folsom'),
+ ('raring', 'grizzly'),
+ ('saucy', 'havana'),
+ ('trusty', 'icehouse'),
+ ('utopic', 'juno'),
+ ('vivid', 'kilo'),
+ ('wily', 'liberty'),
+ ('xenial', 'mitaka'),
+ ('yakkety', 'newton'),
+ ('zesty', 'ocata'),
+ ('artful', 'pike'),
+ ('bionic', 'queens'),
+])
+
+
+OPENSTACK_CODENAMES = OrderedDict([
+ ('2011.2', 'diablo'),
+ ('2012.1', 'essex'),
+ ('2012.2', 'folsom'),
+ ('2013.1', 'grizzly'),
+ ('2013.2', 'havana'),
+ ('2014.1', 'icehouse'),
+ ('2014.2', 'juno'),
+ ('2015.1', 'kilo'),
+ ('2015.2', 'liberty'),
+ ('2016.1', 'mitaka'),
+ ('2016.2', 'newton'),
+ ('2017.1', 'ocata'),
+ ('2017.2', 'pike'),
+ ('2018.1', 'queens'),
+])
+
+# The ugly duckling - must list releases oldest to newest
+SWIFT_CODENAMES = OrderedDict([
+ ('diablo',
+ ['1.4.3']),
+ ('essex',
+ ['1.4.8']),
+ ('folsom',
+ ['1.7.4']),
+ ('grizzly',
+ ['1.7.6', '1.7.7', '1.8.0']),
+ ('havana',
+ ['1.9.0', '1.9.1', '1.10.0']),
+ ('icehouse',
+ ['1.11.0', '1.12.0', '1.13.0', '1.13.1']),
+ ('juno',
+ ['2.0.0', '2.1.0', '2.2.0']),
+ ('kilo',
+ ['2.2.1', '2.2.2']),
+ ('liberty',
+ ['2.3.0', '2.4.0', '2.5.0']),
+ ('mitaka',
+ ['2.5.0', '2.6.0', '2.7.0']),
+ ('newton',
+ ['2.8.0', '2.9.0', '2.10.0']),
+ ('ocata',
+ ['2.11.0', '2.12.0', '2.13.0']),
+ ('pike',
+ ['2.13.0', '2.15.0']),
+ ('queens',
+ ['2.16.0']),
+])
+
+# >= Liberty version->codename mapping
+PACKAGE_CODENAMES = {
+ 'nova-common': OrderedDict([
+ ('12', 'liberty'),
+ ('13', 'mitaka'),
+ ('14', 'newton'),
+ ('15', 'ocata'),
+ ('16', 'pike'),
+ ('17', 'queens'),
+ ('18', 'rocky'),
+ ]),
+ 'neutron-common': OrderedDict([
+ ('7', 'liberty'),
+ ('8', 'mitaka'),
+ ('9', 'newton'),
+ ('10', 'ocata'),
+ ('11', 'pike'),
+ ('12', 'queens'),
+ ('13', 'rocky'),
+ ]),
+ 'cinder-common': OrderedDict([
+ ('7', 'liberty'),
+ ('8', 'mitaka'),
+ ('9', 'newton'),
+ ('10', 'ocata'),
+ ('11', 'pike'),
+ ('12', 'queens'),
+ ('13', 'rocky'),
+ ]),
+ 'keystone': OrderedDict([
+ ('8', 'liberty'),
+ ('9', 'mitaka'),
+ ('10', 'newton'),
+ ('11', 'ocata'),
+ ('12', 'pike'),
+ ('13', 'queens'),
+ ('14', 'rocky'),
+ ]),
+ 'horizon-common': OrderedDict([
+ ('8', 'liberty'),
+ ('9', 'mitaka'),
+ ('10', 'newton'),
+ ('11', 'ocata'),
+ ('12', 'pike'),
+ ('13', 'queens'),
+ ('14', 'rocky'),
+ ]),
+ 'ceilometer-common': OrderedDict([
+ ('5', 'liberty'),
+ ('6', 'mitaka'),
+ ('7', 'newton'),
+ ('8', 'ocata'),
+ ('9', 'pike'),
+ ('10', 'queens'),
+ ('11', 'rocky'),
+ ]),
+ 'heat-common': OrderedDict([
+ ('5', 'liberty'),
+ ('6', 'mitaka'),
+ ('7', 'newton'),
+ ('8', 'ocata'),
+ ('9', 'pike'),
+ ('10', 'queens'),
+ ('11', 'rocky'),
+ ]),
+ 'glance-common': OrderedDict([
+ ('11', 'liberty'),
+ ('12', 'mitaka'),
+ ('13', 'newton'),
+ ('14', 'ocata'),
+ ('15', 'pike'),
+ ('16', 'queens'),
+ ('17', 'rocky'),
+ ]),
+ 'openstack-dashboard': OrderedDict([
+ ('8', 'liberty'),
+ ('9', 'mitaka'),
+ ('10', 'newton'),
+ ('11', 'ocata'),
+ ('12', 'pike'),
+ ('13', 'queens'),
+ ('14', 'rocky'),
+ ]),
+}
+
+GIT_DEFAULT_REPOS = {
+ 'requirements': 'git://github.com/openstack/requirements',
+ 'cinder': 'git://github.com/openstack/cinder',
+ 'glance': 'git://github.com/openstack/glance',
+ 'horizon': 'git://github.com/openstack/horizon',
+ 'keystone': 'git://github.com/openstack/keystone',
+ 'networking-hyperv': 'git://github.com/openstack/networking-hyperv',
+ 'neutron': 'git://github.com/openstack/neutron',
+ 'neutron-fwaas': 'git://github.com/openstack/neutron-fwaas',
+ 'neutron-lbaas': 'git://github.com/openstack/neutron-lbaas',
+ 'neutron-vpnaas': 'git://github.com/openstack/neutron-vpnaas',
+ 'nova': 'git://github.com/openstack/nova',
+}
+
+GIT_DEFAULT_BRANCHES = {
+ 'liberty': 'stable/liberty',
+ 'mitaka': 'stable/mitaka',
+ 'newton': 'stable/newton',
+ 'master': 'master',
+}
+
+DEFAULT_LOOPBACK_SIZE = '5G'
+
+
+class CompareOpenStackReleases(BasicStringComparator):
+ """Provide comparisons of OpenStack releases.
+
+ Use in the form of
+
+ if CompareOpenStackReleases(release) > 'mitaka':
+ # do something with mitaka
+ """
+ _list = OPENSTACK_RELEASES
+
+
+def error_out(msg):
+ juju_log("FATAL ERROR: %s" % msg, level='ERROR')
+ sys.exit(1)
+
+
+def get_os_codename_install_source(src):
+ '''Derive OpenStack release codename from a given installation source.'''
+ ubuntu_rel = lsb_release()['DISTRIB_CODENAME']
+ rel = ''
+ if src is None:
+ return rel
+ if src in ['distro', 'distro-proposed']:
+ try:
+ rel = UBUNTU_OPENSTACK_RELEASE[ubuntu_rel]
+ except KeyError:
+ e = 'Could not derive openstack release for '\
+ 'this Ubuntu release: %s' % ubuntu_rel
+ error_out(e)
+ return rel
+
+ if src.startswith('cloud:'):
+ ca_rel = src.split(':')[1]
+ ca_rel = ca_rel.split('%s-' % ubuntu_rel)[1].split('/')[0]
+ return ca_rel
+
+ # Best guess match based on deb string provided
+ if (src.startswith('deb') or
+ src.startswith('ppa') or
+ src.startswith('snap')):
+ for v in OPENSTACK_CODENAMES.values():
+ if v in src:
+ return v
+
+
+def get_os_version_install_source(src):
+ codename = get_os_codename_install_source(src)
+ return get_os_version_codename(codename)
+
+
+def get_os_codename_version(vers):
+ '''Determine OpenStack codename from version number.'''
+ try:
+ return OPENSTACK_CODENAMES[vers]
+ except KeyError:
+ e = 'Could not determine OpenStack codename for version %s' % vers
+ error_out(e)
+
+
+def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES):
+ '''Determine OpenStack version number from codename.'''
+ for k, v in six.iteritems(version_map):
+ if v == codename:
+ return k
+ e = 'Could not derive OpenStack version for '\
+ 'codename: %s' % codename
+ error_out(e)
+
+
+def get_os_version_codename_swift(codename):
+ '''Determine OpenStack version number of swift from codename.'''
+ for k, v in six.iteritems(SWIFT_CODENAMES):
+ if k == codename:
+ return v[-1]
+ e = 'Could not derive swift version for '\
+ 'codename: %s' % codename
+ error_out(e)
+
+
+def get_swift_codename(version):
+ '''Determine OpenStack codename that corresponds to swift version.'''
+ codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v]
+
+ if len(codenames) > 1:
+ # If more than one release codename contains this version we determine
+ # the actual codename based on the highest available install source.
+ for codename in reversed(codenames):
+ releases = UBUNTU_OPENSTACK_RELEASE
+ release = [k for k, v in six.iteritems(releases) if codename in v]
+ ret = subprocess.check_output(['apt-cache', 'policy', 'swift'])
+ if codename in ret or release[0] in ret:
+ return codename
+ elif len(codenames) == 1:
+ return codenames[0]
+
+ # NOTE: fallback - attempt to match with just major.minor version
+ match = re.match('^(\d+)\.(\d+)', version)
+ if match:
+ major_minor_version = match.group(0)
+ for codename, versions in six.iteritems(SWIFT_CODENAMES):
+ for release_version in versions:
+ if release_version.startswith(major_minor_version):
+ return codename
+
+ return None
+
+
+def get_os_codename_package(package, fatal=True):
+ '''Derive OpenStack release codename from an installed package.'''
+
+ if snap_install_requested():
+ cmd = ['snap', 'list', package]
+ try:
+ out = subprocess.check_output(cmd)
+ if six.PY3:
+ out = out.decode('UTF-8')
+ except subprocess.CalledProcessError as e:
+ return None
+ lines = out.split('\n')
+ for line in lines:
+ if package in line:
+ # Second item in list is Version
+ return line.split()[1]
+
+ import apt_pkg as apt
+
+ cache = apt_cache()
+
+ try:
+ pkg = cache[package]
+ except Exception:
+ if not fatal:
+ return None
+ # the package is unknown to the current apt cache.
+ e = 'Could not determine version of package with no installation '\
+ 'candidate: %s' % package
+ error_out(e)
+
+ if not pkg.current_ver:
+ if not fatal:
+ return None
+ # package is known, but no version is currently installed.
+ e = 'Could not determine version of uninstalled package: %s' % package
+ error_out(e)
+
+ vers = apt.upstream_version(pkg.current_ver.ver_str)
+ if 'swift' in pkg.name:
+ # Fully x.y.z match for swift versions
+ match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
+ else:
+ # x.y match only for 20XX.X
+ # and ignore patch level for other packages
+ match = re.match('^(\d+)\.(\d+)', vers)
+
+ if match:
+ vers = match.group(0)
+
+ # Generate a major version number for newer semantic
+ # versions of openstack projects
+ major_vers = vers.split('.')[0]
+ # >= Liberty independent project versions
+ if (package in PACKAGE_CODENAMES and
+ major_vers in PACKAGE_CODENAMES[package]):
+ return PACKAGE_CODENAMES[package][major_vers]
+ else:
+ # < Liberty co-ordinated project versions
+ try:
+ if 'swift' in pkg.name:
+ return get_swift_codename(vers)
+ else:
+ return OPENSTACK_CODENAMES[vers]
+ except KeyError:
+ if not fatal:
+ return None
+ e = 'Could not determine OpenStack codename for version %s' % vers
+ error_out(e)
+
+
+def get_os_version_package(pkg, fatal=True):
+ '''Derive OpenStack version number from an installed package.'''
+ codename = get_os_codename_package(pkg, fatal=fatal)
+
+ if not codename:
+ return None
+
+ if 'swift' in pkg:
+ vers_map = SWIFT_CODENAMES
+ for cname, version in six.iteritems(vers_map):
+ if cname == codename:
+ return version[-1]
+ else:
+ vers_map = OPENSTACK_CODENAMES
+ for version, cname in six.iteritems(vers_map):
+ if cname == codename:
+ return version
+ # e = "Could not determine OpenStack version for package: %s" % pkg
+ # error_out(e)
+
+
+# Module local cache variable for the os_release.
+_os_rel = None
+
+
+def reset_os_release():
+ '''Unset the cached os_release version'''
+ global _os_rel
+ _os_rel = None
+
+
+def os_release(package, base='essex', reset_cache=False):
+ '''
+ Returns OpenStack release codename from a cached global.
+
+ If reset_cache then unset the cached os_release version and return the
+ freshly determined version.
+
+ If the codename can not be determined from either an installed package or
+ the installation source, the earliest release supported by the charm should
+ be returned.
+ '''
+ global _os_rel
+ if reset_cache:
+ reset_os_release()
+ if _os_rel:
+ return _os_rel
+ _os_rel = (
+ git_os_codename_install_source(config('openstack-origin-git')) or
+ get_os_codename_package(package, fatal=False) or
+ get_os_codename_install_source(config('openstack-origin')) or
+ base)
+ return _os_rel
+
+
+@deprecate("moved to charmhelpers.fetch.import_key()", "2017-07", log=juju_log)
+def import_key(keyid):
+ """Import a key, either ASCII armored, or a GPG key id.
+
+ @param keyid: the key in ASCII armor format, or a GPG key id.
+ @raises SystemExit() via sys.exit() on failure.
+ """
+ try:
+ return fetch_import_key(keyid)
+ except GPGKeyError as e:
+ error_out("Could not import key: {}".format(str(e)))
+
+
+def get_source_and_pgp_key(source_and_key):
+ """Look for a pgp key ID or ascii-armor key in the given input.
+
+ :param source_and_key: Sting, "source_spec|keyid" where '|keyid' is
+ optional.
+ :returns (source_spec, key_id OR None) as a tuple. Returns None for key_id
+ if there was no '|' in the source_and_key string.
+ """
+ try:
+ source, key = source_and_key.split('|', 2)
+ return source, key or None
+ except ValueError:
+ return source_and_key, None
+
+
+@deprecate("use charmhelpers.fetch.add_source() instead.",
+ "2017-07", log=juju_log)
+def configure_installation_source(source_plus_key):
+ """Configure an installation source.
+
+ The functionality is provided by charmhelpers.fetch.add_source()
+ The difference between the two functions is that add_source() signature
+ requires the key to be passed directly, whereas this function passes an
+ optional key by appending '|' to the end of the source specificiation
+ 'source'.
+
+ Another difference from add_source() is that the function calls sys.exit(1)
+ if the configuration fails, whereas add_source() raises
+ SourceConfigurationError(). Another difference, is that add_source()
+ silently fails (with a juju_log command) if there is no matching source to
+ configure, whereas this function fails with a sys.exit(1)
+
+ :param source: String_plus_key -- see above for details.
+
+ Note that the behaviour on error is to log the error to the juju log and
+ then call sys.exit(1).
+ """
+ if source_plus_key.startswith('snap'):
+ # Do nothing for snap installs
+ return
+ # extract the key if there is one, denoted by a '|' in the rel
+ source, key = get_source_and_pgp_key(source_plus_key)
+
+ # handle the ordinary sources via add_source
+ try:
+ fetch_add_source(source, key, fail_invalid=True)
+ except SourceConfigError as se:
+ error_out(str(se))
+
+
+def config_value_changed(option):
+ """
+ Determine if config value changed since last call to this function.
+ """
+ hook_data = unitdata.HookData()
+ with hook_data():
+ db = unitdata.kv()
+ current = config(option)
+ saved = db.get(option)
+ db.set(option, current)
+ if saved is None:
+ return False
+ return current != saved
+
+
+def save_script_rc(script_path="scripts/scriptrc", **env_vars):
+ """
+ Write an rc file in the charm-delivered directory containing
+ exported environment variables provided by env_vars. Any charm scripts run
+ outside the juju hook environment can source this scriptrc to obtain
+ updated config information necessary to perform health checks or
+ service changes.
+ """
+ juju_rc_path = "%s/%s" % (charm_dir(), script_path)
+ if not os.path.exists(os.path.dirname(juju_rc_path)):
+ os.mkdir(os.path.dirname(juju_rc_path))
+ with open(juju_rc_path, 'wt') as rc_script:
+ rc_script.write(
+ "#!/bin/bash\n")
+ [rc_script.write('export %s=%s\n' % (u, p))
+ for u, p in six.iteritems(env_vars) if u != "script_path"]
+
+
+def openstack_upgrade_available(package):
+ """
+ Determines if an OpenStack upgrade is available from installation
+ source, based on version of installed package.
+
+ :param package: str: Name of installed package.
+
+ :returns: bool: : Returns True if configured installation source offers
+ a newer version of package.
+ """
+
+ import apt_pkg as apt
+ src = config('openstack-origin')
+ cur_vers = get_os_version_package(package)
+ if not cur_vers:
+ # The package has not been installed yet do not attempt upgrade
+ return False
+ if "swift" in package:
+ codename = get_os_codename_install_source(src)
+ avail_vers = get_os_version_codename_swift(codename)
+ else:
+ avail_vers = get_os_version_install_source(src)
+ apt.init()
+ if "swift" in package:
+ major_cur_vers = cur_vers.split('.', 1)[0]
+ major_avail_vers = avail_vers.split('.', 1)[0]
+ major_diff = apt.version_compare(major_avail_vers, major_cur_vers)
+ return avail_vers > cur_vers and (major_diff == 1 or major_diff == 0)
+ return apt.version_compare(avail_vers, cur_vers) == 1
+
+
+def ensure_block_device(block_device):
+ '''
+ Confirm block_device, create as loopback if necessary.
+
+ :param block_device: str: Full path of block device to ensure.
+
+ :returns: str: Full path of ensured block device.
+ '''
+ _none = ['None', 'none', None]
+ if (block_device in _none):
+ error_out('prepare_storage(): Missing required input: block_device=%s.'
+ % block_device)
+
+ if block_device.startswith('/dev/'):
+ bdev = block_device
+ elif block_device.startswith('/'):
+ _bd = block_device.split('|')
+ if len(_bd) == 2:
+ bdev, size = _bd
+ else:
+ bdev = block_device
+ size = DEFAULT_LOOPBACK_SIZE
+ bdev = ensure_loopback_device(bdev, size)
+ else:
+ bdev = '/dev/%s' % block_device
+
+ if not is_block_device(bdev):
+ error_out('Failed to locate valid block device at %s' % bdev)
+
+ return bdev
+
+
+def clean_storage(block_device):
+ '''
+ Ensures a block device is clean. That is:
+ - unmounted
+ - any lvm volume groups are deactivated
+ - any lvm physical device signatures removed
+ - partition table wiped
+
+ :param block_device: str: Full path to block device to clean.
+ '''
+ for mp, d in mounts():
+ if d == block_device:
+ juju_log('clean_storage(): %s is mounted @ %s, unmounting.' %
+ (d, mp), level=INFO)
+ umount(mp, persist=True)
+
+ if is_lvm_physical_volume(block_device):
+ deactivate_lvm_volume_group(block_device)
+ remove_lvm_physical_volume(block_device)
+ else:
+ zap_disk(block_device)
+
+
+is_ip = ip.is_ip
+ns_query = ip.ns_query
+get_host_ip = ip.get_host_ip
+get_hostname = ip.get_hostname
+
+
+def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'):
+ mm_map = {}
+ if os.path.isfile(mm_file):
+ with open(mm_file, 'r') as f:
+ mm_map = json.load(f)
+ return mm_map
+
+
+def sync_db_with_multi_ipv6_addresses(database, database_user,
+ relation_prefix=None):
+ hosts = get_ipv6_addr(dynamic_only=False)
+
+ if config('vip'):
+ vips = config('vip').split()
+ for vip in vips:
+ if vip and is_ipv6(vip):
+ hosts.append(vip)
+
+ kwargs = {'database': database,
+ 'username': database_user,
+ 'hostname': json.dumps(hosts)}
+
+ if relation_prefix:
+ for key in list(kwargs.keys()):
+ kwargs["%s_%s" % (relation_prefix, key)] = kwargs[key]
+ del kwargs[key]
+
+ for rid in relation_ids('shared-db'):
+ relation_set(relation_id=rid, **kwargs)
+
+
+def os_requires_version(ostack_release, pkg):
+ """
+ Decorator for hook to specify minimum supported release
+ """
+ def wrap(f):
+ @wraps(f)
+ def wrapped_f(*args):
+ if os_release(pkg) < ostack_release:
+ raise Exception("This hook is not supported on releases"
+ " before %s" % ostack_release)
+ f(*args)
+ return wrapped_f
+ return wrap
+
+
+def git_install_requested():
+ """
+ Returns true if openstack-origin-git is specified.
+ """
+ return config('openstack-origin-git') is not None
+
+
+def git_os_codename_install_source(projects_yaml):
+ """
+ Returns OpenStack codename of release being installed from source.
+ """
+ if git_install_requested():
+ projects = _git_yaml_load(projects_yaml)
+
+ if projects in GIT_DEFAULT_BRANCHES.keys():
+ if projects == 'master':
+ return 'ocata'
+ return projects
+
+ if 'release' in projects:
+ if projects['release'] == 'master':
+ return 'ocata'
+ return projects['release']
+
+ return None
+
+
+def git_default_repos(projects_yaml):
+ """
+ Returns default repos if a default openstack-origin-git value is specified.
+ """
+ service = service_name()
+ core_project = service
+
+ for default, branch in six.iteritems(GIT_DEFAULT_BRANCHES):
+ if projects_yaml == default:
+
+ # add the requirements repo first
+ repo = {
+ 'name': 'requirements',
+ 'repository': GIT_DEFAULT_REPOS['requirements'],
+ 'branch': branch,
+ }
+ repos = [repo]
+
+ # neutron-* and nova-* charms require some additional repos
+ if service in ['neutron-api', 'neutron-gateway',
+ 'neutron-openvswitch']:
+ core_project = 'neutron'
+ if service == 'neutron-api':
+ repo = {
+ 'name': 'networking-hyperv',
+ 'repository': GIT_DEFAULT_REPOS['networking-hyperv'],
+ 'branch': branch,
+ }
+ repos.append(repo)
+ for project in ['neutron-fwaas', 'neutron-lbaas',
+ 'neutron-vpnaas', 'nova']:
+ repo = {
+ 'name': project,
+ 'repository': GIT_DEFAULT_REPOS[project],
+ 'branch': branch,
+ }
+ repos.append(repo)
+
+ elif service in ['nova-cloud-controller', 'nova-compute']:
+ core_project = 'nova'
+ repo = {
+ 'name': 'neutron',
+ 'repository': GIT_DEFAULT_REPOS['neutron'],
+ 'branch': branch,
+ }
+ repos.append(repo)
+ elif service == 'openstack-dashboard':
+ core_project = 'horizon'
+
+ # finally add the current service's core project repo
+ repo = {
+ 'name': core_project,
+ 'repository': GIT_DEFAULT_REPOS[core_project],
+ 'branch': branch,
+ }
+ repos.append(repo)
+
+ return yaml.dump(dict(repositories=repos, release=default))
+
+ return projects_yaml
+
+
+def _git_yaml_load(projects_yaml):
+ """
+ Load the specified yaml into a dictionary.
+ """
+ if not projects_yaml:
+ return None
+
+ return yaml.load(projects_yaml)
+
+
+requirements_dir = None
+
+
+def git_clone_and_install(projects_yaml, core_project):
+ """
+ Clone/install all specified OpenStack repositories.
+
+ The expected format of projects_yaml is:
+
+ repositories:
+ - {name: keystone,
+ repository: 'git://git.openstack.org/openstack/keystone.git',
+ branch: 'stable/icehouse'}
+ - {name: requirements,
+ repository: 'git://git.openstack.org/openstack/requirements.git',
+ branch: 'stable/icehouse'}
+
+ directory: /mnt/openstack-git
+ http_proxy: squid-proxy-url
+ https_proxy: squid-proxy-url
+
+ The directory, http_proxy, and https_proxy keys are optional.
+
+ """
+ global requirements_dir
+ parent_dir = '/mnt/openstack-git'
+ http_proxy = None
+
+ projects = _git_yaml_load(projects_yaml)
+ _git_validate_projects_yaml(projects, core_project)
+
+ old_environ = dict(os.environ)
+
+ if 'http_proxy' in projects.keys():
+ http_proxy = projects['http_proxy']
+ os.environ['http_proxy'] = projects['http_proxy']
+ if 'https_proxy' in projects.keys():
+ os.environ['https_proxy'] = projects['https_proxy']
+
+ if 'directory' in projects.keys():
+ parent_dir = projects['directory']
+
+ pip_create_virtualenv(os.path.join(parent_dir, 'venv'))
+
+ # Upgrade setuptools and pip from default virtualenv versions. The default
+ # versions in trusty break master OpenStack branch deployments.
+ for p in ['pip', 'setuptools']:
+ pip_install(p, upgrade=True, proxy=http_proxy,
+ venv=os.path.join(parent_dir, 'venv'))
+
+ constraints = None
+ for p in projects['repositories']:
+ repo = p['repository']
+ branch = p['branch']
+ depth = '1'
+ if 'depth' in p.keys():
+ depth = p['depth']
+ if p['name'] == 'requirements':
+ repo_dir = _git_clone_and_install_single(repo, branch, depth,
+ parent_dir, http_proxy,
+ update_requirements=False)
+ requirements_dir = repo_dir
+ constraints = os.path.join(repo_dir, "upper-constraints.txt")
+ # upper-constraints didn't exist until after icehouse
+ if not os.path.isfile(constraints):
+ constraints = None
+ # use constraints unless project yaml sets use_constraints to false
+ if 'use_constraints' in projects.keys():
+ if not projects['use_constraints']:
+ constraints = None
+ else:
+ repo_dir = _git_clone_and_install_single(repo, branch, depth,
+ parent_dir, http_proxy,
+ update_requirements=True,
+ constraints=constraints)
+
+ os.environ = old_environ
+
+
+def _git_validate_projects_yaml(projects, core_project):
+ """
+ Validate the projects yaml.
+ """
+ _git_ensure_key_exists('repositories', projects)
+
+ for project in projects['repositories']:
+ _git_ensure_key_exists('name', project.keys())
+ _git_ensure_key_exists('repository', project.keys())
+ _git_ensure_key_exists('branch', project.keys())
+
+ if projects['repositories'][0]['name'] != 'requirements':
+ error_out('{} git repo must be specified first'.format('requirements'))
+
+ if projects['repositories'][-1]['name'] != core_project:
+ error_out('{} git repo must be specified last'.format(core_project))
+
+ _git_ensure_key_exists('release', projects)
+
+
+def _git_ensure_key_exists(key, keys):
+ """
+ Ensure that key exists in keys.
+ """
+ if key not in keys:
+ error_out('openstack-origin-git key \'{}\' is missing'.format(key))
+
+
+def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
+ update_requirements, constraints=None):
+ """
+ Clone and install a single git repository.
+ """
+ if not os.path.exists(parent_dir):
+ juju_log('Directory already exists at {}. '
+ 'No need to create directory.'.format(parent_dir))
+ os.mkdir(parent_dir)
+
+ juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
+ repo_dir = install_remote(
+ repo, dest=parent_dir, branch=branch, depth=depth)
+
+ venv = os.path.join(parent_dir, 'venv')
+
+ if update_requirements:
+ if not requirements_dir:
+ error_out('requirements repo must be cloned before '
+ 'updating from global requirements.')
+ _git_update_requirements(venv, repo_dir, requirements_dir)
+
+ juju_log('Installing git repo from dir: {}'.format(repo_dir))
+ if http_proxy:
+ pip_install(repo_dir, proxy=http_proxy, venv=venv,
+ constraints=constraints)
+ else:
+ pip_install(repo_dir, venv=venv, constraints=constraints)
+
+ return repo_dir
+
+
+def _git_update_requirements(venv, package_dir, reqs_dir):
+ """
+ Update from global requirements.
+
+ Update an OpenStack git directory's requirements.txt and
+ test-requirements.txt from global-requirements.txt.
+ """
+ orig_dir = os.getcwd()
+ os.chdir(reqs_dir)
+ python = os.path.join(venv, 'bin/python')
+ cmd = [python, 'update.py', package_dir]
+ try:
+ subprocess.check_call(cmd)
+ except subprocess.CalledProcessError:
+ package = os.path.basename(package_dir)
+ error_out("Error updating {} from "
+ "global-requirements.txt".format(package))
+ os.chdir(orig_dir)
+
+
+def git_pip_venv_dir(projects_yaml):
+ """
+ Return the pip virtualenv path.
+ """
+ parent_dir = '/mnt/openstack-git'
+
+ projects = _git_yaml_load(projects_yaml)
+
+ if 'directory' in projects.keys():
+ parent_dir = projects['directory']
+
+ return os.path.join(parent_dir, 'venv')
+
+
+def git_src_dir(projects_yaml, project):
+ """
+ Return the directory where the specified project's source is located.
+ """
+ parent_dir = '/mnt/openstack-git'
+
+ projects = _git_yaml_load(projects_yaml)
+
+ if 'directory' in projects.keys():
+ parent_dir = projects['directory']
+
+ for p in projects['repositories']:
+ if p['name'] == project:
+ return os.path.join(parent_dir, os.path.basename(p['repository']))
+
+ return None
+
+
+def git_yaml_value(projects_yaml, key):
+ """
+ Return the value in projects_yaml for the specified key.
+ """
+ projects = _git_yaml_load(projects_yaml)
+
+ if key in projects.keys():
+ return projects[key]
+
+ return None
+
+
+def git_generate_systemd_init_files(templates_dir):
+ """
+ Generate systemd init files.
+
+ Generates and installs systemd init units and script files based on the
+ *.init.in files contained in the templates_dir directory.
+
+ This code is based on the openstack-pkg-tools package and its init
+ script generation, which is used by the OpenStack packages.
+ """
+ for f in os.listdir(templates_dir):
+ # Create the init script and systemd unit file from the template
+ if f.endswith(".init.in"):
+ init_in_file = f
+ init_file = f[:-8]
+ service_file = "{}.service".format(init_file)
+
+ init_in_source = os.path.join(templates_dir, init_in_file)
+ init_source = os.path.join(templates_dir, init_file)
+ service_source = os.path.join(templates_dir, service_file)
+
+ init_dest = os.path.join('/etc/init.d', init_file)
+ service_dest = os.path.join('/lib/systemd/system', service_file)
+
+ shutil.copyfile(init_in_source, init_source)
+ with open(init_source, 'a') as outfile:
+ template = ('/usr/share/openstack-pkg-tools/'
+ 'init-script-template')
+ with open(template) as infile:
+ outfile.write('\n\n{}'.format(infile.read()))
+
+ cmd = ['pkgos-gen-systemd-unit', init_in_source]
+ subprocess.check_call(cmd)
+
+ if os.path.exists(init_dest):
+ os.remove(init_dest)
+ if os.path.exists(service_dest):
+ os.remove(service_dest)
+ shutil.copyfile(init_source, init_dest)
+ shutil.copyfile(service_source, service_dest)
+ os.chmod(init_dest, 0o755)
+
+ for f in os.listdir(templates_dir):
+ # If there's a service.in file, use it instead of the generated one
+ if f.endswith(".service.in"):
+ service_in_file = f
+ service_file = f[:-3]
+
+ service_in_source = os.path.join(templates_dir, service_in_file)
+ service_source = os.path.join(templates_dir, service_file)
+ service_dest = os.path.join('/lib/systemd/system', service_file)
+
+ shutil.copyfile(service_in_source, service_source)
+
+ if os.path.exists(service_dest):
+ os.remove(service_dest)
+ shutil.copyfile(service_source, service_dest)
+
+ for f in os.listdir(templates_dir):
+ # Generate the systemd unit if there's no existing .service.in
+ if f.endswith(".init.in"):
+ init_in_file = f
+ init_file = f[:-8]
+ service_in_file = "{}.service.in".format(init_file)
+ service_file = "{}.service".format(init_file)
+
+ init_in_source = os.path.join(templates_dir, init_in_file)
+ service_in_source = os.path.join(templates_dir, service_in_file)
+ service_source = os.path.join(templates_dir, service_file)
+ service_dest = os.path.join('/lib/systemd/system', service_file)
+
+ if not os.path.exists(service_in_source):
+ cmd = ['pkgos-gen-systemd-unit', init_in_source]
+ subprocess.check_call(cmd)
+
+ if os.path.exists(service_dest):
+ os.remove(service_dest)
+ shutil.copyfile(service_source, service_dest)
+
+
+def git_determine_usr_bin():
+ """Return the /usr/bin path for Apache2 config.
+
+ The /usr/bin path will be located in the virtualenv if the charm
+ is configured to deploy from source.
+ """
+ if git_install_requested():
+ projects_yaml = config('openstack-origin-git')
+ projects_yaml = git_default_repos(projects_yaml)
+ return os.path.join(git_pip_venv_dir(projects_yaml), 'bin')
+ else:
+ return '/usr/bin'
+
+
+def git_determine_python_path():
+ """Return the python-path for Apache2 config.
+
+ Returns 'None' unless the charm is configured to deploy from source,
+ in which case the path of the virtualenv's site-packages is returned.
+ """
+ if git_install_requested():
+ projects_yaml = config('openstack-origin-git')
+ projects_yaml = git_default_repos(projects_yaml)
+ return os.path.join(git_pip_venv_dir(projects_yaml),
+ 'lib/python2.7/site-packages')
+ else:
+ return None
+
+
+def os_workload_status(configs, required_interfaces, charm_func=None):
+ """
+ Decorator to set workload status based on complete contexts
+ """
+ def wrap(f):
+ @wraps(f)
+ def wrapped_f(*args, **kwargs):
+ # Run the original function first
+ f(*args, **kwargs)
+ # Set workload status now that contexts have been
+ # acted on
+ set_os_workload_status(configs, required_interfaces, charm_func)
+ return wrapped_f
+ return wrap
+
+
+def set_os_workload_status(configs, required_interfaces, charm_func=None,
+ services=None, ports=None):
+ """Set the state of the workload status for the charm.
+
+ This calls _determine_os_workload_status() to get the new state, message
+ and sets the status using status_set()
+
+ @param configs: a templating.OSConfigRenderer() object
+ @param required_interfaces: {generic: [specific, specific2, ...]}
+ @param charm_func: a callable function that returns state, message. The
+ signature is charm_func(configs) -> (state, message)
+ @param services: list of strings OR dictionary specifying services/ports
+ @param ports: OPTIONAL list of port numbers.
+ @returns state, message: the new workload status, user message
+ """
+ state, message = _determine_os_workload_status(
+ configs, required_interfaces, charm_func, services, ports)
+ status_set(state, message)
+
+
+def _determine_os_workload_status(
+ configs, required_interfaces, charm_func=None,
+ services=None, ports=None):
+ """Determine the state of the workload status for the charm.
+
+ This function returns the new workload status for the charm based
+ on the state of the interfaces, the paused state and whether the
+ services are actually running and any specified ports are open.
+
+ This checks:
+
+ 1. if the unit should be paused, that it is actually paused. If so the
+ state is 'maintenance' + message, else 'broken'.
+ 2. that the interfaces/relations are complete. If they are not then
+ it sets the state to either 'broken' or 'waiting' and an appropriate
+ message.
+ 3. If all the relation data is set, then it checks that the actual
+ services really are running. If not it sets the state to 'broken'.
+
+ If everything is okay then the state returns 'active'.
+
+ @param configs: a templating.OSConfigRenderer() object
+ @param required_interfaces: {generic: [specific, specific2, ...]}
+ @param charm_func: a callable function that returns state, message. The
+ signature is charm_func(configs) -> (state, message)
+ @param services: list of strings OR dictionary specifying services/ports
+ @param ports: OPTIONAL list of port numbers.
+ @returns state, message: the new workload status, user message
+ """
+ state, message = _ows_check_if_paused(services, ports)
+
+ if state is None:
+ state, message = _ows_check_generic_interfaces(
+ configs, required_interfaces)
+
+ if state != 'maintenance' and charm_func:
+ # _ows_check_charm_func() may modify the state, message
+ state, message = _ows_check_charm_func(
+ state, message, lambda: charm_func(configs))
+
+ if state is None:
+ state, message = _ows_check_services_running(services, ports)
+
+ if state is None:
+ state = 'active'
+ message = "Unit is ready"
+ juju_log(message, 'INFO')
+
+ return state, message
+
+
+def _ows_check_if_paused(services=None, ports=None):
+ """Check if the unit is supposed to be paused, and if so check that the
+ services/ports (if passed) are actually stopped/not being listened to.
+
+ if the unit isn't supposed to be paused, just return None, None
+
+ @param services: OPTIONAL services spec or list of service names.
+ @param ports: OPTIONAL list of port numbers.
+ @returns state, message or None, None
+ """
+ if is_unit_paused_set():
+ state, message = check_actually_paused(services=services,
+ ports=ports)
+ if state is None:
+ # we're paused okay, so set maintenance and return
+ state = "maintenance"
+ message = "Paused. Use 'resume' action to resume normal service."
+ return state, message
+ return None, None
+
+
+def _ows_check_generic_interfaces(configs, required_interfaces):
+ """Check the complete contexts to determine the workload status.
+
+ - Checks for missing or incomplete contexts
+ - juju log details of missing required data.
+ - determines the correct workload status
+ - creates an appropriate message for status_set(...)
+
+ if there are no problems then the function returns None, None
+
+ @param configs: a templating.OSConfigRenderer() object
+ @params required_interfaces: {generic_interface: [specific_interface], }
+ @returns state, message or None, None
+ """
+ incomplete_rel_data = incomplete_relation_data(configs,
+ required_interfaces)
+ state = None
+ message = None
+ missing_relations = set()
+ incomplete_relations = set()
+
+ for generic_interface, relations_states in incomplete_rel_data.items():
+ related_interface = None
+ missing_data = {}
+ # Related or not?
+ for interface, relation_state in relations_states.items():
+ if relation_state.get('related'):
+ related_interface = interface
+ missing_data = relation_state.get('missing_data')
+ break
+ # No relation ID for the generic_interface?
+ if not related_interface:
+ juju_log("{} relation is missing and must be related for "
+ "functionality. ".format(generic_interface), 'WARN')
+ state = 'blocked'
+ missing_relations.add(generic_interface)
+ else:
+ # Relation ID eists but no related unit
+ if not missing_data:
+ # Edge case - relation ID exists but departings
+ _hook_name = hook_name()
+ if (('departed' in _hook_name or 'broken' in _hook_name) and
+ related_interface in _hook_name):
+ state = 'blocked'
+ missing_relations.add(generic_interface)
+ juju_log("{} relation's interface, {}, "
+ "relationship is departed or broken "
+ "and is required for functionality."
+ "".format(generic_interface, related_interface),
+ "WARN")
+ # Normal case relation ID exists but no related unit
+ # (joining)
+ else:
+ juju_log("{} relations's interface, {}, is related but has"
+ " no units in the relation."
+ "".format(generic_interface, related_interface),
+ "INFO")
+ # Related unit exists and data missing on the relation
+ else:
+ juju_log("{} relation's interface, {}, is related awaiting "
+ "the following data from the relationship: {}. "
+ "".format(generic_interface, related_interface,
+ ", ".join(missing_data)), "INFO")
+ if state != 'blocked':
+ state = 'waiting'
+ if generic_interface not in missing_relations:
+ incomplete_relations.add(generic_interface)
+
+ if missing_relations:
+ message = "Missing relations: {}".format(", ".join(missing_relations))
+ if incomplete_relations:
+ message += "; incomplete relations: {}" \
+ "".format(", ".join(incomplete_relations))
+ state = 'blocked'
+ elif incomplete_relations:
+ message = "Incomplete relations: {}" \
+ "".format(", ".join(incomplete_relations))
+ state = 'waiting'
+
+ return state, message
+
+
+def _ows_check_charm_func(state, message, charm_func_with_configs):
+ """Run a custom check function for the charm to see if it wants to
+ change the state. This is only run if not in 'maintenance' and
+ tests to see if the new state is more important that the previous
+ one determined by the interfaces/relations check.
+
+ @param state: the previously determined state so far.
+ @param message: the user orientated message so far.
+ @param charm_func: a callable function that returns state, message
+ @returns state, message strings.
+ """
+ if charm_func_with_configs:
+ charm_state, charm_message = charm_func_with_configs()
+ if charm_state != 'active' and charm_state != 'unknown':
+ state = workload_state_compare(state, charm_state)
+ if message:
+ charm_message = charm_message.replace("Incomplete relations: ",
+ "")
+ message = "{}, {}".format(message, charm_message)
+ else:
+ message = charm_message
+ return state, message
+
+
+def _ows_check_services_running(services, ports):
+ """Check that the services that should be running are actually running
+ and that any ports specified are being listened to.
+
+ @param services: list of strings OR dictionary specifying services/ports
+ @param ports: list of ports
+ @returns state, message: strings or None, None
+ """
+ messages = []
+ state = None
+ if services is not None:
+ services = _extract_services_list_helper(services)
+ services_running, running = _check_running_services(services)
+ if not all(running):
+ messages.append(
+ "Services not running that should be: {}"
+ .format(", ".join(_filter_tuples(services_running, False))))
+ state = 'blocked'
+ # also verify that the ports that should be open are open
+ # NB, that ServiceManager objects only OPTIONALLY have ports
+ map_not_open, ports_open = (
+ _check_listening_on_services_ports(services))
+ if not all(ports_open):
+ # find which service has missing ports. They are in service
+ # order which makes it a bit easier.
+ message_parts = {service: ", ".join([str(v) for v in open_ports])
+ for service, open_ports in map_not_open.items()}
+ message = ", ".join(
+ ["{}: [{}]".format(s, sp) for s, sp in message_parts.items()])
+ messages.append(
+ "Services with ports not open that should be: {}"
+ .format(message))
+ state = 'blocked'
+
+ if ports is not None:
+ # and we can also check ports which we don't know the service for
+ ports_open, ports_open_bools = _check_listening_on_ports_list(ports)
+ if not all(ports_open_bools):
+ messages.append(
+ "Ports which should be open, but are not: {}"
+ .format(", ".join([str(p) for p, v in ports_open
+ if not v])))
+ state = 'blocked'
+
+ if state is not None:
+ message = "; ".join(messages)
+ return state, message
+
+ return None, None
+
+
+def _extract_services_list_helper(services):
+ """Extract a OrderedDict of {service: [ports]} of the supplied services
+ for use by the other functions.
+
+ The services object can either be:
+ - None : no services were passed (an empty dict is returned)
+ - a list of strings
+ - A dictionary (optionally OrderedDict) {service_name: {'service': ..}}
+ - An array of [{'service': service_name, ...}, ...]
+
+ @param services: see above
+ @returns OrderedDict(service: [ports], ...)
+ """
+ if services is None:
+ return {}
+ if isinstance(services, dict):
+ services = services.values()
+ # either extract the list of services from the dictionary, or if
+ # it is a simple string, use that. i.e. works with mixed lists.
+ _s = OrderedDict()
+ for s in services:
+ if isinstance(s, dict) and 'service' in s:
+ _s[s['service']] = s.get('ports', [])
+ if isinstance(s, str):
+ _s[s] = []
+ return _s
+
+
+def _check_running_services(services):
+ """Check that the services dict provided is actually running and provide
+ a list of (service, boolean) tuples for each service.
+
+ Returns both a zipped list of (service, boolean) and a list of booleans
+ in the same order as the services.
+
+ @param services: OrderedDict of strings: [ports], one for each service to
+ check.
+ @returns [(service, boolean), ...], : results for checks
+ [boolean] : just the result of the service checks
+ """
+ services_running = [service_running(s) for s in services]
+ return list(zip(services, services_running)), services_running
+
+
+def _check_listening_on_services_ports(services, test=False):
+ """Check that the unit is actually listening (has the port open) on the
+ ports that the service specifies are open. If test is True then the
+ function returns the services with ports that are open rather than
+ closed.
+
+ Returns an OrderedDict of service: ports and a list of booleans
+
+ @param services: OrderedDict(service: [port, ...], ...)
+ @param test: default=False, if False, test for closed, otherwise open.
+ @returns OrderedDict(service: [port-not-open, ...]...), [boolean]
+ """
+ test = not(not(test)) # ensure test is True or False
+ all_ports = list(itertools.chain(*services.values()))
+ ports_states = [port_has_listener('0.0.0.0', p) for p in all_ports]
+ map_ports = OrderedDict()
+ matched_ports = [p for p, opened in zip(all_ports, ports_states)
+ if opened == test] # essentially opened xor test
+ for service, ports in services.items():
+ set_ports = set(ports).intersection(matched_ports)
+ if set_ports:
+ map_ports[service] = set_ports
+ return map_ports, ports_states
+
+
+def _check_listening_on_ports_list(ports):
+ """Check that the ports list given are being listened to
+
+ Returns a list of ports being listened to and a list of the
+ booleans.
+
+ @param ports: LIST or port numbers.
+ @returns [(port_num, boolean), ...], [boolean]
+ """
+ ports_open = [port_has_listener('0.0.0.0', p) for p in ports]
+ return zip(ports, ports_open), ports_open
+
+
+def _filter_tuples(services_states, state):
+ """Return a simple list from a list of tuples according to the condition
+
+ @param services_states: LIST of (string, boolean): service and running
+ state.
+ @param state: Boolean to match the tuple against.
+ @returns [LIST of strings] that matched the tuple RHS.
+ """
+ return [s for s, b in services_states if b == state]
+
+
+def workload_state_compare(current_workload_state, workload_state):
+ """ Return highest priority of two states"""
+ hierarchy = {'unknown': -1,
+ 'active': 0,
+ 'maintenance': 1,
+ 'waiting': 2,
+ 'blocked': 3,
+ }
+
+ if hierarchy.get(workload_state) is None:
+ workload_state = 'unknown'
+ if hierarchy.get(current_workload_state) is None:
+ current_workload_state = 'unknown'
+
+ # Set workload_state based on hierarchy of statuses
+ if hierarchy.get(current_workload_state) > hierarchy.get(workload_state):
+ return current_workload_state
+ else:
+ return workload_state
+
+
+def incomplete_relation_data(configs, required_interfaces):
+ """Check complete contexts against required_interfaces
+ Return dictionary of incomplete relation data.
+
+ configs is an OSConfigRenderer object with configs registered
+
+ required_interfaces is a dictionary of required general interfaces
+ with dictionary values of possible specific interfaces.
+ Example:
+ required_interfaces = {'database': ['shared-db', 'pgsql-db']}
+
+ The interface is said to be satisfied if anyone of the interfaces in the
+ list has a complete context.
+
+ Return dictionary of incomplete or missing required contexts with relation
+ status of interfaces and any missing data points. Example:
+ {'message':
+ {'amqp': {'missing_data': ['rabbitmq_password'], 'related': True},
+ 'zeromq-configuration': {'related': False}},
+ 'identity':
+ {'identity-service': {'related': False}},
+ 'database':
+ {'pgsql-db': {'related': False},
+ 'shared-db': {'related': True}}}
+ """
+ complete_ctxts = configs.complete_contexts()
+ incomplete_relations = [
+ svc_type
+ for svc_type, interfaces in required_interfaces.items()
+ if not set(interfaces).intersection(complete_ctxts)]
+ return {
+ i: configs.get_incomplete_context_data(required_interfaces[i])
+ for i in incomplete_relations}
+
+
+def do_action_openstack_upgrade(package, upgrade_callback, configs):
+ """Perform action-managed OpenStack upgrade.
+
+ Upgrades packages to the configured openstack-origin version and sets
+ the corresponding action status as a result.
+
+ If the charm was installed from source we cannot upgrade it.
+ For backwards compatibility a config flag (action-managed-upgrade) must
+ be set for this code to run, otherwise a full service level upgrade will
+ fire on config-changed.
+
+ @param package: package name for determining if upgrade available
+ @param upgrade_callback: function callback to charm's upgrade function
+ @param configs: templating object derived from OSConfigRenderer class
+
+ @return: True if upgrade successful; False if upgrade failed or skipped
+ """
+ ret = False
+
+ if git_install_requested():
+ action_set({'outcome': 'installed from source, skipped upgrade.'})
+ else:
+ if openstack_upgrade_available(package):
+ if config('action-managed-upgrade'):
+ juju_log('Upgrading OpenStack release')
+
+ try:
+ upgrade_callback(configs=configs)
+ action_set({'outcome': 'success, upgrade completed.'})
+ ret = True
+ except Exception:
+ action_set({'outcome': 'upgrade failed, see traceback.'})
+ action_set({'traceback': traceback.format_exc()})
+ action_fail('do_openstack_upgrade resulted in an '
+ 'unexpected error')
+ else:
+ action_set({'outcome': 'action-managed-upgrade config is '
+ 'False, skipped upgrade.'})
+ else:
+ action_set({'outcome': 'no upgrade available.'})
+
+ return ret
+
+
+def remote_restart(rel_name, remote_service=None):
+ trigger = {
+ 'restart-trigger': str(uuid.uuid4()),
+ }
+ if remote_service:
+ trigger['remote-service'] = remote_service
+ for rid in relation_ids(rel_name):
+ # This subordinate can be related to two seperate services using
+ # different subordinate relations so only issue the restart if
+ # the principle is conencted down the relation we think it is
+ if related_units(relid=rid):
+ relation_set(relation_id=rid,
+ relation_settings=trigger,
+ )
+
+
+def check_actually_paused(services=None, ports=None):
+ """Check that services listed in the services object and and ports
+ are actually closed (not listened to), to verify that the unit is
+ properly paused.
+
+ @param services: See _extract_services_list_helper
+ @returns status, : string for status (None if okay)
+ message : string for problem for status_set
+ """
+ state = None
+ message = None
+ messages = []
+ if services is not None:
+ services = _extract_services_list_helper(services)
+ services_running, services_states = _check_running_services(services)
+ if any(services_states):
+ # there shouldn't be any running so this is a problem
+ messages.append("these services running: {}"
+ .format(", ".join(
+ _filter_tuples(services_running, True))))
+ state = "blocked"
+ ports_open, ports_open_bools = (
+ _check_listening_on_services_ports(services, True))
+ if any(ports_open_bools):
+ message_parts = {service: ", ".join([str(v) for v in open_ports])
+ for service, open_ports in ports_open.items()}
+ message = ", ".join(
+ ["{}: [{}]".format(s, sp) for s, sp in message_parts.items()])
+ messages.append(
+ "these service:ports are open: {}".format(message))
+ state = 'blocked'
+ if ports is not None:
+ ports_open, bools = _check_listening_on_ports_list(ports)
+ if any(bools):
+ messages.append(
+ "these ports which should be closed, but are open: {}"
+ .format(", ".join([str(p) for p, v in ports_open if v])))
+ state = 'blocked'
+ if messages:
+ message = ("Services should be paused but {}"
+ .format(", ".join(messages)))
+ return state, message
+
+
+def set_unit_paused():
+ """Set the unit to a paused state in the local kv() store.
+ This does NOT actually pause the unit
+ """
+ with unitdata.HookData()() as t:
+ kv = t[0]
+ kv.set('unit-paused', True)
+
+
+def clear_unit_paused():
+ """Clear the unit from a paused state in the local kv() store
+ This does NOT actually restart any services - it only clears the
+ local state.
+ """
+ with unitdata.HookData()() as t:
+ kv = t[0]
+ kv.set('unit-paused', False)
+
+
+def is_unit_paused_set():
+ """Return the state of the kv().get('unit-paused').
+ This does NOT verify that the unit really is paused.
+
+ To help with units that don't have HookData() (testing)
+ if it excepts, return False
+ """
+ try:
+ with unitdata.HookData()() as t:
+ kv = t[0]
+ # transform something truth-y into a Boolean.
+ return not(not(kv.get('unit-paused')))
+ except Exception:
+ return False
+
+
+def pause_unit(assess_status_func, services=None, ports=None,
+ charm_func=None):
+ """Pause a unit by stopping the services and setting 'unit-paused'
+ in the local kv() store.
+
+ Also checks that the services have stopped and ports are no longer
+ being listened to.
+
+ An optional charm_func() can be called that can either raise an
+ Exception or return non None, None to indicate that the unit
+ didn't pause cleanly.
+
+ The signature for charm_func is:
+ charm_func() -> message: string
+
+ charm_func() is executed after any services are stopped, if supplied.
+
+ The services object can either be:
+ - None : no services were passed (an empty dict is returned)
+ - a list of strings
+ - A dictionary (optionally OrderedDict) {service_name: {'service': ..}}
+ - An array of [{'service': service_name, ...}, ...]
+
+ @param assess_status_func: (f() -> message: string | None) or None
+ @param services: OPTIONAL see above
+ @param ports: OPTIONAL list of port
+ @param charm_func: function to run for custom charm pausing.
+ @returns None
+ @raises Exception(message) on an error for action_fail().
+ """
+ services = _extract_services_list_helper(services)
+ messages = []
+ if services:
+ for service in services.keys():
+ stopped = service_pause(service)
+ if not stopped:
+ messages.append("{} didn't stop cleanly.".format(service))
+ if charm_func:
+ try:
+ message = charm_func()
+ if message:
+ messages.append(message)
+ except Exception as e:
+ message.append(str(e))
+ set_unit_paused()
+ if assess_status_func:
+ message = assess_status_func()
+ if message:
+ messages.append(message)
+ if messages:
+ raise Exception("Couldn't pause: {}".format("; ".join(messages)))
+
+
+def resume_unit(assess_status_func, services=None, ports=None,
+ charm_func=None):
+ """Resume a unit by starting the services and clearning 'unit-paused'
+ in the local kv() store.
+
+ Also checks that the services have started and ports are being listened to.
+
+ An optional charm_func() can be called that can either raise an
+ Exception or return non None to indicate that the unit
+ didn't resume cleanly.
+
+ The signature for charm_func is:
+ charm_func() -> message: string
+
+ charm_func() is executed after any services are started, if supplied.
+
+ The services object can either be:
+ - None : no services were passed (an empty dict is returned)
+ - a list of strings
+ - A dictionary (optionally OrderedDict) {service_name: {'service': ..}}
+ - An array of [{'service': service_name, ...}, ...]
+
+ @param assess_status_func: (f() -> message: string | None) or None
+ @param services: OPTIONAL see above
+ @param ports: OPTIONAL list of port
+ @param charm_func: function to run for custom charm resuming.
+ @returns None
+ @raises Exception(message) on an error for action_fail().
+ """
+ services = _extract_services_list_helper(services)
+ messages = []
+ if services:
+ for service in services.keys():
+ started = service_resume(service)
+ if not started:
+ messages.append("{} didn't start cleanly.".format(service))
+ if charm_func:
+ try:
+ message = charm_func()
+ if message:
+ messages.append(message)
+ except Exception as e:
+ message.append(str(e))
+ clear_unit_paused()
+ if assess_status_func:
+ message = assess_status_func()
+ if message:
+ messages.append(message)
+ if messages:
+ raise Exception("Couldn't resume: {}".format("; ".join(messages)))
+
+
+def make_assess_status_func(*args, **kwargs):
+ """Creates an assess_status_func() suitable for handing to pause_unit()
+ and resume_unit().
+
+ This uses the _determine_os_workload_status(...) function to determine
+ what the workload_status should be for the unit. If the unit is
+ not in maintenance or active states, then the message is returned to
+ the caller. This is so an action that doesn't result in either a
+ complete pause or complete resume can signal failure with an action_fail()
+ """
+ def _assess_status_func():
+ state, message = _determine_os_workload_status(*args, **kwargs)
+ status_set(state, message)
+ if state not in ['maintenance', 'active']:
+ return message
+ return None
+
+ return _assess_status_func
+
+
+def pausable_restart_on_change(restart_map, stopstart=False,
+ restart_functions=None):
+ """A restart_on_change decorator that checks to see if the unit is
+ paused. If it is paused then the decorated function doesn't fire.
+
+ This is provided as a helper, as the @restart_on_change(...) decorator
+ is in core.host, yet the openstack specific helpers are in this file
+ (contrib.openstack.utils). Thus, this needs to be an optional feature
+ for openstack charms (or charms that wish to use the openstack
+ pause/resume type features).
+
+ It is used as follows:
+
+ from contrib.openstack.utils import (
+ pausable_restart_on_change as restart_on_change)
+
+ @restart_on_change(restart_map, stopstart=)
+ def some_hook(...):
+ pass
+
+ see core.utils.restart_on_change() for more details.
+
+ @param f: the function to decorate
+ @param restart_map: the restart map {conf_file: [services]}
+ @param stopstart: DEFAULT false; whether to stop, start or just restart
+ @returns decorator to use a restart_on_change with pausability
+ """
+ def wrap(f):
+ @functools.wraps(f)
+ def wrapped_f(*args, **kwargs):
+ if is_unit_paused_set():
+ return f(*args, **kwargs)
+ # otherwise, normal restart_on_change functionality
+ return restart_on_change_helper(
+ (lambda: f(*args, **kwargs)), restart_map, stopstart,
+ restart_functions)
+ return wrapped_f
+ return wrap
+
+
+def ordered(orderme):
+ """Converts the provided dictionary into a collections.OrderedDict.
+
+ The items in the returned OrderedDict will be inserted based on the
+ natural sort order of the keys. Nested dictionaries will also be sorted
+ in order to ensure fully predictable ordering.
+
+ :param orderme: the dict to order
+ :return: collections.OrderedDict
+ :raises: ValueError: if `orderme` isn't a dict instance.
+ """
+ if not isinstance(orderme, dict):
+ raise ValueError('argument must be a dict type')
+
+ result = OrderedDict()
+ for k, v in sorted(six.iteritems(orderme), key=lambda x: x[0]):
+ if isinstance(v, dict):
+ result[k] = ordered(v)
+ else:
+ result[k] = v
+
+ return result
+
+
+def config_flags_parser(config_flags):
+ """Parses config flags string into dict.
+
+ This parsing method supports a few different formats for the config
+ flag values to be parsed:
+
+ 1. A string in the simple format of key=value pairs, with the possibility
+ of specifying multiple key value pairs within the same string. For
+ example, a string in the format of 'key1=value1, key2=value2' will
+ return a dict of:
+
+ {'key1': 'value1', 'key2': 'value2'}.
+
+ 2. A string in the above format, but supporting a comma-delimited list
+ of values for the same key. For example, a string in the format of
+ 'key1=value1, key2=value3,value4,value5' will return a dict of:
+
+ {'key1': 'value1', 'key2': 'value2,value3,value4'}
+
+ 3. A string containing a colon character (:) prior to an equal
+ character (=) will be treated as yaml and parsed as such. This can be
+ used to specify more complex key value pairs. For example,
+ a string in the format of 'key1: subkey1=value1, subkey2=value2' will
+ return a dict of:
+
+ {'key1', 'subkey1=value1, subkey2=value2'}
+
+ The provided config_flags string may be a list of comma-separated values
+ which themselves may be comma-separated list of values.
+ """
+ # If we find a colon before an equals sign then treat it as yaml.
+ # Note: limit it to finding the colon first since this indicates assignment
+ # for inline yaml.
+ colon = config_flags.find(':')
+ equals = config_flags.find('=')
+ if colon > 0:
+ if colon < equals or equals < 0:
+ return ordered(yaml.safe_load(config_flags))
+
+ if config_flags.find('==') >= 0:
+ juju_log("config_flags is not in expected format (key=value)",
+ level=ERROR)
+ raise OSContextError
+
+ # strip the following from each value.
+ post_strippers = ' ,'
+ # we strip any leading/trailing '=' or ' ' from the string then
+ # split on '='.
+ split = config_flags.strip(' =').split('=')
+ limit = len(split)
+ flags = OrderedDict()
+ for i in range(0, limit - 1):
+ current = split[i]
+ next = split[i + 1]
+ vindex = next.rfind(',')
+ if (i == limit - 2) or (vindex < 0):
+ value = next
+ else:
+ value = next[:vindex]
+
+ if i == 0:
+ key = current
+ else:
+ # if this not the first entry, expect an embedded key.
+ index = current.rfind(',')
+ if index < 0:
+ juju_log("Invalid config value(s) at index %s" % (i),
+ level=ERROR)
+ raise OSContextError
+ key = current[index + 1:]
+
+ # Add to collection.
+ flags[key.strip(post_strippers)] = value.rstrip(post_strippers)
+
+ return flags
+
+
+def os_application_version_set(package):
+ '''Set version of application for Juju 2.0 and later'''
+ application_version = get_upstream_version(package)
+ # NOTE(jamespage) if not able to figure out package version, fallback to
+ # openstack codename version detection.
+ if not application_version:
+ application_version_set(os_release(package))
+ else:
+ application_version_set(application_version)
+
+
+def enable_memcache(source=None, release=None, package=None):
+ """Determine if memcache should be enabled on the local unit
+
+ @param release: release of OpenStack currently deployed
+ @param package: package to derive OpenStack version deployed
+ @returns boolean Whether memcache should be enabled
+ """
+ _release = None
+ if release:
+ _release = release
+ else:
+ _release = os_release(package, base='icehouse')
+ if not _release:
+ _release = get_os_codename_install_source(source)
+
+ return CompareOpenStackReleases(_release) >= 'mitaka'
+
+
+def token_cache_pkgs(source=None, release=None):
+ """Determine additional packages needed for token caching
+
+ @param source: source string for charm
+ @param release: release of OpenStack currently deployed
+ @returns List of package to enable token caching
+ """
+ packages = []
+ if enable_memcache(source=source, release=release):
+ packages.extend(['memcached', 'python-memcache'])
+ return packages
+
+
+def update_json_file(filename, items):
+ """Updates the json `filename` with a given dict.
+ :param filename: json filename (i.e.: /etc/glance/policy.json)
+ :param items: dict of items to update
+ """
+ with open(filename) as fd:
+ policy = json.load(fd)
+ policy.update(items)
+ with open(filename, "w") as fd:
+ fd.write(json.dumps(policy, indent=4))
+
+
+@cached
+def snap_install_requested():
+ """ Determine if installing from snaps
+
+ If openstack-origin is of the form snap:track/channel[/branch]
+ and channel is in SNAPS_CHANNELS return True.
+ """
+ origin = config('openstack-origin') or ""
+ if not origin.startswith('snap:'):
+ return False
+
+ _src = origin[5:]
+ if '/' in _src:
+ channel = _src.split('/')[1]
+ else:
+ # Handle snap:track with no channel
+ channel = 'stable'
+ return valid_snap_channel(channel)
+
+
+def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
+ """Generate a dictionary of snap install information from origin
+
+ @param snaps: List of snaps
+ @param src: String of openstack-origin or source of the form
+ snap:track/channel
+ @param mode: String classic, devmode or jailmode
+ @returns: Dictionary of snaps with channels and modes
+ """
+
+ if not src.startswith('snap:'):
+ juju_log("Snap source is not a snap origin", 'WARN')
+ return {}
+
+ _src = src[5:]
+ channel = '--channel={}'.format(_src)
+
+ return {snap: {'channel': channel, 'mode': mode}
+ for snap in snaps}
+
+
+def install_os_snaps(snaps, refresh=False):
+ """Install OpenStack snaps from channel and with mode
+
+ @param snaps: Dictionary of snaps with channels and modes of the form:
+ {'snap_name': {'channel': 'snap_channel',
+ 'mode': 'snap_mode'}}
+ Where channel is a snapstore channel and mode is --classic, --devmode
+ or --jailmode.
+ @param post_snap_install: Callback function to run after snaps have been
+ installed
+ """
+
+ def _ensure_flag(flag):
+ if flag.startswith('--'):
+ return flag
+ return '--{}'.format(flag)
+
+ if refresh:
+ for snap in snaps.keys():
+ snap_refresh(snap,
+ _ensure_flag(snaps[snap]['channel']),
+ _ensure_flag(snaps[snap]['mode']))
+ else:
+ for snap in snaps.keys():
+ snap_install(snap,
+ _ensure_flag(snaps[snap]['channel']),
+ _ensure_flag(snaps[snap]['mode']))
diff --git a/tests/charmhelpers/contrib/python/__init__.py b/tests/charmhelpers/contrib/python/__init__.py
new file mode 100644
index 0000000..d7567b8
--- /dev/null
+++ b/tests/charmhelpers/contrib/python/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/tests/charmhelpers/contrib/python/debug.py b/tests/charmhelpers/contrib/python/debug.py
new file mode 100644
index 0000000..d2142c7
--- /dev/null
+++ b/tests/charmhelpers/contrib/python/debug.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import atexit
+import sys
+
+from charmhelpers.contrib.python.rpdb import Rpdb
+from charmhelpers.core.hookenv import (
+ open_port,
+ close_port,
+ ERROR,
+ log
+)
+
+__author__ = "Jorge Niedbalski "
+
+DEFAULT_ADDR = "0.0.0.0"
+DEFAULT_PORT = 4444
+
+
+def _error(message):
+ log(message, level=ERROR)
+
+
+def set_trace(addr=DEFAULT_ADDR, port=DEFAULT_PORT):
+ """
+ Set a trace point using the remote debugger
+ """
+ atexit.register(close_port, port)
+ try:
+ log("Starting a remote python debugger session on %s:%s" % (addr,
+ port))
+ open_port(port)
+ debugger = Rpdb(addr=addr, port=port)
+ debugger.set_trace(sys._getframe().f_back)
+ except Exception:
+ _error("Cannot start a remote debug session on %s:%s" % (addr,
+ port))
diff --git a/tests/charmhelpers/contrib/python/packages.py b/tests/charmhelpers/contrib/python/packages.py
new file mode 100644
index 0000000..6e95028
--- /dev/null
+++ b/tests/charmhelpers/contrib/python/packages.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import six
+import subprocess
+import sys
+
+from charmhelpers.fetch import apt_install, apt_update
+from charmhelpers.core.hookenv import charm_dir, log
+
+__author__ = "Jorge Niedbalski "
+
+
+def pip_execute(*args, **kwargs):
+ """Overriden pip_execute() to stop sys.path being changed.
+
+ The act of importing main from the pip module seems to cause add wheels
+ from the /usr/share/python-wheels which are installed by various tools.
+ This function ensures that sys.path remains the same after the call is
+ executed.
+ """
+ try:
+ _path = sys.path
+ try:
+ from pip import main as _pip_execute
+ except ImportError:
+ apt_update()
+ if six.PY2:
+ apt_install('python-pip')
+ else:
+ apt_install('python3-pip')
+ from pip import main as _pip_execute
+ _pip_execute(*args, **kwargs)
+ finally:
+ sys.path = _path
+
+
+def parse_options(given, available):
+ """Given a set of options, check if available"""
+ for key, value in sorted(given.items()):
+ if not value:
+ continue
+ if key in available:
+ yield "--{0}={1}".format(key, value)
+
+
+def pip_install_requirements(requirements, constraints=None, **options):
+ """Install a requirements file.
+
+ :param constraints: Path to pip constraints file.
+ http://pip.readthedocs.org/en/stable/user_guide/#constraints-files
+ """
+ command = ["install"]
+
+ available_options = ('proxy', 'src', 'log', )
+ for option in parse_options(options, available_options):
+ command.append(option)
+
+ command.append("-r {0}".format(requirements))
+ if constraints:
+ command.append("-c {0}".format(constraints))
+ log("Installing from file: {} with constraints {} "
+ "and options: {}".format(requirements, constraints, command))
+ else:
+ log("Installing from file: {} with options: {}".format(requirements,
+ command))
+ pip_execute(command)
+
+
+def pip_install(package, fatal=False, upgrade=False, venv=None,
+ constraints=None, **options):
+ """Install a python package"""
+ if venv:
+ venv_python = os.path.join(venv, 'bin/pip')
+ command = [venv_python, "install"]
+ else:
+ command = ["install"]
+
+ available_options = ('proxy', 'src', 'log', 'index-url', )
+ for option in parse_options(options, available_options):
+ command.append(option)
+
+ if upgrade:
+ command.append('--upgrade')
+
+ if constraints:
+ command.extend(['-c', constraints])
+
+ if isinstance(package, list):
+ command.extend(package)
+ else:
+ command.append(package)
+
+ log("Installing {} package with options: {}".format(package,
+ command))
+ if venv:
+ subprocess.check_call(command)
+ else:
+ pip_execute(command)
+
+
+def pip_uninstall(package, **options):
+ """Uninstall a python package"""
+ command = ["uninstall", "-q", "-y"]
+
+ available_options = ('proxy', 'log', )
+ for option in parse_options(options, available_options):
+ command.append(option)
+
+ if isinstance(package, list):
+ command.extend(package)
+ else:
+ command.append(package)
+
+ log("Uninstalling {} package with options: {}".format(package,
+ command))
+ pip_execute(command)
+
+
+def pip_list():
+ """Returns the list of current python installed packages
+ """
+ return pip_execute(["list"])
+
+
+def pip_create_virtualenv(path=None):
+ """Create an isolated Python environment."""
+ if six.PY2:
+ apt_install('python-virtualenv')
+ else:
+ apt_install('python3-virtualenv')
+
+ if path:
+ venv_path = path
+ else:
+ venv_path = os.path.join(charm_dir(), 'venv')
+
+ if not os.path.exists(venv_path):
+ subprocess.check_call(['virtualenv', venv_path])
diff --git a/tests/charmhelpers/contrib/python/rpdb.py b/tests/charmhelpers/contrib/python/rpdb.py
new file mode 100644
index 0000000..9b31610
--- /dev/null
+++ b/tests/charmhelpers/contrib/python/rpdb.py
@@ -0,0 +1,56 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Remote Python Debugger (pdb wrapper)."""
+
+import pdb
+import socket
+import sys
+
+__author__ = "Bertrand Janin "
+__version__ = "0.1.3"
+
+
+class Rpdb(pdb.Pdb):
+
+ def __init__(self, addr="127.0.0.1", port=4444):
+ """Initialize the socket and initialize pdb."""
+
+ # Backup stdin and stdout before replacing them by the socket handle
+ self.old_stdout = sys.stdout
+ self.old_stdin = sys.stdin
+
+ # Open a 'reusable' socket to let the webapp reload on the same port
+ self.skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
+ self.skt.bind((addr, port))
+ self.skt.listen(1)
+ (clientsocket, address) = self.skt.accept()
+ handle = clientsocket.makefile('rw')
+ pdb.Pdb.__init__(self, completekey='tab', stdin=handle, stdout=handle)
+ sys.stdout = sys.stdin = handle
+
+ def shutdown(self):
+ """Revert stdin and stdout, close the socket."""
+ sys.stdout = self.old_stdout
+ sys.stdin = self.old_stdin
+ self.skt.close()
+ self.set_continue()
+
+ def do_continue(self, arg):
+ """Stop all operation on ``continue``."""
+ self.shutdown()
+ return 1
+
+ do_EOF = do_quit = do_exit = do_c = do_cont = do_continue
diff --git a/tests/charmhelpers/contrib/python/version.py b/tests/charmhelpers/contrib/python/version.py
new file mode 100644
index 0000000..3eb4210
--- /dev/null
+++ b/tests/charmhelpers/contrib/python/version.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+__author__ = "Jorge Niedbalski "
+
+
+def current_version():
+ """Current system python version"""
+ return sys.version_info
+
+
+def current_version_string():
+ """Current system python version as string major.minor.micro"""
+ return "{0}.{1}.{2}".format(sys.version_info.major,
+ sys.version_info.minor,
+ sys.version_info.micro)
diff --git a/tests/charmhelpers/contrib/storage/__init__.py b/tests/charmhelpers/contrib/storage/__init__.py
new file mode 100644
index 0000000..d7567b8
--- /dev/null
+++ b/tests/charmhelpers/contrib/storage/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/tests/charmhelpers/contrib/storage/linux/__init__.py b/tests/charmhelpers/contrib/storage/linux/__init__.py
new file mode 100644
index 0000000..d7567b8
--- /dev/null
+++ b/tests/charmhelpers/contrib/storage/linux/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/tests/charmhelpers/contrib/storage/linux/bcache.py b/tests/charmhelpers/contrib/storage/linux/bcache.py
new file mode 100644
index 0000000..605991e
--- /dev/null
+++ b/tests/charmhelpers/contrib/storage/linux/bcache.py
@@ -0,0 +1,74 @@
+# Copyright 2017 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import json
+
+from charmhelpers.core.hookenv import log
+
+stats_intervals = ['stats_day', 'stats_five_minute',
+ 'stats_hour', 'stats_total']
+
+SYSFS = '/sys'
+
+
+class Bcache(object):
+ """Bcache behaviour
+ """
+
+ def __init__(self, cachepath):
+ self.cachepath = cachepath
+
+ @classmethod
+ def fromdevice(cls, devname):
+ return cls('{}/block/{}/bcache'.format(SYSFS, devname))
+
+ def __str__(self):
+ return self.cachepath
+
+ def get_stats(self, interval):
+ """Get cache stats
+ """
+ intervaldir = 'stats_{}'.format(interval)
+ path = "{}/{}".format(self.cachepath, intervaldir)
+ out = dict()
+ for elem in os.listdir(path):
+ out[elem] = open('{}/{}'.format(path, elem)).read().strip()
+ return out
+
+
+def get_bcache_fs():
+ """Return all cache sets
+ """
+ cachesetroot = "{}/fs/bcache".format(SYSFS)
+ try:
+ dirs = os.listdir(cachesetroot)
+ except OSError:
+ log("No bcache fs found")
+ return []
+ cacheset = set([Bcache('{}/{}'.format(cachesetroot, d)) for d in dirs if not d.startswith('register')])
+ return cacheset
+
+
+def get_stats_action(cachespec, interval):
+ """Action for getting bcache statistics for a given cachespec.
+ Cachespec can either be a device name, eg. 'sdb', which will retrieve
+ cache stats for the given device, or 'global', which will retrieve stats
+ for all cachesets
+ """
+ if cachespec == 'global':
+ caches = get_bcache_fs()
+ else:
+ caches = [Bcache.fromdevice(cachespec)]
+ res = dict((c.cachepath, c.get_stats(interval)) for c in caches)
+ return json.dumps(res, indent=4, separators=(',', ': '))
diff --git a/tests/charmhelpers/contrib/storage/linux/ceph.py b/tests/charmhelpers/contrib/storage/linux/ceph.py
new file mode 100644
index 0000000..3923161
--- /dev/null
+++ b/tests/charmhelpers/contrib/storage/linux/ceph.py
@@ -0,0 +1,1411 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Copyright 2012 Canonical Ltd.
+#
+# This file is sourced from lp:openstack-charm-helpers
+#
+# Authors:
+# James Page
+# Adam Gandelman
+#
+
+import errno
+import hashlib
+import math
+import six
+
+import os
+import shutil
+import json
+import time
+import uuid
+
+from subprocess import (
+ check_call,
+ check_output,
+ CalledProcessError,
+)
+from charmhelpers.core.hookenv import (
+ config,
+ service_name,
+ local_unit,
+ relation_get,
+ relation_ids,
+ relation_set,
+ related_units,
+ log,
+ DEBUG,
+ INFO,
+ WARNING,
+ ERROR,
+)
+from charmhelpers.core.host import (
+ mount,
+ mounts,
+ service_start,
+ service_stop,
+ service_running,
+ umount,
+)
+from charmhelpers.fetch import (
+ apt_install,
+)
+from charmhelpers.core.unitdata import kv
+
+from charmhelpers.core.kernel import modprobe
+from charmhelpers.contrib.openstack.utils import config_flags_parser
+
+KEYRING = '/etc/ceph/ceph.client.{}.keyring'
+KEYFILE = '/etc/ceph/ceph.client.{}.key'
+
+CEPH_CONF = """[global]
+auth supported = {auth}
+keyring = {keyring}
+mon host = {mon_hosts}
+log to syslog = {use_syslog}
+err to syslog = {use_syslog}
+clog to syslog = {use_syslog}
+"""
+
+# The number of placement groups per OSD to target for placement group
+# calculations. This number is chosen as 100 due to the ceph PG Calc
+# documentation recommending to choose 100 for clusters which are not
+# expected to increase in the foreseeable future. Since the majority of the
+# calculations are done on deployment, target the case of non-expanding
+# clusters as the default.
+DEFAULT_PGS_PER_OSD_TARGET = 100
+DEFAULT_POOL_WEIGHT = 10.0
+LEGACY_PG_COUNT = 200
+DEFAULT_MINIMUM_PGS = 2
+
+
+def validator(value, valid_type, valid_range=None):
+ """
+ Used to validate these: http://docs.ceph.com/docs/master/rados/operations/pools/#set-pool-values
+ Example input:
+ validator(value=1,
+ valid_type=int,
+ valid_range=[0, 2])
+ This says I'm testing value=1. It must be an int inclusive in [0,2]
+
+ :param value: The value to validate
+ :param valid_type: The type that value should be.
+ :param valid_range: A range of values that value can assume.
+ :return:
+ """
+ assert isinstance(value, valid_type), "{} is not a {}".format(
+ value,
+ valid_type)
+ if valid_range is not None:
+ assert isinstance(valid_range, list), \
+ "valid_range must be a list, was given {}".format(valid_range)
+ # If we're dealing with strings
+ if valid_type is six.string_types:
+ assert value in valid_range, \
+ "{} is not in the list {}".format(value, valid_range)
+ # Integer, float should have a min and max
+ else:
+ if len(valid_range) != 2:
+ raise ValueError(
+ "Invalid valid_range list of {} for {}. "
+ "List must be [min,max]".format(valid_range, value))
+ assert value >= valid_range[0], \
+ "{} is less than minimum allowed value of {}".format(
+ value, valid_range[0])
+ assert value <= valid_range[1], \
+ "{} is greater than maximum allowed value of {}".format(
+ value, valid_range[1])
+
+
+class PoolCreationError(Exception):
+ """
+ A custom error to inform the caller that a pool creation failed. Provides an error message
+ """
+
+ def __init__(self, message):
+ super(PoolCreationError, self).__init__(message)
+
+
+class Pool(object):
+ """
+ An object oriented approach to Ceph pool creation. This base class is inherited by ReplicatedPool and ErasurePool.
+ Do not call create() on this base class as it will not do anything. Instantiate a child class and call create().
+ """
+
+ def __init__(self, service, name):
+ self.service = service
+ self.name = name
+
+ # Create the pool if it doesn't exist already
+ # To be implemented by subclasses
+ def create(self):
+ pass
+
+ def add_cache_tier(self, cache_pool, mode):
+ """
+ Adds a new cache tier to an existing pool.
+ :param cache_pool: six.string_types. The cache tier pool name to add.
+ :param mode: six.string_types. The caching mode to use for this pool. valid range = ["readonly", "writeback"]
+ :return: None
+ """
+ # Check the input types and values
+ validator(value=cache_pool, valid_type=six.string_types)
+ validator(value=mode, valid_type=six.string_types, valid_range=["readonly", "writeback"])
+
+ check_call(['ceph', '--id', self.service, 'osd', 'tier', 'add', self.name, cache_pool])
+ check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, mode])
+ check_call(['ceph', '--id', self.service, 'osd', 'tier', 'set-overlay', self.name, cache_pool])
+ check_call(['ceph', '--id', self.service, 'osd', 'pool', 'set', cache_pool, 'hit_set_type', 'bloom'])
+
+ def remove_cache_tier(self, cache_pool):
+ """
+ Removes a cache tier from Ceph. Flushes all dirty objects from writeback pools and waits for that to complete.
+ :param cache_pool: six.string_types. The cache tier pool name to remove.
+ :return: None
+ """
+ # read-only is easy, writeback is much harder
+ mode = get_cache_mode(self.service, cache_pool)
+ version = ceph_version()
+ if mode == 'readonly':
+ check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, 'none'])
+ check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool])
+
+ elif mode == 'writeback':
+ pool_forward_cmd = ['ceph', '--id', self.service, 'osd', 'tier',
+ 'cache-mode', cache_pool, 'forward']
+ if version >= '10.1':
+ # Jewel added a mandatory flag
+ pool_forward_cmd.append('--yes-i-really-mean-it')
+
+ check_call(pool_forward_cmd)
+ # Flush the cache and wait for it to return
+ check_call(['rados', '--id', self.service, '-p', cache_pool, 'cache-flush-evict-all'])
+ check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove-overlay', self.name])
+ check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool])
+
+ def get_pgs(self, pool_size, percent_data=DEFAULT_POOL_WEIGHT):
+ """Return the number of placement groups to use when creating the pool.
+
+ Returns the number of placement groups which should be specified when
+ creating the pool. This is based upon the calculation guidelines
+ provided by the Ceph Placement Group Calculator (located online at
+ http://ceph.com/pgcalc/).
+
+ The number of placement groups are calculated using the following:
+
+ (Target PGs per OSD) * (OSD #) * (%Data)
+ ----------------------------------------
+ (Pool size)
+
+ Per the upstream guidelines, the OSD # should really be considered
+ based on the number of OSDs which are eligible to be selected by the
+ pool. Since the pool creation doesn't specify any of CRUSH set rules,
+ the default rule will be dependent upon the type of pool being
+ created (replicated or erasure).
+
+ This code makes no attempt to determine the number of OSDs which can be
+ selected for the specific rule, rather it is left to the user to tune
+ in the form of 'expected-osd-count' config option.
+
+ :param pool_size: int. pool_size is either the number of replicas for
+ replicated pools or the K+M sum for erasure coded pools
+ :param percent_data: float. the percentage of data that is expected to
+ be contained in the pool for the specific OSD set. Default value
+ is to assume 10% of the data is for this pool, which is a
+ relatively low % of the data but allows for the pg_num to be
+ increased. NOTE: the default is primarily to handle the scenario
+ where related charms requiring pools has not been upgraded to
+ include an update to indicate their relative usage of the pools.
+ :return: int. The number of pgs to use.
+ """
+
+ # Note: This calculation follows the approach that is provided
+ # by the Ceph PG Calculator located at http://ceph.com/pgcalc/.
+ validator(value=pool_size, valid_type=int)
+
+ # Ensure that percent data is set to something - even with a default
+ # it can be set to None, which would wreak havoc below.
+ if percent_data is None:
+ percent_data = DEFAULT_POOL_WEIGHT
+
+ # If the expected-osd-count is specified, then use the max between
+ # the expected-osd-count and the actual osd_count
+ osd_list = get_osds(self.service)
+ expected = config('expected-osd-count') or 0
+
+ if osd_list:
+ osd_count = max(expected, len(osd_list))
+
+ # Log a message to provide some insight if the calculations claim
+ # to be off because someone is setting the expected count and
+ # there are more OSDs in reality. Try to make a proper guess
+ # based upon the cluster itself.
+ if expected and osd_count != expected:
+ log("Found more OSDs than provided expected count. "
+ "Using the actual count instead", INFO)
+ elif expected:
+ # Use the expected-osd-count in older ceph versions to allow for
+ # a more accurate pg calculations
+ osd_count = expected
+ else:
+ # NOTE(james-page): Default to 200 for older ceph versions
+ # which don't support OSD query from cli
+ return LEGACY_PG_COUNT
+
+ percent_data /= 100.0
+ target_pgs_per_osd = config('pgs-per-osd') or DEFAULT_PGS_PER_OSD_TARGET
+ num_pg = (target_pgs_per_osd * osd_count * percent_data) // pool_size
+
+ # NOTE: ensure a sane minimum number of PGS otherwise we don't get any
+ # reasonable data distribution in minimal OSD configurations
+ if num_pg < DEFAULT_MINIMUM_PGS:
+ num_pg = DEFAULT_MINIMUM_PGS
+
+ # The CRUSH algorithm has a slight optimization for placement groups
+ # with powers of 2 so find the nearest power of 2. If the nearest
+ # power of 2 is more than 25% below the original value, the next
+ # highest value is used. To do this, find the nearest power of 2 such
+ # that 2^n <= num_pg, check to see if its within the 25% tolerance.
+ exponent = math.floor(math.log(num_pg, 2))
+ nearest = 2 ** exponent
+ if (num_pg - nearest) > (num_pg * 0.25):
+ # Choose the next highest power of 2 since the nearest is more
+ # than 25% below the original value.
+ return int(nearest * 2)
+ else:
+ return int(nearest)
+
+
+class ReplicatedPool(Pool):
+ def __init__(self, service, name, pg_num=None, replicas=2,
+ percent_data=10.0):
+ super(ReplicatedPool, self).__init__(service=service, name=name)
+ self.replicas = replicas
+ if pg_num:
+ # Since the number of placement groups were specified, ensure
+ # that there aren't too many created.
+ max_pgs = self.get_pgs(self.replicas, 100.0)
+ self.pg_num = min(pg_num, max_pgs)
+ else:
+ self.pg_num = self.get_pgs(self.replicas, percent_data)
+
+ def create(self):
+ if not pool_exists(self.service, self.name):
+ # Create it
+ cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create',
+ self.name, str(self.pg_num)]
+ try:
+ check_call(cmd)
+ # Set the pool replica size
+ update_pool(client=self.service,
+ pool=self.name,
+ settings={'size': str(self.replicas)})
+ except CalledProcessError:
+ raise
+
+
+# Default jerasure erasure coded pool
+class ErasurePool(Pool):
+ def __init__(self, service, name, erasure_code_profile="default",
+ percent_data=10.0):
+ super(ErasurePool, self).__init__(service=service, name=name)
+ self.erasure_code_profile = erasure_code_profile
+ self.percent_data = percent_data
+
+ def create(self):
+ if not pool_exists(self.service, self.name):
+ # Try to find the erasure profile information in order to properly
+ # size the number of placement groups. The size of an erasure
+ # coded placement group is calculated as k+m.
+ erasure_profile = get_erasure_profile(self.service,
+ self.erasure_code_profile)
+
+ # Check for errors
+ if erasure_profile is None:
+ msg = ("Failed to discover erasure profile named "
+ "{}".format(self.erasure_code_profile))
+ log(msg, level=ERROR)
+ raise PoolCreationError(msg)
+ if 'k' not in erasure_profile or 'm' not in erasure_profile:
+ # Error
+ msg = ("Unable to find k (data chunks) or m (coding chunks) "
+ "in erasure profile {}".format(erasure_profile))
+ log(msg, level=ERROR)
+ raise PoolCreationError(msg)
+
+ k = int(erasure_profile['k'])
+ m = int(erasure_profile['m'])
+ pgs = self.get_pgs(k + m, self.percent_data)
+ # Create it
+ cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create',
+ self.name, str(pgs), str(pgs),
+ 'erasure', self.erasure_code_profile]
+ try:
+ check_call(cmd)
+ except CalledProcessError:
+ raise
+
+ """Get an existing erasure code profile if it already exists.
+ Returns json formatted output"""
+
+
+def get_mon_map(service):
+ """
+ Returns the current monitor map.
+ :param service: six.string_types. The Ceph user name to run the command under
+ :return: json string. :raise: ValueError if the monmap fails to parse.
+ Also raises CalledProcessError if our ceph command fails
+ """
+ try:
+ mon_status = check_output(['ceph', '--id', service,
+ 'mon_status', '--format=json'])
+ if six.PY3:
+ mon_status = mon_status.decode('UTF-8')
+ try:
+ return json.loads(mon_status)
+ except ValueError as v:
+ log("Unable to parse mon_status json: {}. Error: {}".format(
+ mon_status, v.message))
+ raise
+ except CalledProcessError as e:
+ log("mon_status command failed with message: {}".format(
+ e.message))
+ raise
+
+
+def hash_monitor_names(service):
+ """
+ Uses the get_mon_map() function to get information about the monitor
+ cluster.
+ Hash the name of each monitor. Return a sorted list of monitor hashes
+ in an ascending order.
+ :param service: six.string_types. The Ceph user name to run the command under
+ :rtype : dict. json dict of monitor name, ip address and rank
+ example: {
+ 'name': 'ip-172-31-13-165',
+ 'rank': 0,
+ 'addr': '172.31.13.165:6789/0'}
+ """
+ try:
+ hash_list = []
+ monitor_list = get_mon_map(service=service)
+ if monitor_list['monmap']['mons']:
+ for mon in monitor_list['monmap']['mons']:
+ hash_list.append(
+ hashlib.sha224(mon['name'].encode('utf-8')).hexdigest())
+ return sorted(hash_list)
+ else:
+ return None
+ except (ValueError, CalledProcessError):
+ raise
+
+
+def monitor_key_delete(service, key):
+ """
+ Delete a key and value pair from the monitor cluster
+ :param service: six.string_types. The Ceph user name to run the command under
+ Deletes a key value pair on the monitor cluster.
+ :param key: six.string_types. The key to delete.
+ """
+ try:
+ check_output(
+ ['ceph', '--id', service,
+ 'config-key', 'del', str(key)])
+ except CalledProcessError as e:
+ log("Monitor config-key put failed with message: {}".format(
+ e.output))
+ raise
+
+
+def monitor_key_set(service, key, value):
+ """
+ Sets a key value pair on the monitor cluster.
+ :param service: six.string_types. The Ceph user name to run the command under
+ :param key: six.string_types. The key to set.
+ :param value: The value to set. This will be converted to a string
+ before setting
+ """
+ try:
+ check_output(
+ ['ceph', '--id', service,
+ 'config-key', 'put', str(key), str(value)])
+ except CalledProcessError as e:
+ log("Monitor config-key put failed with message: {}".format(
+ e.output))
+ raise
+
+
+def monitor_key_get(service, key):
+ """
+ Gets the value of an existing key in the monitor cluster.
+ :param service: six.string_types. The Ceph user name to run the command under
+ :param key: six.string_types. The key to search for.
+ :return: Returns the value of that key or None if not found.
+ """
+ try:
+ output = check_output(
+ ['ceph', '--id', service,
+ 'config-key', 'get', str(key)]).decode('UTF-8')
+ return output
+ except CalledProcessError as e:
+ log("Monitor config-key get failed with message: {}".format(
+ e.output))
+ return None
+
+
+def monitor_key_exists(service, key):
+ """
+ Searches for the existence of a key in the monitor cluster.
+ :param service: six.string_types. The Ceph user name to run the command under
+ :param key: six.string_types. The key to search for
+ :return: Returns True if the key exists, False if not and raises an
+ exception if an unknown error occurs. :raise: CalledProcessError if
+ an unknown error occurs
+ """
+ try:
+ check_call(
+ ['ceph', '--id', service,
+ 'config-key', 'exists', str(key)])
+ # I can return true here regardless because Ceph returns
+ # ENOENT if the key wasn't found
+ return True
+ except CalledProcessError as e:
+ if e.returncode == errno.ENOENT:
+ return False
+ else:
+ log("Unknown error from ceph config-get exists: {} {}".format(
+ e.returncode, e.output))
+ raise
+
+
+def get_erasure_profile(service, name):
+ """
+ :param service: six.string_types. The Ceph user name to run the command under
+ :param name:
+ :return:
+ """
+ try:
+ out = check_output(['ceph', '--id', service,
+ 'osd', 'erasure-code-profile', 'get',
+ name, '--format=json'])
+ if six.PY3:
+ out = out.decode('UTF-8')
+ return json.loads(out)
+ except (CalledProcessError, OSError, ValueError):
+ return None
+
+
+def pool_set(service, pool_name, key, value):
+ """
+ Sets a value for a RADOS pool in ceph.
+ :param service: six.string_types. The Ceph user name to run the command under
+ :param pool_name: six.string_types
+ :param key: six.string_types
+ :param value:
+ :return: None. Can raise CalledProcessError
+ """
+ cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', pool_name, key, value]
+ try:
+ check_call(cmd)
+ except CalledProcessError:
+ raise
+
+
+def snapshot_pool(service, pool_name, snapshot_name):
+ """
+ Snapshots a RADOS pool in ceph.
+ :param service: six.string_types. The Ceph user name to run the command under
+ :param pool_name: six.string_types
+ :param snapshot_name: six.string_types
+ :return: None. Can raise CalledProcessError
+ """
+ cmd = ['ceph', '--id', service, 'osd', 'pool', 'mksnap', pool_name, snapshot_name]
+ try:
+ check_call(cmd)
+ except CalledProcessError:
+ raise
+
+
+def remove_pool_snapshot(service, pool_name, snapshot_name):
+ """
+ Remove a snapshot from a RADOS pool in ceph.
+ :param service: six.string_types. The Ceph user name to run the command under
+ :param pool_name: six.string_types
+ :param snapshot_name: six.string_types
+ :return: None. Can raise CalledProcessError
+ """
+ cmd = ['ceph', '--id', service, 'osd', 'pool', 'rmsnap', pool_name, snapshot_name]
+ try:
+ check_call(cmd)
+ except CalledProcessError:
+ raise
+
+
+# max_bytes should be an int or long
+def set_pool_quota(service, pool_name, max_bytes):
+ """
+ :param service: six.string_types. The Ceph user name to run the command under
+ :param pool_name: six.string_types
+ :param max_bytes: int or long
+ :return: None. Can raise CalledProcessError
+ """
+ # Set a byte quota on a RADOS pool in ceph.
+ cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name,
+ 'max_bytes', str(max_bytes)]
+ try:
+ check_call(cmd)
+ except CalledProcessError:
+ raise
+
+
+def remove_pool_quota(service, pool_name):
+ """
+ Set a byte quota on a RADOS pool in ceph.
+ :param service: six.string_types. The Ceph user name to run the command under
+ :param pool_name: six.string_types
+ :return: None. Can raise CalledProcessError
+ """
+ cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, 'max_bytes', '0']
+ try:
+ check_call(cmd)
+ except CalledProcessError:
+ raise
+
+
+def remove_erasure_profile(service, profile_name):
+ """
+ Create a new erasure code profile if one does not already exist for it. Updates
+ the profile if it exists. Please see http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/
+ for more details
+ :param service: six.string_types. The Ceph user name to run the command under
+ :param profile_name: six.string_types
+ :return: None. Can raise CalledProcessError
+ """
+ cmd = ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'rm',
+ profile_name]
+ try:
+ check_call(cmd)
+ except CalledProcessError:
+ raise
+
+
+def create_erasure_profile(service, profile_name, erasure_plugin_name='jerasure',
+ failure_domain='host',
+ data_chunks=2, coding_chunks=1,
+ locality=None, durability_estimator=None):
+ """
+ Create a new erasure code profile if one does not already exist for it. Updates
+ the profile if it exists. Please see http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/
+ for more details
+ :param service: six.string_types. The Ceph user name to run the command under
+ :param profile_name: six.string_types
+ :param erasure_plugin_name: six.string_types
+ :param failure_domain: six.string_types. One of ['chassis', 'datacenter', 'host', 'osd', 'pdu', 'pod', 'rack', 'region',
+ 'room', 'root', 'row'])
+ :param data_chunks: int
+ :param coding_chunks: int
+ :param locality: int
+ :param durability_estimator: int
+ :return: None. Can raise CalledProcessError
+ """
+ # Ensure this failure_domain is allowed by Ceph
+ validator(failure_domain, six.string_types,
+ ['chassis', 'datacenter', 'host', 'osd', 'pdu', 'pod', 'rack', 'region', 'room', 'root', 'row'])
+
+ cmd = ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'set', profile_name,
+ 'plugin=' + erasure_plugin_name, 'k=' + str(data_chunks), 'm=' + str(coding_chunks),
+ 'ruleset_failure_domain=' + failure_domain]
+ if locality is not None and durability_estimator is not None:
+ raise ValueError("create_erasure_profile should be called with k, m and one of l or c but not both.")
+
+ # Add plugin specific information
+ if locality is not None:
+ # For local erasure codes
+ cmd.append('l=' + str(locality))
+ if durability_estimator is not None:
+ # For Shec erasure codes
+ cmd.append('c=' + str(durability_estimator))
+
+ if erasure_profile_exists(service, profile_name):
+ cmd.append('--force')
+
+ try:
+ check_call(cmd)
+ except CalledProcessError:
+ raise
+
+
+def rename_pool(service, old_name, new_name):
+ """
+ Rename a Ceph pool from old_name to new_name
+ :param service: six.string_types. The Ceph user name to run the command under
+ :param old_name: six.string_types
+ :param new_name: six.string_types
+ :return: None
+ """
+ validator(value=old_name, valid_type=six.string_types)
+ validator(value=new_name, valid_type=six.string_types)
+
+ cmd = ['ceph', '--id', service, 'osd', 'pool', 'rename', old_name, new_name]
+ check_call(cmd)
+
+
+def erasure_profile_exists(service, name):
+ """
+ Check to see if an Erasure code profile already exists.
+ :param service: six.string_types. The Ceph user name to run the command under
+ :param name: six.string_types
+ :return: int or None
+ """
+ validator(value=name, valid_type=six.string_types)
+ try:
+ check_call(['ceph', '--id', service,
+ 'osd', 'erasure-code-profile', 'get',
+ name])
+ return True
+ except CalledProcessError:
+ return False
+
+
+def get_cache_mode(service, pool_name):
+ """
+ Find the current caching mode of the pool_name given.
+ :param service: six.string_types. The Ceph user name to run the command under
+ :param pool_name: six.string_types
+ :return: int or None
+ """
+ validator(value=service, valid_type=six.string_types)
+ validator(value=pool_name, valid_type=six.string_types)
+ out = check_output(['ceph', '--id', service,
+ 'osd', 'dump', '--format=json'])
+ if six.PY3:
+ out = out.decode('UTF-8')
+ try:
+ osd_json = json.loads(out)
+ for pool in osd_json['pools']:
+ if pool['pool_name'] == pool_name:
+ return pool['cache_mode']
+ return None
+ except ValueError:
+ raise
+
+
+def pool_exists(service, name):
+ """Check to see if a RADOS pool already exists."""
+ try:
+ out = check_output(['rados', '--id', service, 'lspools'])
+ if six.PY3:
+ out = out.decode('UTF-8')
+ except CalledProcessError:
+ return False
+
+ return name in out.split()
+
+
+def get_osds(service):
+ """Return a list of all Ceph Object Storage Daemons currently in the
+ cluster.
+ """
+ version = ceph_version()
+ if version and version >= '0.56':
+ out = check_output(['ceph', '--id', service,
+ 'osd', 'ls',
+ '--format=json'])
+ if six.PY3:
+ out = out.decode('UTF-8')
+ return json.loads(out)
+
+ return None
+
+
+def install():
+ """Basic Ceph client installation."""
+ ceph_dir = "/etc/ceph"
+ if not os.path.exists(ceph_dir):
+ os.mkdir(ceph_dir)
+
+ apt_install('ceph-common', fatal=True)
+
+
+def rbd_exists(service, pool, rbd_img):
+ """Check to see if a RADOS block device exists."""
+ try:
+ out = check_output(['rbd', 'list', '--id',
+ service, '--pool', pool])
+ if six.PY3:
+ out = out.decode('UTF-8')
+ except CalledProcessError:
+ return False
+
+ return rbd_img in out
+
+
+def create_rbd_image(service, pool, image, sizemb):
+ """Create a new RADOS block device."""
+ cmd = ['rbd', 'create', image, '--size', str(sizemb), '--id', service,
+ '--pool', pool]
+ check_call(cmd)
+
+
+def update_pool(client, pool, settings):
+ cmd = ['ceph', '--id', client, 'osd', 'pool', 'set', pool]
+ for k, v in six.iteritems(settings):
+ cmd.append(k)
+ cmd.append(v)
+
+ check_call(cmd)
+
+
+def create_pool(service, name, replicas=3, pg_num=None):
+ """Create a new RADOS pool."""
+ if pool_exists(service, name):
+ log("Ceph pool {} already exists, skipping creation".format(name),
+ level=WARNING)
+ return
+
+ if not pg_num:
+ # Calculate the number of placement groups based
+ # on upstream recommended best practices.
+ osds = get_osds(service)
+ if osds:
+ pg_num = (len(osds) * 100 // replicas)
+ else:
+ # NOTE(james-page): Default to 200 for older ceph versions
+ # which don't support OSD query from cli
+ pg_num = 200
+
+ cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pg_num)]
+ check_call(cmd)
+
+ update_pool(service, name, settings={'size': str(replicas)})
+
+
+def delete_pool(service, name):
+ """Delete a RADOS pool from ceph."""
+ cmd = ['ceph', '--id', service, 'osd', 'pool', 'delete', name,
+ '--yes-i-really-really-mean-it']
+ check_call(cmd)
+
+
+def _keyfile_path(service):
+ return KEYFILE.format(service)
+
+
+def _keyring_path(service):
+ return KEYRING.format(service)
+
+
+def create_keyring(service, key):
+ """Create a new Ceph keyring containing key."""
+ keyring = _keyring_path(service)
+ if os.path.exists(keyring):
+ log('Ceph keyring exists at %s.' % keyring, level=WARNING)
+ return
+
+ cmd = ['ceph-authtool', keyring, '--create-keyring',
+ '--name=client.{}'.format(service), '--add-key={}'.format(key)]
+ check_call(cmd)
+ log('Created new ceph keyring at %s.' % keyring, level=DEBUG)
+
+
+def delete_keyring(service):
+ """Delete an existing Ceph keyring."""
+ keyring = _keyring_path(service)
+ if not os.path.exists(keyring):
+ log('Keyring does not exist at %s' % keyring, level=WARNING)
+ return
+
+ os.remove(keyring)
+ log('Deleted ring at %s.' % keyring, level=INFO)
+
+
+def create_key_file(service, key):
+ """Create a file containing key."""
+ keyfile = _keyfile_path(service)
+ if os.path.exists(keyfile):
+ log('Keyfile exists at %s.' % keyfile, level=WARNING)
+ return
+
+ with open(keyfile, 'w') as fd:
+ fd.write(key)
+
+ log('Created new keyfile at %s.' % keyfile, level=INFO)
+
+
+def get_ceph_nodes(relation='ceph'):
+ """Query named relation to determine current nodes."""
+ hosts = []
+ for r_id in relation_ids(relation):
+ for unit in related_units(r_id):
+ hosts.append(relation_get('private-address', unit=unit, rid=r_id))
+
+ return hosts
+
+
+def configure(service, key, auth, use_syslog):
+ """Perform basic configuration of Ceph."""
+ create_keyring(service, key)
+ create_key_file(service, key)
+ hosts = get_ceph_nodes()
+ with open('/etc/ceph/ceph.conf', 'w') as ceph_conf:
+ ceph_conf.write(CEPH_CONF.format(auth=auth,
+ keyring=_keyring_path(service),
+ mon_hosts=",".join(map(str, hosts)),
+ use_syslog=use_syslog))
+ modprobe('rbd')
+
+
+def image_mapped(name):
+ """Determine whether a RADOS block device is mapped locally."""
+ try:
+ out = check_output(['rbd', 'showmapped'])
+ if six.PY3:
+ out = out.decode('UTF-8')
+ except CalledProcessError:
+ return False
+
+ return name in out
+
+
+def map_block_storage(service, pool, image):
+ """Map a RADOS block device for local use."""
+ cmd = [
+ 'rbd',
+ 'map',
+ '{}/{}'.format(pool, image),
+ '--user',
+ service,
+ '--secret',
+ _keyfile_path(service),
+ ]
+ check_call(cmd)
+
+
+def filesystem_mounted(fs):
+ """Determine whether a filesytems is already mounted."""
+ return fs in [f for f, m in mounts()]
+
+
+def make_filesystem(blk_device, fstype='ext4', timeout=10):
+ """Make a new filesystem on the specified block device."""
+ count = 0
+ e_noent = os.errno.ENOENT
+ while not os.path.exists(blk_device):
+ if count >= timeout:
+ log('Gave up waiting on block device %s' % blk_device,
+ level=ERROR)
+ raise IOError(e_noent, os.strerror(e_noent), blk_device)
+
+ log('Waiting for block device %s to appear' % blk_device,
+ level=DEBUG)
+ count += 1
+ time.sleep(1)
+ else:
+ log('Formatting block device %s as filesystem %s.' %
+ (blk_device, fstype), level=INFO)
+ check_call(['mkfs', '-t', fstype, blk_device])
+
+
+def place_data_on_block_device(blk_device, data_src_dst):
+ """Migrate data in data_src_dst to blk_device and then remount."""
+ # mount block device into /mnt
+ mount(blk_device, '/mnt')
+ # copy data to /mnt
+ copy_files(data_src_dst, '/mnt')
+ # umount block device
+ umount('/mnt')
+ # Grab user/group ID's from original source
+ _dir = os.stat(data_src_dst)
+ uid = _dir.st_uid
+ gid = _dir.st_gid
+ # re-mount where the data should originally be
+ # TODO: persist is currently a NO-OP in core.host
+ mount(blk_device, data_src_dst, persist=True)
+ # ensure original ownership of new mount.
+ os.chown(data_src_dst, uid, gid)
+
+
+def copy_files(src, dst, symlinks=False, ignore=None):
+ """Copy files from src to dst."""
+ for item in os.listdir(src):
+ s = os.path.join(src, item)
+ d = os.path.join(dst, item)
+ if os.path.isdir(s):
+ shutil.copytree(s, d, symlinks, ignore)
+ else:
+ shutil.copy2(s, d)
+
+
+def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
+ blk_device, fstype, system_services=[],
+ replicas=3):
+ """NOTE: This function must only be called from a single service unit for
+ the same rbd_img otherwise data loss will occur.
+
+ Ensures given pool and RBD image exists, is mapped to a block device,
+ and the device is formatted and mounted at the given mount_point.
+
+ If formatting a device for the first time, data existing at mount_point
+ will be migrated to the RBD device before being re-mounted.
+
+ All services listed in system_services will be stopped prior to data
+ migration and restarted when complete.
+ """
+ # Ensure pool, RBD image, RBD mappings are in place.
+ if not pool_exists(service, pool):
+ log('Creating new pool {}.'.format(pool), level=INFO)
+ create_pool(service, pool, replicas=replicas)
+
+ if not rbd_exists(service, pool, rbd_img):
+ log('Creating RBD image ({}).'.format(rbd_img), level=INFO)
+ create_rbd_image(service, pool, rbd_img, sizemb)
+
+ if not image_mapped(rbd_img):
+ log('Mapping RBD Image {} as a Block Device.'.format(rbd_img),
+ level=INFO)
+ map_block_storage(service, pool, rbd_img)
+
+ # make file system
+ # TODO: What happens if for whatever reason this is run again and
+ # the data is already in the rbd device and/or is mounted??
+ # When it is mounted already, it will fail to make the fs
+ # XXX: This is really sketchy! Need to at least add an fstab entry
+ # otherwise this hook will blow away existing data if its executed
+ # after a reboot.
+ if not filesystem_mounted(mount_point):
+ make_filesystem(blk_device, fstype)
+
+ for svc in system_services:
+ if service_running(svc):
+ log('Stopping services {} prior to migrating data.'
+ .format(svc), level=DEBUG)
+ service_stop(svc)
+
+ place_data_on_block_device(blk_device, mount_point)
+
+ for svc in system_services:
+ log('Starting service {} after migrating data.'
+ .format(svc), level=DEBUG)
+ service_start(svc)
+
+
+def ensure_ceph_keyring(service, user=None, group=None,
+ relation='ceph', key=None):
+ """Ensures a ceph keyring is created for a named service and optionally
+ ensures user and group ownership.
+
+ @returns boolean: Flag to indicate whether a key was successfully written
+ to disk based on either relation data or a supplied key
+ """
+ if not key:
+ for rid in relation_ids(relation):
+ for unit in related_units(rid):
+ key = relation_get('key', rid=rid, unit=unit)
+ if key:
+ break
+
+ if not key:
+ return False
+
+ create_keyring(service=service, key=key)
+ keyring = _keyring_path(service)
+ if user and group:
+ check_call(['chown', '%s.%s' % (user, group), keyring])
+
+ return True
+
+
+def ceph_version():
+ """Retrieve the local version of ceph."""
+ if os.path.exists('/usr/bin/ceph'):
+ cmd = ['ceph', '-v']
+ output = check_output(cmd)
+ if six.PY3:
+ output = output.decode('UTF-8')
+ output = output.split()
+ if len(output) > 3:
+ return output[2]
+ else:
+ return None
+ else:
+ return None
+
+
+class CephBrokerRq(object):
+ """Ceph broker request.
+
+ Multiple operations can be added to a request and sent to the Ceph broker
+ to be executed.
+
+ Request is json-encoded for sending over the wire.
+
+ The API is versioned and defaults to version 1.
+ """
+
+ def __init__(self, api_version=1, request_id=None):
+ self.api_version = api_version
+ if request_id:
+ self.request_id = request_id
+ else:
+ self.request_id = str(uuid.uuid1())
+ self.ops = []
+
+ def add_op_request_access_to_group(self, name, namespace=None,
+ permission=None, key_name=None):
+ """
+ Adds the requested permissions to the current service's Ceph key,
+ allowing the key to access only the specified pools
+ """
+ self.ops.append({'op': 'add-permissions-to-key', 'group': name,
+ 'namespace': namespace, 'name': key_name or service_name(),
+ 'group-permission': permission})
+
+ def add_op_create_pool(self, name, replica_count=3, pg_num=None,
+ weight=None, group=None, namespace=None):
+ """Adds an operation to create a pool.
+
+ @param pg_num setting: optional setting. If not provided, this value
+ will be calculated by the broker based on how many OSDs are in the
+ cluster at the time of creation. Note that, if provided, this value
+ will be capped at the current available maximum.
+ @param weight: the percentage of data the pool makes up
+ """
+ if pg_num and weight:
+ raise ValueError('pg_num and weight are mutually exclusive')
+
+ self.ops.append({'op': 'create-pool', 'name': name,
+ 'replicas': replica_count, 'pg_num': pg_num,
+ 'weight': weight, 'group': group,
+ 'group-namespace': namespace})
+
+ def set_ops(self, ops):
+ """Set request ops to provided value.
+
+ Useful for injecting ops that come from a previous request
+ to allow comparisons to ensure validity.
+ """
+ self.ops = ops
+
+ @property
+ def request(self):
+ return json.dumps({'api-version': self.api_version, 'ops': self.ops,
+ 'request-id': self.request_id})
+
+ def _ops_equal(self, other):
+ if len(self.ops) == len(other.ops):
+ for req_no in range(0, len(self.ops)):
+ for key in ['replicas', 'name', 'op', 'pg_num', 'weight']:
+ if self.ops[req_no].get(key) != other.ops[req_no].get(key):
+ return False
+ else:
+ return False
+ return True
+
+ def __eq__(self, other):
+ if not isinstance(other, self.__class__):
+ return False
+ if self.api_version == other.api_version and \
+ self._ops_equal(other):
+ return True
+ else:
+ return False
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+
+class CephBrokerRsp(object):
+ """Ceph broker response.
+
+ Response is json-decoded and contents provided as methods/properties.
+
+ The API is versioned and defaults to version 1.
+ """
+
+ def __init__(self, encoded_rsp):
+ self.api_version = None
+ self.rsp = json.loads(encoded_rsp)
+
+ @property
+ def request_id(self):
+ return self.rsp.get('request-id')
+
+ @property
+ def exit_code(self):
+ return self.rsp.get('exit-code')
+
+ @property
+ def exit_msg(self):
+ return self.rsp.get('stderr')
+
+
+# Ceph Broker Conversation:
+# If a charm needs an action to be taken by ceph it can create a CephBrokerRq
+# and send that request to ceph via the ceph relation. The CephBrokerRq has a
+# unique id so that the client can identity which CephBrokerRsp is associated
+# with the request. Ceph will also respond to each client unit individually
+# creating a response key per client unit eg glance/0 will get a CephBrokerRsp
+# via key broker-rsp-glance-0
+#
+# To use this the charm can just do something like:
+#
+# from charmhelpers.contrib.storage.linux.ceph import (
+# send_request_if_needed,
+# is_request_complete,
+# CephBrokerRq,
+# )
+#
+# @hooks.hook('ceph-relation-changed')
+# def ceph_changed():
+# rq = CephBrokerRq()
+# rq.add_op_create_pool(name='poolname', replica_count=3)
+#
+# if is_request_complete(rq):
+#
+# else:
+# send_request_if_needed(get_ceph_request())
+#
+# CephBrokerRq and CephBrokerRsp are serialized into JSON. Below is an example
+# of glance having sent a request to ceph which ceph has successfully processed
+# 'ceph:8': {
+# 'ceph/0': {
+# 'auth': 'cephx',
+# 'broker-rsp-glance-0': '{"request-id": "0bc7dc54", "exit-code": 0}',
+# 'broker_rsp': '{"request-id": "0da543b8", "exit-code": 0}',
+# 'ceph-public-address': '10.5.44.103',
+# 'key': 'AQCLDttVuHXINhAAvI144CB09dYchhHyTUY9BQ==',
+# 'private-address': '10.5.44.103',
+# },
+# 'glance/0': {
+# 'broker_req': ('{"api-version": 1, "request-id": "0bc7dc54", '
+# '"ops": [{"replicas": 3, "name": "glance", '
+# '"op": "create-pool"}]}'),
+# 'private-address': '10.5.44.109',
+# },
+# }
+
+def get_previous_request(rid):
+ """Return the last ceph broker request sent on a given relation
+
+ @param rid: Relation id to query for request
+ """
+ request = None
+ broker_req = relation_get(attribute='broker_req', rid=rid,
+ unit=local_unit())
+ if broker_req:
+ request_data = json.loads(broker_req)
+ request = CephBrokerRq(api_version=request_data['api-version'],
+ request_id=request_data['request-id'])
+ request.set_ops(request_data['ops'])
+
+ return request
+
+
+def get_request_states(request, relation='ceph'):
+ """Return a dict of requests per relation id with their corresponding
+ completion state.
+
+ This allows a charm, which has a request for ceph, to see whether there is
+ an equivalent request already being processed and if so what state that
+ request is in.
+
+ @param request: A CephBrokerRq object
+ """
+ complete = []
+ requests = {}
+ for rid in relation_ids(relation):
+ complete = False
+ previous_request = get_previous_request(rid)
+ if request == previous_request:
+ sent = True
+ complete = is_request_complete_for_rid(previous_request, rid)
+ else:
+ sent = False
+ complete = False
+
+ requests[rid] = {
+ 'sent': sent,
+ 'complete': complete,
+ }
+
+ return requests
+
+
+def is_request_sent(request, relation='ceph'):
+ """Check to see if a functionally equivalent request has already been sent
+
+ Returns True if a similair request has been sent
+
+ @param request: A CephBrokerRq object
+ """
+ states = get_request_states(request, relation=relation)
+ for rid in states.keys():
+ if not states[rid]['sent']:
+ return False
+
+ return True
+
+
+def is_request_complete(request, relation='ceph'):
+ """Check to see if a functionally equivalent request has already been
+ completed
+
+ Returns True if a similair request has been completed
+
+ @param request: A CephBrokerRq object
+ """
+ states = get_request_states(request, relation=relation)
+ for rid in states.keys():
+ if not states[rid]['complete']:
+ return False
+
+ return True
+
+
+def is_request_complete_for_rid(request, rid):
+ """Check if a given request has been completed on the given relation
+
+ @param request: A CephBrokerRq object
+ @param rid: Relation ID
+ """
+ broker_key = get_broker_rsp_key()
+ for unit in related_units(rid):
+ rdata = relation_get(rid=rid, unit=unit)
+ if rdata.get(broker_key):
+ rsp = CephBrokerRsp(rdata.get(broker_key))
+ if rsp.request_id == request.request_id:
+ if not rsp.exit_code:
+ return True
+ else:
+ # The remote unit sent no reply targeted at this unit so either the
+ # remote ceph cluster does not support unit targeted replies or it
+ # has not processed our request yet.
+ if rdata.get('broker_rsp'):
+ request_data = json.loads(rdata['broker_rsp'])
+ if request_data.get('request-id'):
+ log('Ignoring legacy broker_rsp without unit key as remote '
+ 'service supports unit specific replies', level=DEBUG)
+ else:
+ log('Using legacy broker_rsp as remote service does not '
+ 'supports unit specific replies', level=DEBUG)
+ rsp = CephBrokerRsp(rdata['broker_rsp'])
+ if not rsp.exit_code:
+ return True
+
+ return False
+
+
+def get_broker_rsp_key():
+ """Return broker response key for this unit
+
+ This is the key that ceph is going to use to pass request status
+ information back to this unit
+ """
+ return 'broker-rsp-' + local_unit().replace('/', '-')
+
+
+def send_request_if_needed(request, relation='ceph'):
+ """Send broker request if an equivalent request has not already been sent
+
+ @param request: A CephBrokerRq object
+ """
+ if is_request_sent(request, relation=relation):
+ log('Request already sent but not complete, not sending new request',
+ level=DEBUG)
+ else:
+ for rid in relation_ids(relation):
+ log('Sending request {}'.format(request.request_id), level=DEBUG)
+ relation_set(relation_id=rid, broker_req=request.request)
+
+
+def is_broker_action_done(action, rid=None, unit=None):
+ """Check whether broker action has completed yet.
+
+ @param action: name of action to be performed
+ @returns True if action complete otherwise False
+ """
+ rdata = relation_get(rid, unit) or {}
+ broker_rsp = rdata.get(get_broker_rsp_key())
+ if not broker_rsp:
+ return False
+
+ rsp = CephBrokerRsp(broker_rsp)
+ unit_name = local_unit().partition('/')[2]
+ key = "unit_{}_ceph_broker_action.{}".format(unit_name, action)
+ kvstore = kv()
+ val = kvstore.get(key=key)
+ if val and val == rsp.request_id:
+ return True
+
+ return False
+
+
+def mark_broker_action_done(action, rid=None, unit=None):
+ """Mark action as having been completed.
+
+ @param action: name of action to be performed
+ @returns None
+ """
+ rdata = relation_get(rid, unit) or {}
+ broker_rsp = rdata.get(get_broker_rsp_key())
+ if not broker_rsp:
+ return
+
+ rsp = CephBrokerRsp(broker_rsp)
+ unit_name = local_unit().partition('/')[2]
+ key = "unit_{}_ceph_broker_action.{}".format(unit_name, action)
+ kvstore = kv()
+ kvstore.set(key=key, value=rsp.request_id)
+ kvstore.flush()
+
+
+class CephConfContext(object):
+ """Ceph config (ceph.conf) context.
+
+ Supports user-provided Ceph configuration settings. Use can provide a
+ dictionary as the value for the config-flags charm option containing
+ Ceph configuration settings keyede by their section in ceph.conf.
+ """
+ def __init__(self, permitted_sections=None):
+ self.permitted_sections = permitted_sections or []
+
+ def __call__(self):
+ conf = config('config-flags')
+ if not conf:
+ return {}
+
+ conf = config_flags_parser(conf)
+ if not isinstance(conf, dict):
+ log("Provided config-flags is not a dictionary - ignoring",
+ level=WARNING)
+ return {}
+
+ permitted = self.permitted_sections
+ if permitted:
+ diff = set(conf.keys()).difference(set(permitted))
+ if diff:
+ log("Config-flags contains invalid keys '%s' - they will be "
+ "ignored" % (', '.join(diff)), level=WARNING)
+
+ ceph_conf = {}
+ for key in conf:
+ if permitted and key not in permitted:
+ log("Ignoring key '%s'" % key, level=WARNING)
+ continue
+
+ ceph_conf[key] = conf[key]
+
+ return ceph_conf
diff --git a/tests/charmhelpers/contrib/storage/linux/loopback.py b/tests/charmhelpers/contrib/storage/linux/loopback.py
new file mode 100644
index 0000000..1d6ae6f
--- /dev/null
+++ b/tests/charmhelpers/contrib/storage/linux/loopback.py
@@ -0,0 +1,86 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+from subprocess import (
+ check_call,
+ check_output,
+)
+
+import six
+
+
+##################################################
+# loopback device helpers.
+##################################################
+def loopback_devices():
+ '''
+ Parse through 'losetup -a' output to determine currently mapped
+ loopback devices. Output is expected to look like:
+
+ /dev/loop0: [0807]:961814 (/tmp/my.img)
+
+ :returns: dict: a dict mapping {loopback_dev: backing_file}
+ '''
+ loopbacks = {}
+ cmd = ['losetup', '-a']
+ devs = [d.strip().split(' ') for d in
+ check_output(cmd).splitlines() if d != '']
+ for dev, _, f in devs:
+ loopbacks[dev.replace(':', '')] = re.search('\((\S+)\)', f).groups()[0]
+ return loopbacks
+
+
+def create_loopback(file_path):
+ '''
+ Create a loopback device for a given backing file.
+
+ :returns: str: Full path to new loopback device (eg, /dev/loop0)
+ '''
+ file_path = os.path.abspath(file_path)
+ check_call(['losetup', '--find', file_path])
+ for d, f in six.iteritems(loopback_devices()):
+ if f == file_path:
+ return d
+
+
+def ensure_loopback_device(path, size):
+ '''
+ Ensure a loopback device exists for a given backing file path and size.
+ If it a loopback device is not mapped to file, a new one will be created.
+
+ TODO: Confirm size of found loopback device.
+
+ :returns: str: Full path to the ensured loopback device (eg, /dev/loop0)
+ '''
+ for d, f in six.iteritems(loopback_devices()):
+ if f == path:
+ return d
+
+ if not os.path.exists(path):
+ cmd = ['truncate', '--size', size, path]
+ check_call(cmd)
+
+ return create_loopback(path)
+
+
+def is_mapped_loopback_device(device):
+ """
+ Checks if a given device name is an existing/mapped loopback device.
+ :param device: str: Full path to the device (eg, /dev/loop1).
+ :returns: str: Path to the backing file if is a loopback device
+ empty string otherwise
+ """
+ return loopback_devices().get(device, "")
diff --git a/tests/charmhelpers/contrib/storage/linux/lvm.py b/tests/charmhelpers/contrib/storage/linux/lvm.py
new file mode 100644
index 0000000..7f2a060
--- /dev/null
+++ b/tests/charmhelpers/contrib/storage/linux/lvm.py
@@ -0,0 +1,103 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from subprocess import (
+ CalledProcessError,
+ check_call,
+ check_output,
+ Popen,
+ PIPE,
+)
+
+
+##################################################
+# LVM helpers.
+##################################################
+def deactivate_lvm_volume_group(block_device):
+ '''
+ Deactivate any volume gruop associated with an LVM physical volume.
+
+ :param block_device: str: Full path to LVM physical volume
+ '''
+ vg = list_lvm_volume_group(block_device)
+ if vg:
+ cmd = ['vgchange', '-an', vg]
+ check_call(cmd)
+
+
+def is_lvm_physical_volume(block_device):
+ '''
+ Determine whether a block device is initialized as an LVM PV.
+
+ :param block_device: str: Full path of block device to inspect.
+
+ :returns: boolean: True if block device is a PV, False if not.
+ '''
+ try:
+ check_output(['pvdisplay', block_device])
+ return True
+ except CalledProcessError:
+ return False
+
+
+def remove_lvm_physical_volume(block_device):
+ '''
+ Remove LVM PV signatures from a given block device.
+
+ :param block_device: str: Full path of block device to scrub.
+ '''
+ p = Popen(['pvremove', '-ff', block_device],
+ stdin=PIPE)
+ p.communicate(input='y\n')
+
+
+def list_lvm_volume_group(block_device):
+ '''
+ List LVM volume group associated with a given block device.
+
+ Assumes block device is a valid LVM PV.
+
+ :param block_device: str: Full path of block device to inspect.
+
+ :returns: str: Name of volume group associated with block device or None
+ '''
+ vg = None
+ pvd = check_output(['pvdisplay', block_device]).splitlines()
+ for lvm in pvd:
+ lvm = lvm.decode('UTF-8')
+ if lvm.strip().startswith('VG Name'):
+ vg = ' '.join(lvm.strip().split()[2:])
+ return vg
+
+
+def create_lvm_physical_volume(block_device):
+ '''
+ Initialize a block device as an LVM physical volume.
+
+ :param block_device: str: Full path of block device to initialize.
+
+ '''
+ check_call(['pvcreate', block_device])
+
+
+def create_lvm_volume_group(volume_group, block_device):
+ '''
+ Create an LVM volume group backed by a given block device.
+
+ Assumes block device has already been initialized as an LVM PV.
+
+ :param volume_group: str: Name of volume group to create.
+ :block_device: str: Full path of PV-initialized block device.
+ '''
+ check_call(['vgcreate', volume_group, block_device])
diff --git a/tests/charmhelpers/contrib/storage/linux/utils.py b/tests/charmhelpers/contrib/storage/linux/utils.py
new file mode 100644
index 0000000..c942889
--- /dev/null
+++ b/tests/charmhelpers/contrib/storage/linux/utils.py
@@ -0,0 +1,69 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+from stat import S_ISBLK
+
+from subprocess import (
+ check_call,
+ check_output,
+ call
+)
+
+
+def is_block_device(path):
+ '''
+ Confirm device at path is a valid block device node.
+
+ :returns: boolean: True if path is a block device, False if not.
+ '''
+ if not os.path.exists(path):
+ return False
+ return S_ISBLK(os.stat(path).st_mode)
+
+
+def zap_disk(block_device):
+ '''
+ Clear a block device of partition table. Relies on sgdisk, which is
+ installed as pat of the 'gdisk' package in Ubuntu.
+
+ :param block_device: str: Full path of block device to clean.
+ '''
+ # https://github.com/ceph/ceph/commit/fdd7f8d83afa25c4e09aaedd90ab93f3b64a677b
+ # sometimes sgdisk exits non-zero; this is OK, dd will clean up
+ call(['sgdisk', '--zap-all', '--', block_device])
+ call(['sgdisk', '--clear', '--mbrtogpt', '--', block_device])
+ dev_end = check_output(['blockdev', '--getsz',
+ block_device]).decode('UTF-8')
+ gpt_end = int(dev_end.split()[0]) - 100
+ check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device),
+ 'bs=1M', 'count=1'])
+ check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device),
+ 'bs=512', 'count=100', 'seek=%s' % (gpt_end)])
+
+
+def is_device_mounted(device):
+ '''Given a device path, return True if that device is mounted, and False
+ if it isn't.
+
+ :param device: str: Full path of the device to check.
+ :returns: boolean: True if the path represents a mounted device, False if
+ it doesn't.
+ '''
+ try:
+ out = check_output(['lsblk', '-P', device]).decode('UTF-8')
+ except Exception:
+ return False
+ return bool(re.search(r'MOUNTPOINT=".+"', out))
diff --git a/tests/charmhelpers/core/__init__.py b/tests/charmhelpers/core/__init__.py
new file mode 100644
index 0000000..d7567b8
--- /dev/null
+++ b/tests/charmhelpers/core/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/tests/charmhelpers/core/decorators.py b/tests/charmhelpers/core/decorators.py
new file mode 100644
index 0000000..6ad41ee
--- /dev/null
+++ b/tests/charmhelpers/core/decorators.py
@@ -0,0 +1,55 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Copyright 2014 Canonical Ltd.
+#
+# Authors:
+# Edward Hope-Morley
+#
+
+import time
+
+from charmhelpers.core.hookenv import (
+ log,
+ INFO,
+)
+
+
+def retry_on_exception(num_retries, base_delay=0, exc_type=Exception):
+ """If the decorated function raises exception exc_type, allow num_retries
+ retry attempts before raise the exception.
+ """
+ def _retry_on_exception_inner_1(f):
+ def _retry_on_exception_inner_2(*args, **kwargs):
+ retries = num_retries
+ multiplier = 1
+ while True:
+ try:
+ return f(*args, **kwargs)
+ except exc_type:
+ if not retries:
+ raise
+
+ delay = base_delay * multiplier
+ multiplier += 1
+ log("Retrying '%s' %d more times (delay=%s)" %
+ (f.__name__, retries, delay), level=INFO)
+ retries -= 1
+ if delay:
+ time.sleep(delay)
+
+ return _retry_on_exception_inner_2
+
+ return _retry_on_exception_inner_1
diff --git a/tests/charmhelpers/core/files.py b/tests/charmhelpers/core/files.py
new file mode 100644
index 0000000..fdd82b7
--- /dev/null
+++ b/tests/charmhelpers/core/files.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = 'Jorge Niedbalski '
+
+import os
+import subprocess
+
+
+def sed(filename, before, after, flags='g'):
+ """
+ Search and replaces the given pattern on filename.
+
+ :param filename: relative or absolute file path.
+ :param before: expression to be replaced (see 'man sed')
+ :param after: expression to replace with (see 'man sed')
+ :param flags: sed-compatible regex flags in example, to make
+ the search and replace case insensitive, specify ``flags="i"``.
+ The ``g`` flag is always specified regardless, so you do not
+ need to remember to include it when overriding this parameter.
+ :returns: If the sed command exit code was zero then return,
+ otherwise raise CalledProcessError.
+ """
+ expression = r's/{0}/{1}/{2}'.format(before,
+ after, flags)
+
+ return subprocess.check_call(["sed", "-i", "-r", "-e",
+ expression,
+ os.path.expanduser(filename)])
diff --git a/tests/charmhelpers/core/fstab.py b/tests/charmhelpers/core/fstab.py
new file mode 100644
index 0000000..d9fa915
--- /dev/null
+++ b/tests/charmhelpers/core/fstab.py
@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import io
+import os
+
+__author__ = 'Jorge Niedbalski R. '
+
+
+class Fstab(io.FileIO):
+ """This class extends file in order to implement a file reader/writer
+ for file `/etc/fstab`
+ """
+
+ class Entry(object):
+ """Entry class represents a non-comment line on the `/etc/fstab` file
+ """
+ def __init__(self, device, mountpoint, filesystem,
+ options, d=0, p=0):
+ self.device = device
+ self.mountpoint = mountpoint
+ self.filesystem = filesystem
+
+ if not options:
+ options = "defaults"
+
+ self.options = options
+ self.d = int(d)
+ self.p = int(p)
+
+ def __eq__(self, o):
+ return str(self) == str(o)
+
+ def __str__(self):
+ return "{} {} {} {} {} {}".format(self.device,
+ self.mountpoint,
+ self.filesystem,
+ self.options,
+ self.d,
+ self.p)
+
+ DEFAULT_PATH = os.path.join(os.path.sep, 'etc', 'fstab')
+
+ def __init__(self, path=None):
+ if path:
+ self._path = path
+ else:
+ self._path = self.DEFAULT_PATH
+ super(Fstab, self).__init__(self._path, 'rb+')
+
+ def _hydrate_entry(self, line):
+ # NOTE: use split with no arguments to split on any
+ # whitespace including tabs
+ return Fstab.Entry(*filter(
+ lambda x: x not in ('', None),
+ line.strip("\n").split()))
+
+ @property
+ def entries(self):
+ self.seek(0)
+ for line in self.readlines():
+ line = line.decode('us-ascii')
+ try:
+ if line.strip() and not line.strip().startswith("#"):
+ yield self._hydrate_entry(line)
+ except ValueError:
+ pass
+
+ def get_entry_by_attr(self, attr, value):
+ for entry in self.entries:
+ e_attr = getattr(entry, attr)
+ if e_attr == value:
+ return entry
+ return None
+
+ def add_entry(self, entry):
+ if self.get_entry_by_attr('device', entry.device):
+ return False
+
+ self.write((str(entry) + '\n').encode('us-ascii'))
+ self.truncate()
+ return entry
+
+ def remove_entry(self, entry):
+ self.seek(0)
+
+ lines = [l.decode('us-ascii') for l in self.readlines()]
+
+ found = False
+ for index, line in enumerate(lines):
+ if line.strip() and not line.strip().startswith("#"):
+ if self._hydrate_entry(line) == entry:
+ found = True
+ break
+
+ if not found:
+ return False
+
+ lines.remove(line)
+
+ self.seek(0)
+ self.write(''.join(lines).encode('us-ascii'))
+ self.truncate()
+ return True
+
+ @classmethod
+ def remove_by_mountpoint(cls, mountpoint, path=None):
+ fstab = cls(path=path)
+ entry = fstab.get_entry_by_attr('mountpoint', mountpoint)
+ if entry:
+ return fstab.remove_entry(entry)
+ return False
+
+ @classmethod
+ def add(cls, device, mountpoint, filesystem, options=None, path=None):
+ return cls(path=path).add_entry(Fstab.Entry(device,
+ mountpoint, filesystem,
+ options=options))
diff --git a/tests/charmhelpers/core/hookenv.py b/tests/charmhelpers/core/hookenv.py
new file mode 100644
index 0000000..5a88f79
--- /dev/null
+++ b/tests/charmhelpers/core/hookenv.py
@@ -0,0 +1,1206 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"Interactions with the Juju environment"
+# Copyright 2013 Canonical Ltd.
+#
+# Authors:
+# Charm Helpers Developers
+
+from __future__ import print_function
+import copy
+from distutils.version import LooseVersion
+from functools import wraps
+from collections import namedtuple
+import glob
+import os
+import json
+import yaml
+import subprocess
+import sys
+import errno
+import tempfile
+from subprocess import CalledProcessError
+
+import six
+if not six.PY3:
+ from UserDict import UserDict
+else:
+ from collections import UserDict
+
+CRITICAL = "CRITICAL"
+ERROR = "ERROR"
+WARNING = "WARNING"
+INFO = "INFO"
+DEBUG = "DEBUG"
+TRACE = "TRACE"
+MARKER = object()
+
+cache = {}
+
+
+def cached(func):
+ """Cache return values for multiple executions of func + args
+
+ For example::
+
+ @cached
+ def unit_get(attribute):
+ pass
+
+ unit_get('test')
+
+ will cache the result of unit_get + 'test' for future calls.
+ """
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ global cache
+ key = str((func, args, kwargs))
+ try:
+ return cache[key]
+ except KeyError:
+ pass # Drop out of the exception handler scope.
+ res = func(*args, **kwargs)
+ cache[key] = res
+ return res
+ wrapper._wrapped = func
+ return wrapper
+
+
+def flush(key):
+ """Flushes any entries from function cache where the
+ key is found in the function+args """
+ flush_list = []
+ for item in cache:
+ if key in item:
+ flush_list.append(item)
+ for item in flush_list:
+ del cache[item]
+
+
+def log(message, level=None):
+ """Write a message to the juju log"""
+ command = ['juju-log']
+ if level:
+ command += ['-l', level]
+ if not isinstance(message, six.string_types):
+ message = repr(message)
+ command += [message]
+ # Missing juju-log should not cause failures in unit tests
+ # Send log output to stderr
+ try:
+ subprocess.call(command)
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ if level:
+ message = "{}: {}".format(level, message)
+ message = "juju-log: {}".format(message)
+ print(message, file=sys.stderr)
+ else:
+ raise
+
+
+class Serializable(UserDict):
+ """Wrapper, an object that can be serialized to yaml or json"""
+
+ def __init__(self, obj):
+ # wrap the object
+ UserDict.__init__(self)
+ self.data = obj
+
+ def __getattr__(self, attr):
+ # See if this object has attribute.
+ if attr in ("json", "yaml", "data"):
+ return self.__dict__[attr]
+ # Check for attribute in wrapped object.
+ got = getattr(self.data, attr, MARKER)
+ if got is not MARKER:
+ return got
+ # Proxy to the wrapped object via dict interface.
+ try:
+ return self.data[attr]
+ except KeyError:
+ raise AttributeError(attr)
+
+ def __getstate__(self):
+ # Pickle as a standard dictionary.
+ return self.data
+
+ def __setstate__(self, state):
+ # Unpickle into our wrapper.
+ self.data = state
+
+ def json(self):
+ """Serialize the object to json"""
+ return json.dumps(self.data)
+
+ def yaml(self):
+ """Serialize the object to yaml"""
+ return yaml.dump(self.data)
+
+
+def execution_environment():
+ """A convenient bundling of the current execution context"""
+ context = {}
+ context['conf'] = config()
+ if relation_id():
+ context['reltype'] = relation_type()
+ context['relid'] = relation_id()
+ context['rel'] = relation_get()
+ context['unit'] = local_unit()
+ context['rels'] = relations()
+ context['env'] = os.environ
+ return context
+
+
+def in_relation_hook():
+ """Determine whether we're running in a relation hook"""
+ return 'JUJU_RELATION' in os.environ
+
+
+def relation_type():
+ """The scope for the current relation hook"""
+ return os.environ.get('JUJU_RELATION', None)
+
+
+@cached
+def relation_id(relation_name=None, service_or_unit=None):
+ """The relation ID for the current or a specified relation"""
+ if not relation_name and not service_or_unit:
+ return os.environ.get('JUJU_RELATION_ID', None)
+ elif relation_name and service_or_unit:
+ service_name = service_or_unit.split('/')[0]
+ for relid in relation_ids(relation_name):
+ remote_service = remote_service_name(relid)
+ if remote_service == service_name:
+ return relid
+ else:
+ raise ValueError('Must specify neither or both of relation_name and service_or_unit')
+
+
+def local_unit():
+ """Local unit ID"""
+ return os.environ['JUJU_UNIT_NAME']
+
+
+def remote_unit():
+ """The remote unit for the current relation hook"""
+ return os.environ.get('JUJU_REMOTE_UNIT', None)
+
+
+def service_name():
+ """The name service group this unit belongs to"""
+ return local_unit().split('/')[0]
+
+
+def principal_unit():
+ """Returns the principal unit of this unit, otherwise None"""
+ # Juju 2.2 and above provides JUJU_PRINCIPAL_UNIT
+ principal_unit = os.environ.get('JUJU_PRINCIPAL_UNIT', None)
+ # If it's empty, then this unit is the principal
+ if principal_unit == '':
+ return os.environ['JUJU_UNIT_NAME']
+ elif principal_unit is not None:
+ return principal_unit
+ # For Juju 2.1 and below, let's try work out the principle unit by
+ # the various charms' metadata.yaml.
+ for reltype in relation_types():
+ for rid in relation_ids(reltype):
+ for unit in related_units(rid):
+ md = _metadata_unit(unit)
+ if not md:
+ continue
+ subordinate = md.pop('subordinate', None)
+ if not subordinate:
+ return unit
+ return None
+
+
+@cached
+def remote_service_name(relid=None):
+ """The remote service name for a given relation-id (or the current relation)"""
+ if relid is None:
+ unit = remote_unit()
+ else:
+ units = related_units(relid)
+ unit = units[0] if units else None
+ return unit.split('/')[0] if unit else None
+
+
+def hook_name():
+ """The name of the currently executing hook"""
+ return os.environ.get('JUJU_HOOK_NAME', os.path.basename(sys.argv[0]))
+
+
+class Config(dict):
+ """A dictionary representation of the charm's config.yaml, with some
+ extra features:
+
+ - See which values in the dictionary have changed since the previous hook.
+ - For values that have changed, see what the previous value was.
+ - Store arbitrary data for use in a later hook.
+
+ NOTE: Do not instantiate this object directly - instead call
+ ``hookenv.config()``, which will return an instance of :class:`Config`.
+
+ Example usage::
+
+ >>> # inside a hook
+ >>> from charmhelpers.core import hookenv
+ >>> config = hookenv.config()
+ >>> config['foo']
+ 'bar'
+ >>> # store a new key/value for later use
+ >>> config['mykey'] = 'myval'
+
+
+ >>> # user runs `juju set mycharm foo=baz`
+ >>> # now we're inside subsequent config-changed hook
+ >>> config = hookenv.config()
+ >>> config['foo']
+ 'baz'
+ >>> # test to see if this val has changed since last hook
+ >>> config.changed('foo')
+ True
+ >>> # what was the previous value?
+ >>> config.previous('foo')
+ 'bar'
+ >>> # keys/values that we add are preserved across hooks
+ >>> config['mykey']
+ 'myval'
+
+ """
+ CONFIG_FILE_NAME = '.juju-persistent-config'
+
+ def __init__(self, *args, **kw):
+ super(Config, self).__init__(*args, **kw)
+ self.implicit_save = True
+ self._prev_dict = None
+ self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
+ if os.path.exists(self.path):
+ self.load_previous()
+ atexit(self._implicit_save)
+
+ def load_previous(self, path=None):
+ """Load previous copy of config from disk.
+
+ In normal usage you don't need to call this method directly - it
+ is called automatically at object initialization.
+
+ :param path:
+
+ File path from which to load the previous config. If `None`,
+ config is loaded from the default location. If `path` is
+ specified, subsequent `save()` calls will write to the same
+ path.
+
+ """
+ self.path = path or self.path
+ with open(self.path) as f:
+ self._prev_dict = json.load(f)
+ for k, v in copy.deepcopy(self._prev_dict).items():
+ if k not in self:
+ self[k] = v
+
+ def changed(self, key):
+ """Return True if the current value for this key is different from
+ the previous value.
+
+ """
+ if self._prev_dict is None:
+ return True
+ return self.previous(key) != self.get(key)
+
+ def previous(self, key):
+ """Return previous value for this key, or None if there
+ is no previous value.
+
+ """
+ if self._prev_dict:
+ return self._prev_dict.get(key)
+ return None
+
+ def save(self):
+ """Save this config to disk.
+
+ If the charm is using the :mod:`Services Framework `
+ or :meth:'@hook ' decorator, this
+ is called automatically at the end of successful hook execution.
+ Otherwise, it should be called directly by user code.
+
+ To disable automatic saves, set ``implicit_save=False`` on this
+ instance.
+
+ """
+ with open(self.path, 'w') as f:
+ json.dump(self, f)
+
+ def _implicit_save(self):
+ if self.implicit_save:
+ self.save()
+
+
+@cached
+def config(scope=None):
+ """Juju charm configuration"""
+ config_cmd_line = ['config-get']
+ if scope is not None:
+ config_cmd_line.append(scope)
+ else:
+ config_cmd_line.append('--all')
+ config_cmd_line.append('--format=json')
+ try:
+ config_data = json.loads(
+ subprocess.check_output(config_cmd_line).decode('UTF-8'))
+ if scope is not None:
+ return config_data
+ return Config(config_data)
+ except ValueError:
+ return None
+
+
+@cached
+def relation_get(attribute=None, unit=None, rid=None):
+ """Get relation information"""
+ _args = ['relation-get', '--format=json']
+ if rid:
+ _args.append('-r')
+ _args.append(rid)
+ _args.append(attribute or '-')
+ if unit:
+ _args.append(unit)
+ try:
+ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
+ except ValueError:
+ return None
+ except CalledProcessError as e:
+ if e.returncode == 2:
+ return None
+ raise
+
+
+def relation_set(relation_id=None, relation_settings=None, **kwargs):
+ """Set relation information for the current unit"""
+ relation_settings = relation_settings if relation_settings else {}
+ relation_cmd_line = ['relation-set']
+ accepts_file = "--file" in subprocess.check_output(
+ relation_cmd_line + ["--help"], universal_newlines=True)
+ if relation_id is not None:
+ relation_cmd_line.extend(('-r', relation_id))
+ settings = relation_settings.copy()
+ settings.update(kwargs)
+ for key, value in settings.items():
+ # Force value to be a string: it always should, but some call
+ # sites pass in things like dicts or numbers.
+ if value is not None:
+ settings[key] = "{}".format(value)
+ if accepts_file:
+ # --file was introduced in Juju 1.23.2. Use it by default if
+ # available, since otherwise we'll break if the relation data is
+ # too big. Ideally we should tell relation-set to read the data from
+ # stdin, but that feature is broken in 1.23.2: Bug #1454678.
+ with tempfile.NamedTemporaryFile(delete=False) as settings_file:
+ settings_file.write(yaml.safe_dump(settings).encode("utf-8"))
+ subprocess.check_call(
+ relation_cmd_line + ["--file", settings_file.name])
+ os.remove(settings_file.name)
+ else:
+ for key, value in settings.items():
+ if value is None:
+ relation_cmd_line.append('{}='.format(key))
+ else:
+ relation_cmd_line.append('{}={}'.format(key, value))
+ subprocess.check_call(relation_cmd_line)
+ # Flush cache of any relation-gets for local unit
+ flush(local_unit())
+
+
+def relation_clear(r_id=None):
+ ''' Clears any relation data already set on relation r_id '''
+ settings = relation_get(rid=r_id,
+ unit=local_unit())
+ for setting in settings:
+ if setting not in ['public-address', 'private-address']:
+ settings[setting] = None
+ relation_set(relation_id=r_id,
+ **settings)
+
+
+@cached
+def relation_ids(reltype=None):
+ """A list of relation_ids"""
+ reltype = reltype or relation_type()
+ relid_cmd_line = ['relation-ids', '--format=json']
+ if reltype is not None:
+ relid_cmd_line.append(reltype)
+ return json.loads(
+ subprocess.check_output(relid_cmd_line).decode('UTF-8')) or []
+ return []
+
+
+@cached
+def related_units(relid=None):
+ """A list of related units"""
+ relid = relid or relation_id()
+ units_cmd_line = ['relation-list', '--format=json']
+ if relid is not None:
+ units_cmd_line.extend(('-r', relid))
+ return json.loads(
+ subprocess.check_output(units_cmd_line).decode('UTF-8')) or []
+
+
+@cached
+def relation_for_unit(unit=None, rid=None):
+ """Get the json represenation of a unit's relation"""
+ unit = unit or remote_unit()
+ relation = relation_get(unit=unit, rid=rid)
+ for key in relation:
+ if key.endswith('-list'):
+ relation[key] = relation[key].split()
+ relation['__unit__'] = unit
+ return relation
+
+
+@cached
+def relations_for_id(relid=None):
+ """Get relations of a specific relation ID"""
+ relation_data = []
+ relid = relid or relation_ids()
+ for unit in related_units(relid):
+ unit_data = relation_for_unit(unit, relid)
+ unit_data['__relid__'] = relid
+ relation_data.append(unit_data)
+ return relation_data
+
+
+@cached
+def relations_of_type(reltype=None):
+ """Get relations of a specific type"""
+ relation_data = []
+ reltype = reltype or relation_type()
+ for relid in relation_ids(reltype):
+ for relation in relations_for_id(relid):
+ relation['__relid__'] = relid
+ relation_data.append(relation)
+ return relation_data
+
+
+@cached
+def metadata():
+ """Get the current charm metadata.yaml contents as a python object"""
+ with open(os.path.join(charm_dir(), 'metadata.yaml')) as md:
+ return yaml.safe_load(md)
+
+
+def _metadata_unit(unit):
+ """Given the name of a unit (e.g. apache2/0), get the unit charm's
+ metadata.yaml. Very similar to metadata() but allows us to inspect
+ other units. Unit needs to be co-located, such as a subordinate or
+ principal/primary.
+
+ :returns: metadata.yaml as a python object.
+
+ """
+ basedir = os.sep.join(charm_dir().split(os.sep)[:-2])
+ unitdir = 'unit-{}'.format(unit.replace(os.sep, '-'))
+ joineddir = os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')
+ if not os.path.exists(joineddir):
+ return None
+ with open(joineddir) as md:
+ return yaml.safe_load(md)
+
+
+@cached
+def relation_types():
+ """Get a list of relation types supported by this charm"""
+ rel_types = []
+ md = metadata()
+ for key in ('provides', 'requires', 'peers'):
+ section = md.get(key)
+ if section:
+ rel_types.extend(section.keys())
+ return rel_types
+
+
+@cached
+def peer_relation_id():
+ '''Get the peers relation id if a peers relation has been joined, else None.'''
+ md = metadata()
+ section = md.get('peers')
+ if section:
+ for key in section:
+ relids = relation_ids(key)
+ if relids:
+ return relids[0]
+ return None
+
+
+@cached
+def relation_to_interface(relation_name):
+ """
+ Given the name of a relation, return the interface that relation uses.
+
+ :returns: The interface name, or ``None``.
+ """
+ return relation_to_role_and_interface(relation_name)[1]
+
+
+@cached
+def relation_to_role_and_interface(relation_name):
+ """
+ Given the name of a relation, return the role and the name of the interface
+ that relation uses (where role is one of ``provides``, ``requires``, or ``peers``).
+
+ :returns: A tuple containing ``(role, interface)``, or ``(None, None)``.
+ """
+ _metadata = metadata()
+ for role in ('provides', 'requires', 'peers'):
+ interface = _metadata.get(role, {}).get(relation_name, {}).get('interface')
+ if interface:
+ return role, interface
+ return None, None
+
+
+@cached
+def role_and_interface_to_relations(role, interface_name):
+ """
+ Given a role and interface name, return a list of relation names for the
+ current charm that use that interface under that role (where role is one
+ of ``provides``, ``requires``, or ``peers``).
+
+ :returns: A list of relation names.
+ """
+ _metadata = metadata()
+ results = []
+ for relation_name, relation in _metadata.get(role, {}).items():
+ if relation['interface'] == interface_name:
+ results.append(relation_name)
+ return results
+
+
+@cached
+def interface_to_relations(interface_name):
+ """
+ Given an interface, return a list of relation names for the current
+ charm that use that interface.
+
+ :returns: A list of relation names.
+ """
+ results = []
+ for role in ('provides', 'requires', 'peers'):
+ results.extend(role_and_interface_to_relations(role, interface_name))
+ return results
+
+
+@cached
+def charm_name():
+ """Get the name of the current charm as is specified on metadata.yaml"""
+ return metadata().get('name')
+
+
+@cached
+def relations():
+ """Get a nested dictionary of relation data for all related units"""
+ rels = {}
+ for reltype in relation_types():
+ relids = {}
+ for relid in relation_ids(reltype):
+ units = {local_unit(): relation_get(unit=local_unit(), rid=relid)}
+ for unit in related_units(relid):
+ reldata = relation_get(unit=unit, rid=relid)
+ units[unit] = reldata
+ relids[relid] = units
+ rels[reltype] = relids
+ return rels
+
+
+@cached
+def is_relation_made(relation, keys='private-address'):
+ '''
+ Determine whether a relation is established by checking for
+ presence of key(s). If a list of keys is provided, they
+ must all be present for the relation to be identified as made
+ '''
+ if isinstance(keys, str):
+ keys = [keys]
+ for r_id in relation_ids(relation):
+ for unit in related_units(r_id):
+ context = {}
+ for k in keys:
+ context[k] = relation_get(k, rid=r_id,
+ unit=unit)
+ if None not in context.values():
+ return True
+ return False
+
+
+def _port_op(op_name, port, protocol="TCP"):
+ """Open or close a service network port"""
+ _args = [op_name]
+ icmp = protocol.upper() == "ICMP"
+ if icmp:
+ _args.append(protocol)
+ else:
+ _args.append('{}/{}'.format(port, protocol))
+ try:
+ subprocess.check_call(_args)
+ except subprocess.CalledProcessError:
+ # Older Juju pre 2.3 doesn't support ICMP
+ # so treat it as a no-op if it fails.
+ if not icmp:
+ raise
+
+
+def open_port(port, protocol="TCP"):
+ """Open a service network port"""
+ _port_op('open-port', port, protocol)
+
+
+def close_port(port, protocol="TCP"):
+ """Close a service network port"""
+ _port_op('close-port', port, protocol)
+
+
+def open_ports(start, end, protocol="TCP"):
+ """Opens a range of service network ports"""
+ _args = ['open-port']
+ _args.append('{}-{}/{}'.format(start, end, protocol))
+ subprocess.check_call(_args)
+
+
+def close_ports(start, end, protocol="TCP"):
+ """Close a range of service network ports"""
+ _args = ['close-port']
+ _args.append('{}-{}/{}'.format(start, end, protocol))
+ subprocess.check_call(_args)
+
+
+def opened_ports():
+ """Get the opened ports
+
+ *Note that this will only show ports opened in a previous hook*
+
+ :returns: Opened ports as a list of strings: ``['8080/tcp', '8081-8083/tcp']``
+ """
+ _args = ['opened-ports', '--format=json']
+ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
+
+
+@cached
+def unit_get(attribute):
+ """Get the unit ID for the remote unit"""
+ _args = ['unit-get', '--format=json', attribute]
+ try:
+ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
+ except ValueError:
+ return None
+
+
+def unit_public_ip():
+ """Get this unit's public IP address"""
+ return unit_get('public-address')
+
+
+def unit_private_ip():
+ """Get this unit's private IP address"""
+ return unit_get('private-address')
+
+
+@cached
+def storage_get(attribute=None, storage_id=None):
+ """Get storage attributes"""
+ _args = ['storage-get', '--format=json']
+ if storage_id:
+ _args.extend(('-s', storage_id))
+ if attribute:
+ _args.append(attribute)
+ try:
+ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
+ except ValueError:
+ return None
+
+
+@cached
+def storage_list(storage_name=None):
+ """List the storage IDs for the unit"""
+ _args = ['storage-list', '--format=json']
+ if storage_name:
+ _args.append(storage_name)
+ try:
+ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
+ except ValueError:
+ return None
+ except OSError as e:
+ import errno
+ if e.errno == errno.ENOENT:
+ # storage-list does not exist
+ return []
+ raise
+
+
+class UnregisteredHookError(Exception):
+ """Raised when an undefined hook is called"""
+ pass
+
+
+class Hooks(object):
+ """A convenient handler for hook functions.
+
+ Example::
+
+ hooks = Hooks()
+
+ # register a hook, taking its name from the function name
+ @hooks.hook()
+ def install():
+ pass # your code here
+
+ # register a hook, providing a custom hook name
+ @hooks.hook("config-changed")
+ def config_changed():
+ pass # your code here
+
+ if __name__ == "__main__":
+ # execute a hook based on the name the program is called by
+ hooks.execute(sys.argv)
+ """
+
+ def __init__(self, config_save=None):
+ super(Hooks, self).__init__()
+ self._hooks = {}
+
+ # For unknown reasons, we allow the Hooks constructor to override
+ # config().implicit_save.
+ if config_save is not None:
+ config().implicit_save = config_save
+
+ def register(self, name, function):
+ """Register a hook"""
+ self._hooks[name] = function
+
+ def execute(self, args):
+ """Execute a registered hook based on args[0]"""
+ _run_atstart()
+ hook_name = os.path.basename(args[0])
+ if hook_name in self._hooks:
+ try:
+ self._hooks[hook_name]()
+ except SystemExit as x:
+ if x.code is None or x.code == 0:
+ _run_atexit()
+ raise
+ _run_atexit()
+ else:
+ raise UnregisteredHookError(hook_name)
+
+ def hook(self, *hook_names):
+ """Decorator, registering them as hooks"""
+ def wrapper(decorated):
+ for hook_name in hook_names:
+ self.register(hook_name, decorated)
+ else:
+ self.register(decorated.__name__, decorated)
+ if '_' in decorated.__name__:
+ self.register(
+ decorated.__name__.replace('_', '-'), decorated)
+ return decorated
+ return wrapper
+
+
+def charm_dir():
+ """Return the root directory of the current charm"""
+ d = os.environ.get('JUJU_CHARM_DIR')
+ if d is not None:
+ return d
+ return os.environ.get('CHARM_DIR')
+
+
+@cached
+def action_get(key=None):
+ """Gets the value of an action parameter, or all key/value param pairs"""
+ cmd = ['action-get']
+ if key is not None:
+ cmd.append(key)
+ cmd.append('--format=json')
+ action_data = json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+ return action_data
+
+
+def action_set(values):
+ """Sets the values to be returned after the action finishes"""
+ cmd = ['action-set']
+ for k, v in list(values.items()):
+ cmd.append('{}={}'.format(k, v))
+ subprocess.check_call(cmd)
+
+
+def action_fail(message):
+ """Sets the action status to failed and sets the error message.
+
+ The results set by action_set are preserved."""
+ subprocess.check_call(['action-fail', message])
+
+
+def action_name():
+ """Get the name of the currently executing action."""
+ return os.environ.get('JUJU_ACTION_NAME')
+
+
+def action_uuid():
+ """Get the UUID of the currently executing action."""
+ return os.environ.get('JUJU_ACTION_UUID')
+
+
+def action_tag():
+ """Get the tag for the currently executing action."""
+ return os.environ.get('JUJU_ACTION_TAG')
+
+
+def status_set(workload_state, message):
+ """Set the workload state with a message
+
+ Use status-set to set the workload state with a message which is visible
+ to the user via juju status. If the status-set command is not found then
+ assume this is juju < 1.23 and juju-log the message unstead.
+
+ workload_state -- valid juju workload state.
+ message -- status update message
+ """
+ valid_states = ['maintenance', 'blocked', 'waiting', 'active']
+ if workload_state not in valid_states:
+ raise ValueError(
+ '{!r} is not a valid workload state'.format(workload_state)
+ )
+ cmd = ['status-set', workload_state, message]
+ try:
+ ret = subprocess.call(cmd)
+ if ret == 0:
+ return
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ log_message = 'status-set failed: {} {}'.format(workload_state,
+ message)
+ log(log_message, level='INFO')
+
+
+def status_get():
+ """Retrieve the previously set juju workload state and message
+
+ If the status-get command is not found then assume this is juju < 1.23 and
+ return 'unknown', ""
+
+ """
+ cmd = ['status-get', "--format=json", "--include-data"]
+ try:
+ raw_status = subprocess.check_output(cmd)
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ return ('unknown', "")
+ else:
+ raise
+ else:
+ status = json.loads(raw_status.decode("UTF-8"))
+ return (status["status"], status["message"])
+
+
+def translate_exc(from_exc, to_exc):
+ def inner_translate_exc1(f):
+ @wraps(f)
+ def inner_translate_exc2(*args, **kwargs):
+ try:
+ return f(*args, **kwargs)
+ except from_exc:
+ raise to_exc
+
+ return inner_translate_exc2
+
+ return inner_translate_exc1
+
+
+def application_version_set(version):
+ """Charm authors may trigger this command from any hook to output what
+ version of the application is running. This could be a package version,
+ for instance postgres version 9.5. It could also be a build number or
+ version control revision identifier, for instance git sha 6fb7ba68. """
+
+ cmd = ['application-version-set']
+ cmd.append(version)
+ try:
+ subprocess.check_call(cmd)
+ except OSError:
+ log("Application Version: {}".format(version))
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def is_leader():
+ """Does the current unit hold the juju leadership
+
+ Uses juju to determine whether the current unit is the leader of its peers
+ """
+ cmd = ['is-leader', '--format=json']
+ return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_get(attribute=None):
+ """Juju leader get value(s)"""
+ cmd = ['leader-get', '--format=json'] + [attribute or '-']
+ return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_set(settings=None, **kwargs):
+ """Juju leader set value(s)"""
+ # Don't log secrets.
+ # log("Juju leader-set '%s'" % (settings), level=DEBUG)
+ cmd = ['leader-set']
+ settings = settings or {}
+ settings.update(kwargs)
+ for k, v in settings.items():
+ if v is None:
+ cmd.append('{}='.format(k))
+ else:
+ cmd.append('{}={}'.format(k, v))
+ subprocess.check_call(cmd)
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def payload_register(ptype, klass, pid):
+ """ is used while a hook is running to let Juju know that a
+ payload has been started."""
+ cmd = ['payload-register']
+ for x in [ptype, klass, pid]:
+ cmd.append(x)
+ subprocess.check_call(cmd)
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def payload_unregister(klass, pid):
+ """ is used while a hook is running to let Juju know
+ that a payload has been manually stopped. The and provided
+ must match a payload that has been previously registered with juju using
+ payload-register."""
+ cmd = ['payload-unregister']
+ for x in [klass, pid]:
+ cmd.append(x)
+ subprocess.check_call(cmd)
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def payload_status_set(klass, pid, status):
+ """is used to update the current status of a registered payload.
+ The and provided must match a payload that has been previously
+ registered with juju using payload-register. The must be one of the
+ follow: starting, started, stopping, stopped"""
+ cmd = ['payload-status-set']
+ for x in [klass, pid, status]:
+ cmd.append(x)
+ subprocess.check_call(cmd)
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def resource_get(name):
+ """used to fetch the resource path of the given name.
+
+ must match a name of defined resource in metadata.yaml
+
+ returns either a path or False if resource not available
+ """
+ if not name:
+ return False
+
+ cmd = ['resource-get', name]
+ try:
+ return subprocess.check_output(cmd).decode('UTF-8')
+ except subprocess.CalledProcessError:
+ return False
+
+
+@cached
+def juju_version():
+ """Full version string (eg. '1.23.3.1-trusty-amd64')"""
+ # Per https://bugs.launchpad.net/juju-core/+bug/1455368/comments/1
+ jujud = glob.glob('/var/lib/juju/tools/machine-*/jujud')[0]
+ return subprocess.check_output([jujud, 'version'],
+ universal_newlines=True).strip()
+
+
+@cached
+def has_juju_version(minimum_version):
+ """Return True if the Juju version is at least the provided version"""
+ return LooseVersion(juju_version()) >= LooseVersion(minimum_version)
+
+
+_atexit = []
+_atstart = []
+
+
+def atstart(callback, *args, **kwargs):
+ '''Schedule a callback to run before the main hook.
+
+ Callbacks are run in the order they were added.
+
+ This is useful for modules and classes to perform initialization
+ and inject behavior. In particular:
+
+ - Run common code before all of your hooks, such as logging
+ the hook name or interesting relation data.
+ - Defer object or module initialization that requires a hook
+ context until we know there actually is a hook context,
+ making testing easier.
+ - Rather than requiring charm authors to include boilerplate to
+ invoke your helper's behavior, have it run automatically if
+ your object is instantiated or module imported.
+
+ This is not at all useful after your hook framework as been launched.
+ '''
+ global _atstart
+ _atstart.append((callback, args, kwargs))
+
+
+def atexit(callback, *args, **kwargs):
+ '''Schedule a callback to run on successful hook completion.
+
+ Callbacks are run in the reverse order that they were added.'''
+ _atexit.append((callback, args, kwargs))
+
+
+def _run_atstart():
+ '''Hook frameworks must invoke this before running the main hook body.'''
+ global _atstart
+ for callback, args, kwargs in _atstart:
+ callback(*args, **kwargs)
+ del _atstart[:]
+
+
+def _run_atexit():
+ '''Hook frameworks must invoke this after the main hook body has
+ successfully completed. Do not invoke it if the hook fails.'''
+ global _atexit
+ for callback, args, kwargs in reversed(_atexit):
+ callback(*args, **kwargs)
+ del _atexit[:]
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def network_get_primary_address(binding):
+ '''
+ Retrieve the primary network address for a named binding
+
+ :param binding: string. The name of a relation of extra-binding
+ :return: string. The primary IP address for the named binding
+ :raise: NotImplementedError if run on Juju < 2.0
+ '''
+ cmd = ['network-get', '--primary-address', binding]
+ return subprocess.check_output(cmd).decode('UTF-8').strip()
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def network_get(endpoint, relation_id=None):
+ """
+ Retrieve the network details for a relation endpoint
+
+ :param endpoint: string. The name of a relation endpoint
+ :param relation_id: int. The ID of the relation for the current context.
+ :return: dict. The loaded YAML output of the network-get query.
+ :raise: NotImplementedError if run on Juju < 2.1
+ """
+ cmd = ['network-get', endpoint, '--format', 'yaml']
+ if relation_id:
+ cmd.append('-r')
+ cmd.append(relation_id)
+ try:
+ response = subprocess.check_output(
+ cmd,
+ stderr=subprocess.STDOUT).decode('UTF-8').strip()
+ except CalledProcessError as e:
+ # Early versions of Juju 2.0.x required the --primary-address argument.
+ # We catch that condition here and raise NotImplementedError since
+ # the requested semantics are not available - the caller can then
+ # use the network_get_primary_address() method instead.
+ if '--primary-address is currently required' in e.output.decode('UTF-8'):
+ raise NotImplementedError
+ raise
+ return yaml.safe_load(response)
+
+
+def add_metric(*args, **kwargs):
+ """Add metric values. Values may be expressed with keyword arguments. For
+ metric names containing dashes, these may be expressed as one or more
+ 'key=value' positional arguments. May only be called from the collect-metrics
+ hook."""
+ _args = ['add-metric']
+ _kvpairs = []
+ _kvpairs.extend(args)
+ _kvpairs.extend(['{}={}'.format(k, v) for k, v in kwargs.items()])
+ _args.extend(sorted(_kvpairs))
+ try:
+ subprocess.check_call(_args)
+ return
+ except EnvironmentError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ log_message = 'add-metric failed: {}'.format(' '.join(_kvpairs))
+ log(log_message, level='INFO')
+
+
+def meter_status():
+ """Get the meter status, if running in the meter-status-changed hook."""
+ return os.environ.get('JUJU_METER_STATUS')
+
+
+def meter_info():
+ """Get the meter status information, if running in the meter-status-changed
+ hook."""
+ return os.environ.get('JUJU_METER_INFO')
+
+
+def iter_units_for_relation_name(relation_name):
+ """Iterate through all units in a relation
+
+ Generator that iterates through all the units in a relation and yields
+ a named tuple with rid and unit field names.
+
+ Usage:
+ data = [(u.rid, u.unit)
+ for u in iter_units_for_relation_name(relation_name)]
+
+ :param relation_name: string relation name
+ :yield: Named Tuple with rid and unit field names
+ """
+ RelatedUnit = namedtuple('RelatedUnit', 'rid, unit')
+ for rid in relation_ids(relation_name):
+ for unit in related_units(rid):
+ yield RelatedUnit(rid, unit)
+
+
+def ingress_address(rid=None, unit=None):
+ """
+ Retrieve the ingress-address from a relation when available. Otherwise,
+ return the private-address. This function is to be used on the consuming
+ side of the relation.
+
+ Usage:
+ addresses = [ingress_address(rid=u.rid, unit=u.unit)
+ for u in iter_units_for_relation_name(relation_name)]
+
+ :param rid: string relation id
+ :param unit: string unit name
+ :side effect: calls relation_get
+ :return: string IP address
+ """
+ settings = relation_get(rid=rid, unit=unit)
+ return (settings.get('ingress-address') or
+ settings.get('private-address'))
diff --git a/tests/charmhelpers/core/host.py b/tests/charmhelpers/core/host.py
new file mode 100644
index 0000000..5cc5c86
--- /dev/null
+++ b/tests/charmhelpers/core/host.py
@@ -0,0 +1,1019 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tools for working with the host system"""
+# Copyright 2012 Canonical Ltd.
+#
+# Authors:
+# Nick Moffitt
+# Matthew Wedgwood
+
+import os
+import re
+import pwd
+import glob
+import grp
+import random
+import string
+import subprocess
+import hashlib
+import functools
+import itertools
+import six
+
+from contextlib import contextmanager
+from collections import OrderedDict
+from .hookenv import log, DEBUG, local_unit
+from .fstab import Fstab
+from charmhelpers.osplatform import get_platform
+
+__platform__ = get_platform()
+if __platform__ == "ubuntu":
+ from charmhelpers.core.host_factory.ubuntu import (
+ service_available,
+ add_new_group,
+ lsb_release,
+ cmp_pkgrevno,
+ CompareHostReleases,
+ ) # flake8: noqa -- ignore F401 for this import
+elif __platform__ == "centos":
+ from charmhelpers.core.host_factory.centos import (
+ service_available,
+ add_new_group,
+ lsb_release,
+ cmp_pkgrevno,
+ CompareHostReleases,
+ ) # flake8: noqa -- ignore F401 for this import
+
+UPDATEDB_PATH = '/etc/updatedb.conf'
+
+def service_start(service_name, **kwargs):
+ """Start a system service.
+
+ The specified service name is managed via the system level init system.
+ Some init systems (e.g. upstart) require that additional arguments be
+ provided in order to directly control service instances whereas other init
+ systems allow for addressing instances of a service directly by name (e.g.
+ systemd).
+
+ The kwargs allow for the additional parameters to be passed to underlying
+ init systems for those systems which require/allow for them. For example,
+ the ceph-osd upstart script requires the id parameter to be passed along
+ in order to identify which running daemon should be reloaded. The follow-
+ ing example stops the ceph-osd service for instance id=4:
+
+ service_stop('ceph-osd', id=4)
+
+ :param service_name: the name of the service to stop
+ :param **kwargs: additional parameters to pass to the init system when
+ managing services. These will be passed as key=value
+ parameters to the init system's commandline. kwargs
+ are ignored for systemd enabled systems.
+ """
+ return service('start', service_name, **kwargs)
+
+
+def service_stop(service_name, **kwargs):
+ """Stop a system service.
+
+ The specified service name is managed via the system level init system.
+ Some init systems (e.g. upstart) require that additional arguments be
+ provided in order to directly control service instances whereas other init
+ systems allow for addressing instances of a service directly by name (e.g.
+ systemd).
+
+ The kwargs allow for the additional parameters to be passed to underlying
+ init systems for those systems which require/allow for them. For example,
+ the ceph-osd upstart script requires the id parameter to be passed along
+ in order to identify which running daemon should be reloaded. The follow-
+ ing example stops the ceph-osd service for instance id=4:
+
+ service_stop('ceph-osd', id=4)
+
+ :param service_name: the name of the service to stop
+ :param **kwargs: additional parameters to pass to the init system when
+ managing services. These will be passed as key=value
+ parameters to the init system's commandline. kwargs
+ are ignored for systemd enabled systems.
+ """
+ return service('stop', service_name, **kwargs)
+
+
+def service_restart(service_name, **kwargs):
+ """Restart a system service.
+
+ The specified service name is managed via the system level init system.
+ Some init systems (e.g. upstart) require that additional arguments be
+ provided in order to directly control service instances whereas other init
+ systems allow for addressing instances of a service directly by name (e.g.
+ systemd).
+
+ The kwargs allow for the additional parameters to be passed to underlying
+ init systems for those systems which require/allow for them. For example,
+ the ceph-osd upstart script requires the id parameter to be passed along
+ in order to identify which running daemon should be restarted. The follow-
+ ing example restarts the ceph-osd service for instance id=4:
+
+ service_restart('ceph-osd', id=4)
+
+ :param service_name: the name of the service to restart
+ :param **kwargs: additional parameters to pass to the init system when
+ managing services. These will be passed as key=value
+ parameters to the init system's commandline. kwargs
+ are ignored for init systems not allowing additional
+ parameters via the commandline (systemd).
+ """
+ return service('restart', service_name)
+
+
+def service_reload(service_name, restart_on_failure=False, **kwargs):
+ """Reload a system service, optionally falling back to restart if
+ reload fails.
+
+ The specified service name is managed via the system level init system.
+ Some init systems (e.g. upstart) require that additional arguments be
+ provided in order to directly control service instances whereas other init
+ systems allow for addressing instances of a service directly by name (e.g.
+ systemd).
+
+ The kwargs allow for the additional parameters to be passed to underlying
+ init systems for those systems which require/allow for them. For example,
+ the ceph-osd upstart script requires the id parameter to be passed along
+ in order to identify which running daemon should be reloaded. The follow-
+ ing example restarts the ceph-osd service for instance id=4:
+
+ service_reload('ceph-osd', id=4)
+
+ :param service_name: the name of the service to reload
+ :param restart_on_failure: boolean indicating whether to fallback to a
+ restart if the reload fails.
+ :param **kwargs: additional parameters to pass to the init system when
+ managing services. These will be passed as key=value
+ parameters to the init system's commandline. kwargs
+ are ignored for init systems not allowing additional
+ parameters via the commandline (systemd).
+ """
+ service_result = service('reload', service_name, **kwargs)
+ if not service_result and restart_on_failure:
+ service_result = service('restart', service_name, **kwargs)
+ return service_result
+
+
+def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d",
+ **kwargs):
+ """Pause a system service.
+
+ Stop it, and prevent it from starting again at boot.
+
+ :param service_name: the name of the service to pause
+ :param init_dir: path to the upstart init directory
+ :param initd_dir: path to the sysv init directory
+ :param **kwargs: additional parameters to pass to the init system when
+ managing services. These will be passed as key=value
+ parameters to the init system's commandline. kwargs
+ are ignored for init systems which do not support
+ key=value arguments via the commandline.
+ """
+ stopped = True
+ if service_running(service_name, **kwargs):
+ stopped = service_stop(service_name, **kwargs)
+ upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
+ sysv_file = os.path.join(initd_dir, service_name)
+ if init_is_systemd():
+ service('disable', service_name)
+ service('mask', service_name)
+ elif os.path.exists(upstart_file):
+ override_path = os.path.join(
+ init_dir, '{}.override'.format(service_name))
+ with open(override_path, 'w') as fh:
+ fh.write("manual\n")
+ elif os.path.exists(sysv_file):
+ subprocess.check_call(["update-rc.d", service_name, "disable"])
+ else:
+ raise ValueError(
+ "Unable to detect {0} as SystemD, Upstart {1} or"
+ " SysV {2}".format(
+ service_name, upstart_file, sysv_file))
+ return stopped
+
+
+def service_resume(service_name, init_dir="/etc/init",
+ initd_dir="/etc/init.d", **kwargs):
+ """Resume a system service.
+
+ Reenable starting again at boot. Start the service.
+
+ :param service_name: the name of the service to resume
+ :param init_dir: the path to the init dir
+ :param initd dir: the path to the initd dir
+ :param **kwargs: additional parameters to pass to the init system when
+ managing services. These will be passed as key=value
+ parameters to the init system's commandline. kwargs
+ are ignored for systemd enabled systems.
+ """
+ upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
+ sysv_file = os.path.join(initd_dir, service_name)
+ if init_is_systemd():
+ service('unmask', service_name)
+ service('enable', service_name)
+ elif os.path.exists(upstart_file):
+ override_path = os.path.join(
+ init_dir, '{}.override'.format(service_name))
+ if os.path.exists(override_path):
+ os.unlink(override_path)
+ elif os.path.exists(sysv_file):
+ subprocess.check_call(["update-rc.d", service_name, "enable"])
+ else:
+ raise ValueError(
+ "Unable to detect {0} as SystemD, Upstart {1} or"
+ " SysV {2}".format(
+ service_name, upstart_file, sysv_file))
+ started = service_running(service_name, **kwargs)
+
+ if not started:
+ started = service_start(service_name, **kwargs)
+ return started
+
+
+def service(action, service_name, **kwargs):
+ """Control a system service.
+
+ :param action: the action to take on the service
+ :param service_name: the name of the service to perform th action on
+ :param **kwargs: additional params to be passed to the service command in
+ the form of key=value.
+ """
+ if init_is_systemd():
+ cmd = ['systemctl', action, service_name]
+ else:
+ cmd = ['service', service_name, action]
+ for key, value in six.iteritems(kwargs):
+ parameter = '%s=%s' % (key, value)
+ cmd.append(parameter)
+ return subprocess.call(cmd) == 0
+
+
+_UPSTART_CONF = "/etc/init/{}.conf"
+_INIT_D_CONF = "/etc/init.d/{}"
+
+
+def service_running(service_name, **kwargs):
+ """Determine whether a system service is running.
+
+ :param service_name: the name of the service
+ :param **kwargs: additional args to pass to the service command. This is
+ used to pass additional key=value arguments to the
+ service command line for managing specific instance
+ units (e.g. service ceph-osd status id=2). The kwargs
+ are ignored in systemd services.
+ """
+ if init_is_systemd():
+ return service('is-active', service_name)
+ else:
+ if os.path.exists(_UPSTART_CONF.format(service_name)):
+ try:
+ cmd = ['status', service_name]
+ for key, value in six.iteritems(kwargs):
+ parameter = '%s=%s' % (key, value)
+ cmd.append(parameter)
+ output = subprocess.check_output(cmd,
+ stderr=subprocess.STDOUT).decode('UTF-8')
+ except subprocess.CalledProcessError:
+ return False
+ else:
+ # This works for upstart scripts where the 'service' command
+ # returns a consistent string to represent running
+ # 'start/running'
+ if ("start/running" in output or
+ "is running" in output or
+ "up and running" in output):
+ return True
+ elif os.path.exists(_INIT_D_CONF.format(service_name)):
+ # Check System V scripts init script return codes
+ return service('status', service_name)
+ return False
+
+
+SYSTEMD_SYSTEM = '/run/systemd/system'
+
+
+def init_is_systemd():
+ """Return True if the host system uses systemd, False otherwise."""
+ if lsb_release()['DISTRIB_CODENAME'] == 'trusty':
+ return False
+ return os.path.isdir(SYSTEMD_SYSTEM)
+
+
+def adduser(username, password=None, shell='/bin/bash',
+ system_user=False, primary_group=None,
+ secondary_groups=None, uid=None, home_dir=None):
+ """Add a user to the system.
+
+ Will log but otherwise succeed if the user already exists.
+
+ :param str username: Username to create
+ :param str password: Password for user; if ``None``, create a system user
+ :param str shell: The default shell for the user
+ :param bool system_user: Whether to create a login or system user
+ :param str primary_group: Primary group for user; defaults to username
+ :param list secondary_groups: Optional list of additional groups
+ :param int uid: UID for user being created
+ :param str home_dir: Home directory for user
+
+ :returns: The password database entry struct, as returned by `pwd.getpwnam`
+ """
+ try:
+ user_info = pwd.getpwnam(username)
+ log('user {0} already exists!'.format(username))
+ if uid:
+ user_info = pwd.getpwuid(int(uid))
+ log('user with uid {0} already exists!'.format(uid))
+ except KeyError:
+ log('creating user {0}'.format(username))
+ cmd = ['useradd']
+ if uid:
+ cmd.extend(['--uid', str(uid)])
+ if home_dir:
+ cmd.extend(['--home', str(home_dir)])
+ if system_user or password is None:
+ cmd.append('--system')
+ else:
+ cmd.extend([
+ '--create-home',
+ '--shell', shell,
+ '--password', password,
+ ])
+ if not primary_group:
+ try:
+ grp.getgrnam(username)
+ primary_group = username # avoid "group exists" error
+ except KeyError:
+ pass
+ if primary_group:
+ cmd.extend(['-g', primary_group])
+ if secondary_groups:
+ cmd.extend(['-G', ','.join(secondary_groups)])
+ cmd.append(username)
+ subprocess.check_call(cmd)
+ user_info = pwd.getpwnam(username)
+ return user_info
+
+
+def user_exists(username):
+ """Check if a user exists"""
+ try:
+ pwd.getpwnam(username)
+ user_exists = True
+ except KeyError:
+ user_exists = False
+ return user_exists
+
+
+def uid_exists(uid):
+ """Check if a uid exists"""
+ try:
+ pwd.getpwuid(uid)
+ uid_exists = True
+ except KeyError:
+ uid_exists = False
+ return uid_exists
+
+
+def group_exists(groupname):
+ """Check if a group exists"""
+ try:
+ grp.getgrnam(groupname)
+ group_exists = True
+ except KeyError:
+ group_exists = False
+ return group_exists
+
+
+def gid_exists(gid):
+ """Check if a gid exists"""
+ try:
+ grp.getgrgid(gid)
+ gid_exists = True
+ except KeyError:
+ gid_exists = False
+ return gid_exists
+
+
+def add_group(group_name, system_group=False, gid=None):
+ """Add a group to the system
+
+ Will log but otherwise succeed if the group already exists.
+
+ :param str group_name: group to create
+ :param bool system_group: Create system group
+ :param int gid: GID for user being created
+
+ :returns: The password database entry struct, as returned by `grp.getgrnam`
+ """
+ try:
+ group_info = grp.getgrnam(group_name)
+ log('group {0} already exists!'.format(group_name))
+ if gid:
+ group_info = grp.getgrgid(gid)
+ log('group with gid {0} already exists!'.format(gid))
+ except KeyError:
+ log('creating group {0}'.format(group_name))
+ add_new_group(group_name, system_group, gid)
+ group_info = grp.getgrnam(group_name)
+ return group_info
+
+
+def add_user_to_group(username, group):
+ """Add a user to a group"""
+ cmd = ['gpasswd', '-a', username, group]
+ log("Adding user {} to group {}".format(username, group))
+ subprocess.check_call(cmd)
+
+
+def chage(username, lastday=None, expiredate=None, inactive=None,
+ mindays=None, maxdays=None, root=None, warndays=None):
+ """Change user password expiry information
+
+ :param str username: User to update
+ :param str lastday: Set when password was changed in YYYY-MM-DD format
+ :param str expiredate: Set when user's account will no longer be
+ accessible in YYYY-MM-DD format.
+ -1 will remove an account expiration date.
+ :param str inactive: Set the number of days of inactivity after a password
+ has expired before the account is locked.
+ -1 will remove an account's inactivity.
+ :param str mindays: Set the minimum number of days between password
+ changes to MIN_DAYS.
+ 0 indicates the password can be changed anytime.
+ :param str maxdays: Set the maximum number of days during which a
+ password is valid.
+ -1 as MAX_DAYS will remove checking maxdays
+ :param str root: Apply changes in the CHROOT_DIR directory
+ :param str warndays: Set the number of days of warning before a password
+ change is required
+ :raises subprocess.CalledProcessError: if call to chage fails
+ """
+ cmd = ['chage']
+ if root:
+ cmd.extend(['--root', root])
+ if lastday:
+ cmd.extend(['--lastday', lastday])
+ if expiredate:
+ cmd.extend(['--expiredate', expiredate])
+ if inactive:
+ cmd.extend(['--inactive', inactive])
+ if mindays:
+ cmd.extend(['--mindays', mindays])
+ if maxdays:
+ cmd.extend(['--maxdays', maxdays])
+ if warndays:
+ cmd.extend(['--warndays', warndays])
+ cmd.append(username)
+ subprocess.check_call(cmd)
+
+remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1')
+
+def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
+ """Replicate the contents of a path"""
+ options = options or ['--delete', '--executability']
+ cmd = ['/usr/bin/rsync', flags]
+ if timeout:
+ cmd = ['timeout', str(timeout)] + cmd
+ cmd.extend(options)
+ cmd.append(from_path)
+ cmd.append(to_path)
+ log(" ".join(cmd))
+ return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8').strip()
+
+
+def symlink(source, destination):
+ """Create a symbolic link"""
+ log("Symlinking {} as {}".format(source, destination))
+ cmd = [
+ 'ln',
+ '-sf',
+ source,
+ destination,
+ ]
+ subprocess.check_call(cmd)
+
+
+def mkdir(path, owner='root', group='root', perms=0o555, force=False):
+ """Create a directory"""
+ log("Making dir {} {}:{} {:o}".format(path, owner, group,
+ perms))
+ uid = pwd.getpwnam(owner).pw_uid
+ gid = grp.getgrnam(group).gr_gid
+ realpath = os.path.abspath(path)
+ path_exists = os.path.exists(realpath)
+ if path_exists and force:
+ if not os.path.isdir(realpath):
+ log("Removing non-directory file {} prior to mkdir()".format(path))
+ os.unlink(realpath)
+ os.makedirs(realpath, perms)
+ elif not path_exists:
+ os.makedirs(realpath, perms)
+ os.chown(realpath, uid, gid)
+ os.chmod(realpath, perms)
+
+
+def write_file(path, content, owner='root', group='root', perms=0o444):
+ """Create or overwrite a file with the contents of a byte string."""
+ uid = pwd.getpwnam(owner).pw_uid
+ gid = grp.getgrnam(group).gr_gid
+ # lets see if we can grab the file and compare the context, to avoid doing
+ # a write.
+ existing_content = None
+ existing_uid, existing_gid = None, None
+ try:
+ with open(path, 'rb') as target:
+ existing_content = target.read()
+ stat = os.stat(path)
+ existing_uid, existing_gid = stat.st_uid, stat.st_gid
+ except:
+ pass
+ if content != existing_content:
+ log("Writing file {} {}:{} {:o}".format(path, owner, group, perms),
+ level=DEBUG)
+ with open(path, 'wb') as target:
+ os.fchown(target.fileno(), uid, gid)
+ os.fchmod(target.fileno(), perms)
+ target.write(content)
+ return
+ # the contents were the same, but we might still need to change the
+ # ownership.
+ if existing_uid != uid:
+ log("Changing uid on already existing content: {} -> {}"
+ .format(existing_uid, uid), level=DEBUG)
+ os.chown(path, uid, -1)
+ if existing_gid != gid:
+ log("Changing gid on already existing content: {} -> {}"
+ .format(existing_gid, gid), level=DEBUG)
+ os.chown(path, -1, gid)
+
+
+def fstab_remove(mp):
+ """Remove the given mountpoint entry from /etc/fstab"""
+ return Fstab.remove_by_mountpoint(mp)
+
+
+def fstab_add(dev, mp, fs, options=None):
+ """Adds the given device entry to the /etc/fstab file"""
+ return Fstab.add(dev, mp, fs, options=options)
+
+
+def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"):
+ """Mount a filesystem at a particular mountpoint"""
+ cmd_args = ['mount']
+ if options is not None:
+ cmd_args.extend(['-o', options])
+ cmd_args.extend([device, mountpoint])
+ try:
+ subprocess.check_output(cmd_args)
+ except subprocess.CalledProcessError as e:
+ log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output))
+ return False
+
+ if persist:
+ return fstab_add(device, mountpoint, filesystem, options=options)
+ return True
+
+
+def umount(mountpoint, persist=False):
+ """Unmount a filesystem"""
+ cmd_args = ['umount', mountpoint]
+ try:
+ subprocess.check_output(cmd_args)
+ except subprocess.CalledProcessError as e:
+ log('Error unmounting {}\n{}'.format(mountpoint, e.output))
+ return False
+
+ if persist:
+ return fstab_remove(mountpoint)
+ return True
+
+
+def mounts():
+ """Get a list of all mounted volumes as [[mountpoint,device],[...]]"""
+ with open('/proc/mounts') as f:
+ # [['/mount/point','/dev/path'],[...]]
+ system_mounts = [m[1::-1] for m in [l.strip().split()
+ for l in f.readlines()]]
+ return system_mounts
+
+
+def fstab_mount(mountpoint):
+ """Mount filesystem using fstab"""
+ cmd_args = ['mount', mountpoint]
+ try:
+ subprocess.check_output(cmd_args)
+ except subprocess.CalledProcessError as e:
+ log('Error unmounting {}\n{}'.format(mountpoint, e.output))
+ return False
+ return True
+
+
+def file_hash(path, hash_type='md5'):
+ """Generate a hash checksum of the contents of 'path' or None if not found.
+
+ :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`,
+ such as md5, sha1, sha256, sha512, etc.
+ """
+ if os.path.exists(path):
+ h = getattr(hashlib, hash_type)()
+ with open(path, 'rb') as source:
+ h.update(source.read())
+ return h.hexdigest()
+ else:
+ return None
+
+
+def path_hash(path):
+ """Generate a hash checksum of all files matching 'path'. Standard
+ wildcards like '*' and '?' are supported, see documentation for the 'glob'
+ module for more information.
+
+ :return: dict: A { filename: hash } dictionary for all matched files.
+ Empty if none found.
+ """
+ return {
+ filename: file_hash(filename)
+ for filename in glob.iglob(path)
+ }
+
+
+def check_hash(path, checksum, hash_type='md5'):
+ """Validate a file using a cryptographic checksum.
+
+ :param str checksum: Value of the checksum used to validate the file.
+ :param str hash_type: Hash algorithm used to generate `checksum`.
+ Can be any hash alrgorithm supported by :mod:`hashlib`,
+ such as md5, sha1, sha256, sha512, etc.
+ :raises ChecksumError: If the file fails the checksum
+
+ """
+ actual_checksum = file_hash(path, hash_type)
+ if checksum != actual_checksum:
+ raise ChecksumError("'%s' != '%s'" % (checksum, actual_checksum))
+
+
+class ChecksumError(ValueError):
+ """A class derived from Value error to indicate the checksum failed."""
+ pass
+
+
+def restart_on_change(restart_map, stopstart=False, restart_functions=None):
+ """Restart services based on configuration files changing
+
+ This function is used a decorator, for example::
+
+ @restart_on_change({
+ '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ]
+ '/etc/apache/sites-enabled/*': [ 'apache2' ]
+ })
+ def config_changed():
+ pass # your code here
+
+ In this example, the cinder-api and cinder-volume services
+ would be restarted if /etc/ceph/ceph.conf is changed by the
+ ceph_client_changed function. The apache2 service would be
+ restarted if any file matching the pattern got changed, created
+ or removed. Standard wildcards are supported, see documentation
+ for the 'glob' module for more information.
+
+ @param restart_map: {path_file_name: [service_name, ...]
+ @param stopstart: DEFAULT false; whether to stop, start OR restart
+ @param restart_functions: nonstandard functions to use to restart services
+ {svc: func, ...}
+ @returns result from decorated function
+ """
+ def wrap(f):
+ @functools.wraps(f)
+ def wrapped_f(*args, **kwargs):
+ return restart_on_change_helper(
+ (lambda: f(*args, **kwargs)), restart_map, stopstart,
+ restart_functions)
+ return wrapped_f
+ return wrap
+
+
+def restart_on_change_helper(lambda_f, restart_map, stopstart=False,
+ restart_functions=None):
+ """Helper function to perform the restart_on_change function.
+
+ This is provided for decorators to restart services if files described
+ in the restart_map have changed after an invocation of lambda_f().
+
+ @param lambda_f: function to call.
+ @param restart_map: {file: [service, ...]}
+ @param stopstart: whether to stop, start or restart a service
+ @param restart_functions: nonstandard functions to use to restart services
+ {svc: func, ...}
+ @returns result of lambda_f()
+ """
+ if restart_functions is None:
+ restart_functions = {}
+ checksums = {path: path_hash(path) for path in restart_map}
+ r = lambda_f()
+ # create a list of lists of the services to restart
+ restarts = [restart_map[path]
+ for path in restart_map
+ if path_hash(path) != checksums[path]]
+ # create a flat list of ordered services without duplicates from lists
+ services_list = list(OrderedDict.fromkeys(itertools.chain(*restarts)))
+ if services_list:
+ actions = ('stop', 'start') if stopstart else ('restart',)
+ for service_name in services_list:
+ if service_name in restart_functions:
+ restart_functions[service_name](service_name)
+ else:
+ for action in actions:
+ service(action, service_name)
+ return r
+
+
+def pwgen(length=None):
+ """Generate a random pasword."""
+ if length is None:
+ # A random length is ok to use a weak PRNG
+ length = random.choice(range(35, 45))
+ alphanumeric_chars = [
+ l for l in (string.ascii_letters + string.digits)
+ if l not in 'l0QD1vAEIOUaeiou']
+ # Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the
+ # actual password
+ random_generator = random.SystemRandom()
+ random_chars = [
+ random_generator.choice(alphanumeric_chars) for _ in range(length)]
+ return(''.join(random_chars))
+
+
+def is_phy_iface(interface):
+ """Returns True if interface is not virtual, otherwise False."""
+ if interface:
+ sys_net = '/sys/class/net'
+ if os.path.isdir(sys_net):
+ for iface in glob.glob(os.path.join(sys_net, '*')):
+ if '/virtual/' in os.path.realpath(iface):
+ continue
+
+ if interface == os.path.basename(iface):
+ return True
+
+ return False
+
+
+def get_bond_master(interface):
+ """Returns bond master if interface is bond slave otherwise None.
+
+ NOTE: the provided interface is expected to be physical
+ """
+ if interface:
+ iface_path = '/sys/class/net/%s' % (interface)
+ if os.path.exists(iface_path):
+ if '/virtual/' in os.path.realpath(iface_path):
+ return None
+
+ master = os.path.join(iface_path, 'master')
+ if os.path.exists(master):
+ master = os.path.realpath(master)
+ # make sure it is a bond master
+ if os.path.exists(os.path.join(master, 'bonding')):
+ return os.path.basename(master)
+
+ return None
+
+
+def list_nics(nic_type=None):
+ """Return a list of nics of given type(s)"""
+ if isinstance(nic_type, six.string_types):
+ int_types = [nic_type]
+ else:
+ int_types = nic_type
+
+ interfaces = []
+ if nic_type:
+ for int_type in int_types:
+ cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
+ ip_output = subprocess.check_output(cmd).decode('UTF-8')
+ ip_output = ip_output.split('\n')
+ ip_output = (line for line in ip_output if line)
+ for line in ip_output:
+ if line.split()[1].startswith(int_type):
+ matched = re.search('.*: (' + int_type +
+ r'[0-9]+\.[0-9]+)@.*', line)
+ if matched:
+ iface = matched.groups()[0]
+ else:
+ iface = line.split()[1].replace(":", "")
+
+ if iface not in interfaces:
+ interfaces.append(iface)
+ else:
+ cmd = ['ip', 'a']
+ ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
+ ip_output = (line.strip() for line in ip_output if line)
+
+ key = re.compile('^[0-9]+:\s+(.+):')
+ for line in ip_output:
+ matched = re.search(key, line)
+ if matched:
+ iface = matched.group(1)
+ iface = iface.partition("@")[0]
+ if iface not in interfaces:
+ interfaces.append(iface)
+
+ return interfaces
+
+
+def set_nic_mtu(nic, mtu):
+ """Set the Maximum Transmission Unit (MTU) on a network interface."""
+ cmd = ['ip', 'link', 'set', nic, 'mtu', mtu]
+ subprocess.check_call(cmd)
+
+
+def get_nic_mtu(nic):
+ """Return the Maximum Transmission Unit (MTU) for a network interface."""
+ cmd = ['ip', 'addr', 'show', nic]
+ ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
+ mtu = ""
+ for line in ip_output:
+ words = line.split()
+ if 'mtu' in words:
+ mtu = words[words.index("mtu") + 1]
+ return mtu
+
+
+def get_nic_hwaddr(nic):
+ """Return the Media Access Control (MAC) for a network interface."""
+ cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
+ ip_output = subprocess.check_output(cmd).decode('UTF-8')
+ hwaddr = ""
+ words = ip_output.split()
+ if 'link/ether' in words:
+ hwaddr = words[words.index('link/ether') + 1]
+ return hwaddr
+
+
+@contextmanager
+def chdir(directory):
+ """Change the current working directory to a different directory for a code
+ block and return the previous directory after the block exits. Useful to
+ run commands from a specificed directory.
+
+ :param str directory: The directory path to change to for this context.
+ """
+ cur = os.getcwd()
+ try:
+ yield os.chdir(directory)
+ finally:
+ os.chdir(cur)
+
+
+def chownr(path, owner, group, follow_links=True, chowntopdir=False):
+ """Recursively change user and group ownership of files and directories
+ in given path. Doesn't chown path itself by default, only its children.
+
+ :param str path: The string path to start changing ownership.
+ :param str owner: The owner string to use when looking up the uid.
+ :param str group: The group string to use when looking up the gid.
+ :param bool follow_links: Also follow and chown links if True
+ :param bool chowntopdir: Also chown path itself if True
+ """
+ uid = pwd.getpwnam(owner).pw_uid
+ gid = grp.getgrnam(group).gr_gid
+ if follow_links:
+ chown = os.chown
+ else:
+ chown = os.lchown
+
+ if chowntopdir:
+ broken_symlink = os.path.lexists(path) and not os.path.exists(path)
+ if not broken_symlink:
+ chown(path, uid, gid)
+ for root, dirs, files in os.walk(path, followlinks=follow_links):
+ for name in dirs + files:
+ full = os.path.join(root, name)
+ broken_symlink = os.path.lexists(full) and not os.path.exists(full)
+ if not broken_symlink:
+ chown(full, uid, gid)
+
+
+def lchownr(path, owner, group):
+ """Recursively change user and group ownership of files and directories
+ in a given path, not following symbolic links. See the documentation for
+ 'os.lchown' for more information.
+
+ :param str path: The string path to start changing ownership.
+ :param str owner: The owner string to use when looking up the uid.
+ :param str group: The group string to use when looking up the gid.
+ """
+ chownr(path, owner, group, follow_links=False)
+
+
+def owner(path):
+ """Returns a tuple containing the username & groupname owning the path.
+
+ :param str path: the string path to retrieve the ownership
+ :return tuple(str, str): A (username, groupname) tuple containing the
+ name of the user and group owning the path.
+ :raises OSError: if the specified path does not exist
+ """
+ stat = os.stat(path)
+ username = pwd.getpwuid(stat.st_uid)[0]
+ groupname = grp.getgrgid(stat.st_gid)[0]
+ return username, groupname
+
+
+def get_total_ram():
+ """The total amount of system RAM in bytes.
+
+ This is what is reported by the OS, and may be overcommitted when
+ there are multiple containers hosted on the same machine.
+ """
+ with open('/proc/meminfo', 'r') as f:
+ for line in f.readlines():
+ if line:
+ key, value, unit = line.split()
+ if key == 'MemTotal:':
+ assert unit == 'kB', 'Unknown unit'
+ return int(value) * 1024 # Classic, not KiB.
+ raise NotImplementedError()
+
+
+UPSTART_CONTAINER_TYPE = '/run/container_type'
+
+
+def is_container():
+ """Determine whether unit is running in a container
+
+ @return: boolean indicating if unit is in a container
+ """
+ if init_is_systemd():
+ # Detect using systemd-detect-virt
+ return subprocess.call(['systemd-detect-virt',
+ '--container']) == 0
+ else:
+ # Detect using upstart container file marker
+ return os.path.exists(UPSTART_CONTAINER_TYPE)
+
+
+def add_to_updatedb_prunepath(path, updatedb_path=UPDATEDB_PATH):
+ with open(updatedb_path, 'r+') as f_id:
+ updatedb_text = f_id.read()
+ output = updatedb(updatedb_text, path)
+ f_id.seek(0)
+ f_id.write(output)
+ f_id.truncate()
+
+
+def updatedb(updatedb_text, new_path):
+ lines = [line for line in updatedb_text.split("\n")]
+ for i, line in enumerate(lines):
+ if line.startswith("PRUNEPATHS="):
+ paths_line = line.split("=")[1].replace('"', '')
+ paths = paths_line.split(" ")
+ if new_path not in paths:
+ paths.append(new_path)
+ lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths))
+ output = "\n".join(lines)
+ return output
+
+
+def modulo_distribution(modulo=3, wait=30):
+ """ Modulo distribution
+
+ This helper uses the unit number, a modulo value and a constant wait time
+ to produce a calculated wait time distribution. This is useful in large
+ scale deployments to distribute load during an expensive operation such as
+ service restarts.
+
+ If you have 1000 nodes that need to restart 100 at a time 1 minute at a
+ time:
+
+ time.wait(modulo_distribution(modulo=100, wait=60))
+ restart()
+
+ If you need restarts to happen serially set modulo to the exact number of
+ nodes and set a high constant wait time:
+
+ time.wait(modulo_distribution(modulo=10, wait=120))
+ restart()
+
+ @param modulo: int The modulo number creates the group distribution
+ @param wait: int The constant time wait value
+ @return: int Calculated time to wait for unit operation
+ """
+ unit_number = int(local_unit().split('/')[1])
+ return (unit_number % modulo) * wait
diff --git a/tests/charmhelpers/core/host_factory/__init__.py b/tests/charmhelpers/core/host_factory/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/charmhelpers/core/host_factory/centos.py b/tests/charmhelpers/core/host_factory/centos.py
new file mode 100644
index 0000000..7781a39
--- /dev/null
+++ b/tests/charmhelpers/core/host_factory/centos.py
@@ -0,0 +1,72 @@
+import subprocess
+import yum
+import os
+
+from charmhelpers.core.strutils import BasicStringComparator
+
+
+class CompareHostReleases(BasicStringComparator):
+ """Provide comparisons of Host releases.
+
+ Use in the form of
+
+ if CompareHostReleases(release) > 'trusty':
+ # do something with mitaka
+ """
+
+ def __init__(self, item):
+ raise NotImplementedError(
+ "CompareHostReleases() is not implemented for CentOS")
+
+
+def service_available(service_name):
+ # """Determine whether a system service is available."""
+ if os.path.isdir('/run/systemd/system'):
+ cmd = ['systemctl', 'is-enabled', service_name]
+ else:
+ cmd = ['service', service_name, 'is-enabled']
+ return subprocess.call(cmd) == 0
+
+
+def add_new_group(group_name, system_group=False, gid=None):
+ cmd = ['groupadd']
+ if gid:
+ cmd.extend(['--gid', str(gid)])
+ if system_group:
+ cmd.append('-r')
+ cmd.append(group_name)
+ subprocess.check_call(cmd)
+
+
+def lsb_release():
+ """Return /etc/os-release in a dict."""
+ d = {}
+ with open('/etc/os-release', 'r') as lsb:
+ for l in lsb:
+ s = l.split('=')
+ if len(s) != 2:
+ continue
+ d[s[0].strip()] = s[1].strip()
+ return d
+
+
+def cmp_pkgrevno(package, revno, pkgcache=None):
+ """Compare supplied revno with the revno of the installed package.
+
+ * 1 => Installed revno is greater than supplied arg
+ * 0 => Installed revno is the same as supplied arg
+ * -1 => Installed revno is less than supplied arg
+
+ This function imports YumBase function if the pkgcache argument
+ is None.
+ """
+ if not pkgcache:
+ y = yum.YumBase()
+ packages = y.doPackageLists()
+ pkgcache = {i.Name: i.version for i in packages['installed']}
+ pkg = pkgcache[package]
+ if pkg > revno:
+ return 1
+ if pkg < revno:
+ return -1
+ return 0
diff --git a/tests/charmhelpers/core/host_factory/ubuntu.py b/tests/charmhelpers/core/host_factory/ubuntu.py
new file mode 100644
index 0000000..d8dc378
--- /dev/null
+++ b/tests/charmhelpers/core/host_factory/ubuntu.py
@@ -0,0 +1,89 @@
+import subprocess
+
+from charmhelpers.core.strutils import BasicStringComparator
+
+
+UBUNTU_RELEASES = (
+ 'lucid',
+ 'maverick',
+ 'natty',
+ 'oneiric',
+ 'precise',
+ 'quantal',
+ 'raring',
+ 'saucy',
+ 'trusty',
+ 'utopic',
+ 'vivid',
+ 'wily',
+ 'xenial',
+ 'yakkety',
+ 'zesty',
+ 'artful',
+)
+
+
+class CompareHostReleases(BasicStringComparator):
+ """Provide comparisons of Ubuntu releases.
+
+ Use in the form of
+
+ if CompareHostReleases(release) > 'trusty':
+ # do something with mitaka
+ """
+ _list = UBUNTU_RELEASES
+
+
+def service_available(service_name):
+ """Determine whether a system service is available"""
+ try:
+ subprocess.check_output(
+ ['service', service_name, 'status'],
+ stderr=subprocess.STDOUT).decode('UTF-8')
+ except subprocess.CalledProcessError as e:
+ return b'unrecognized service' not in e.output
+ else:
+ return True
+
+
+def add_new_group(group_name, system_group=False, gid=None):
+ cmd = ['addgroup']
+ if gid:
+ cmd.extend(['--gid', str(gid)])
+ if system_group:
+ cmd.append('--system')
+ else:
+ cmd.extend([
+ '--group',
+ ])
+ cmd.append(group_name)
+ subprocess.check_call(cmd)
+
+
+def lsb_release():
+ """Return /etc/lsb-release in a dict"""
+ d = {}
+ with open('/etc/lsb-release', 'r') as lsb:
+ for l in lsb:
+ k, v = l.split('=')
+ d[k.strip()] = v.strip()
+ return d
+
+
+def cmp_pkgrevno(package, revno, pkgcache=None):
+ """Compare supplied revno with the revno of the installed package.
+
+ * 1 => Installed revno is greater than supplied arg
+ * 0 => Installed revno is the same as supplied arg
+ * -1 => Installed revno is less than supplied arg
+
+ This function imports apt_cache function from charmhelpers.fetch if
+ the pkgcache argument is None. Be sure to add charmhelpers.fetch if
+ you call this function, or pass an apt_pkg.Cache() instance.
+ """
+ import apt_pkg
+ if not pkgcache:
+ from charmhelpers.fetch import apt_cache
+ pkgcache = apt_cache()
+ pkg = pkgcache[package]
+ return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)
diff --git a/tests/charmhelpers/core/hugepage.py b/tests/charmhelpers/core/hugepage.py
new file mode 100644
index 0000000..54b5b5e
--- /dev/null
+++ b/tests/charmhelpers/core/hugepage.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import yaml
+from charmhelpers.core import fstab
+from charmhelpers.core import sysctl
+from charmhelpers.core.host import (
+ add_group,
+ add_user_to_group,
+ fstab_mount,
+ mkdir,
+)
+from charmhelpers.core.strutils import bytes_from_string
+from subprocess import check_output
+
+
+def hugepage_support(user, group='hugetlb', nr_hugepages=256,
+ max_map_count=65536, mnt_point='/run/hugepages/kvm',
+ pagesize='2MB', mount=True, set_shmmax=False):
+ """Enable hugepages on system.
+
+ Args:
+ user (str) -- Username to allow access to hugepages to
+ group (str) -- Group name to own hugepages
+ nr_hugepages (int) -- Number of pages to reserve
+ max_map_count (int) -- Number of Virtual Memory Areas a process can own
+ mnt_point (str) -- Directory to mount hugepages on
+ pagesize (str) -- Size of hugepages
+ mount (bool) -- Whether to Mount hugepages
+ """
+ group_info = add_group(group)
+ gid = group_info.gr_gid
+ add_user_to_group(user, group)
+ if max_map_count < 2 * nr_hugepages:
+ max_map_count = 2 * nr_hugepages
+ sysctl_settings = {
+ 'vm.nr_hugepages': nr_hugepages,
+ 'vm.max_map_count': max_map_count,
+ 'vm.hugetlb_shm_group': gid,
+ }
+ if set_shmmax:
+ shmmax_current = int(check_output(['sysctl', '-n', 'kernel.shmmax']))
+ shmmax_minsize = bytes_from_string(pagesize) * nr_hugepages
+ if shmmax_minsize > shmmax_current:
+ sysctl_settings['kernel.shmmax'] = shmmax_minsize
+ sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf')
+ mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False)
+ lfstab = fstab.Fstab()
+ fstab_entry = lfstab.get_entry_by_attr('mountpoint', mnt_point)
+ if fstab_entry:
+ lfstab.remove_entry(fstab_entry)
+ entry = lfstab.Entry('nodev', mnt_point, 'hugetlbfs',
+ 'mode=1770,gid={},pagesize={}'.format(gid, pagesize), 0, 0)
+ lfstab.add_entry(entry)
+ if mount:
+ fstab_mount(mnt_point)
diff --git a/tests/charmhelpers/core/kernel.py b/tests/charmhelpers/core/kernel.py
new file mode 100644
index 0000000..2d40452
--- /dev/null
+++ b/tests/charmhelpers/core/kernel.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import subprocess
+
+from charmhelpers.osplatform import get_platform
+from charmhelpers.core.hookenv import (
+ log,
+ INFO
+)
+
+__platform__ = get_platform()
+if __platform__ == "ubuntu":
+ from charmhelpers.core.kernel_factory.ubuntu import (
+ persistent_modprobe,
+ update_initramfs,
+ ) # flake8: noqa -- ignore F401 for this import
+elif __platform__ == "centos":
+ from charmhelpers.core.kernel_factory.centos import (
+ persistent_modprobe,
+ update_initramfs,
+ ) # flake8: noqa -- ignore F401 for this import
+
+__author__ = "Jorge Niedbalski "
+
+
+def modprobe(module, persist=True):
+ """Load a kernel module and configure for auto-load on reboot."""
+ cmd = ['modprobe', module]
+
+ log('Loading kernel module %s' % module, level=INFO)
+
+ subprocess.check_call(cmd)
+ if persist:
+ persistent_modprobe(module)
+
+
+def rmmod(module, force=False):
+ """Remove a module from the linux kernel"""
+ cmd = ['rmmod']
+ if force:
+ cmd.append('-f')
+ cmd.append(module)
+ log('Removing kernel module %s' % module, level=INFO)
+ return subprocess.check_call(cmd)
+
+
+def lsmod():
+ """Shows what kernel modules are currently loaded"""
+ return subprocess.check_output(['lsmod'],
+ universal_newlines=True)
+
+
+def is_module_loaded(module):
+ """Checks if a kernel module is already loaded"""
+ matches = re.findall('^%s[ ]+' % module, lsmod(), re.M)
+ return len(matches) > 0
diff --git a/tests/charmhelpers/core/kernel_factory/__init__.py b/tests/charmhelpers/core/kernel_factory/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/charmhelpers/core/kernel_factory/centos.py b/tests/charmhelpers/core/kernel_factory/centos.py
new file mode 100644
index 0000000..1c402c1
--- /dev/null
+++ b/tests/charmhelpers/core/kernel_factory/centos.py
@@ -0,0 +1,17 @@
+import subprocess
+import os
+
+
+def persistent_modprobe(module):
+ """Load a kernel module and configure for auto-load on reboot."""
+ if not os.path.exists('/etc/rc.modules'):
+ open('/etc/rc.modules', 'a')
+ os.chmod('/etc/rc.modules', 111)
+ with open('/etc/rc.modules', 'r+') as modules:
+ if module not in modules.read():
+ modules.write('modprobe %s\n' % module)
+
+
+def update_initramfs(version='all'):
+ """Updates an initramfs image."""
+ return subprocess.check_call(["dracut", "-f", version])
diff --git a/tests/charmhelpers/core/kernel_factory/ubuntu.py b/tests/charmhelpers/core/kernel_factory/ubuntu.py
new file mode 100644
index 0000000..3de372f
--- /dev/null
+++ b/tests/charmhelpers/core/kernel_factory/ubuntu.py
@@ -0,0 +1,13 @@
+import subprocess
+
+
+def persistent_modprobe(module):
+ """Load a kernel module and configure for auto-load on reboot."""
+ with open('/etc/modules', 'r+') as modules:
+ if module not in modules.read():
+ modules.write(module + "\n")
+
+
+def update_initramfs(version='all'):
+ """Updates an initramfs image."""
+ return subprocess.check_call(["update-initramfs", "-k", version, "-u"])
diff --git a/tests/charmhelpers/core/services/__init__.py b/tests/charmhelpers/core/services/__init__.py
new file mode 100644
index 0000000..61fd074
--- /dev/null
+++ b/tests/charmhelpers/core/services/__init__.py
@@ -0,0 +1,16 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .base import * # NOQA
+from .helpers import * # NOQA
diff --git a/tests/charmhelpers/core/services/base.py b/tests/charmhelpers/core/services/base.py
new file mode 100644
index 0000000..ca9dc99
--- /dev/null
+++ b/tests/charmhelpers/core/services/base.py
@@ -0,0 +1,351 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import json
+from inspect import getargspec
+from collections import Iterable, OrderedDict
+
+from charmhelpers.core import host
+from charmhelpers.core import hookenv
+
+
+__all__ = ['ServiceManager', 'ManagerCallback',
+ 'PortManagerCallback', 'open_ports', 'close_ports', 'manage_ports',
+ 'service_restart', 'service_stop']
+
+
+class ServiceManager(object):
+ def __init__(self, services=None):
+ """
+ Register a list of services, given their definitions.
+
+ Service definitions are dicts in the following formats (all keys except
+ 'service' are optional)::
+
+ {
+ "service": ,
+ "required_data": ,
+ "provided_data": ,
+ "data_ready": ,
+ "data_lost": ,
+ "start": ,
+ "stop": ,
+ "ports": ,
+ }
+
+ The 'required_data' list should contain dicts of required data (or
+ dependency managers that act like dicts and know how to collect the data).
+ Only when all items in the 'required_data' list are populated are the list
+ of 'data_ready' and 'start' callbacks executed. See `is_ready()` for more
+ information.
+
+ The 'provided_data' list should contain relation data providers, most likely
+ a subclass of :class:`charmhelpers.core.services.helpers.RelationContext`,
+ that will indicate a set of data to set on a given relation.
+
+ The 'data_ready' value should be either a single callback, or a list of
+ callbacks, to be called when all items in 'required_data' pass `is_ready()`.
+ Each callback will be called with the service name as the only parameter.
+ After all of the 'data_ready' callbacks are called, the 'start' callbacks
+ are fired.
+
+ The 'data_lost' value should be either a single callback, or a list of
+ callbacks, to be called when a 'required_data' item no longer passes
+ `is_ready()`. Each callback will be called with the service name as the
+ only parameter. After all of the 'data_lost' callbacks are called,
+ the 'stop' callbacks are fired.
+
+ The 'start' value should be either a single callback, or a list of
+ callbacks, to be called when starting the service, after the 'data_ready'
+ callbacks are complete. Each callback will be called with the service
+ name as the only parameter. This defaults to
+ `[host.service_start, services.open_ports]`.
+
+ The 'stop' value should be either a single callback, or a list of
+ callbacks, to be called when stopping the service. If the service is
+ being stopped because it no longer has all of its 'required_data', this
+ will be called after all of the 'data_lost' callbacks are complete.
+ Each callback will be called with the service name as the only parameter.
+ This defaults to `[services.close_ports, host.service_stop]`.
+
+ The 'ports' value should be a list of ports to manage. The default
+ 'start' handler will open the ports after the service is started,
+ and the default 'stop' handler will close the ports prior to stopping
+ the service.
+
+
+ Examples:
+
+ The following registers an Upstart service called bingod that depends on
+ a mongodb relation and which runs a custom `db_migrate` function prior to
+ restarting the service, and a Runit service called spadesd::
+
+ manager = services.ServiceManager([
+ {
+ 'service': 'bingod',
+ 'ports': [80, 443],
+ 'required_data': [MongoRelation(), config(), {'my': 'data'}],
+ 'data_ready': [
+ services.template(source='bingod.conf'),
+ services.template(source='bingod.ini',
+ target='/etc/bingod.ini',
+ owner='bingo', perms=0400),
+ ],
+ },
+ {
+ 'service': 'spadesd',
+ 'data_ready': services.template(source='spadesd_run.j2',
+ target='/etc/sv/spadesd/run',
+ perms=0555),
+ 'start': runit_start,
+ 'stop': runit_stop,
+ },
+ ])
+ manager.manage()
+ """
+ self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json')
+ self._ready = None
+ self.services = OrderedDict()
+ for service in services or []:
+ service_name = service['service']
+ self.services[service_name] = service
+
+ def manage(self):
+ """
+ Handle the current hook by doing The Right Thing with the registered services.
+ """
+ hookenv._run_atstart()
+ try:
+ hook_name = hookenv.hook_name()
+ if hook_name == 'stop':
+ self.stop_services()
+ else:
+ self.reconfigure_services()
+ self.provide_data()
+ except SystemExit as x:
+ if x.code is None or x.code == 0:
+ hookenv._run_atexit()
+ hookenv._run_atexit()
+
+ def provide_data(self):
+ """
+ Set the relation data for each provider in the ``provided_data`` list.
+
+ A provider must have a `name` attribute, which indicates which relation
+ to set data on, and a `provide_data()` method, which returns a dict of
+ data to set.
+
+ The `provide_data()` method can optionally accept two parameters:
+
+ * ``remote_service`` The name of the remote service that the data will
+ be provided to. The `provide_data()` method will be called once
+ for each connected service (not unit). This allows the method to
+ tailor its data to the given service.
+ * ``service_ready`` Whether or not the service definition had all of
+ its requirements met, and thus the ``data_ready`` callbacks run.
+
+ Note that the ``provided_data`` methods are now called **after** the
+ ``data_ready`` callbacks are run. This gives the ``data_ready`` callbacks
+ a chance to generate any data necessary for the providing to the remote
+ services.
+ """
+ for service_name, service in self.services.items():
+ service_ready = self.is_ready(service_name)
+ for provider in service.get('provided_data', []):
+ for relid in hookenv.relation_ids(provider.name):
+ units = hookenv.related_units(relid)
+ if not units:
+ continue
+ remote_service = units[0].split('/')[0]
+ argspec = getargspec(provider.provide_data)
+ if len(argspec.args) > 1:
+ data = provider.provide_data(remote_service, service_ready)
+ else:
+ data = provider.provide_data()
+ if data:
+ hookenv.relation_set(relid, data)
+
+ def reconfigure_services(self, *service_names):
+ """
+ Update all files for one or more registered services, and,
+ if ready, optionally restart them.
+
+ If no service names are given, reconfigures all registered services.
+ """
+ for service_name in service_names or self.services.keys():
+ if self.is_ready(service_name):
+ self.fire_event('data_ready', service_name)
+ self.fire_event('start', service_name, default=[
+ service_restart,
+ manage_ports])
+ self.save_ready(service_name)
+ else:
+ if self.was_ready(service_name):
+ self.fire_event('data_lost', service_name)
+ self.fire_event('stop', service_name, default=[
+ manage_ports,
+ service_stop])
+ self.save_lost(service_name)
+
+ def stop_services(self, *service_names):
+ """
+ Stop one or more registered services, by name.
+
+ If no service names are given, stops all registered services.
+ """
+ for service_name in service_names or self.services.keys():
+ self.fire_event('stop', service_name, default=[
+ manage_ports,
+ service_stop])
+
+ def get_service(self, service_name):
+ """
+ Given the name of a registered service, return its service definition.
+ """
+ service = self.services.get(service_name)
+ if not service:
+ raise KeyError('Service not registered: %s' % service_name)
+ return service
+
+ def fire_event(self, event_name, service_name, default=None):
+ """
+ Fire a data_ready, data_lost, start, or stop event on a given service.
+ """
+ service = self.get_service(service_name)
+ callbacks = service.get(event_name, default)
+ if not callbacks:
+ return
+ if not isinstance(callbacks, Iterable):
+ callbacks = [callbacks]
+ for callback in callbacks:
+ if isinstance(callback, ManagerCallback):
+ callback(self, service_name, event_name)
+ else:
+ callback(service_name)
+
+ def is_ready(self, service_name):
+ """
+ Determine if a registered service is ready, by checking its 'required_data'.
+
+ A 'required_data' item can be any mapping type, and is considered ready
+ if `bool(item)` evaluates as True.
+ """
+ service = self.get_service(service_name)
+ reqs = service.get('required_data', [])
+ return all(bool(req) for req in reqs)
+
+ def _load_ready_file(self):
+ if self._ready is not None:
+ return
+ if os.path.exists(self._ready_file):
+ with open(self._ready_file) as fp:
+ self._ready = set(json.load(fp))
+ else:
+ self._ready = set()
+
+ def _save_ready_file(self):
+ if self._ready is None:
+ return
+ with open(self._ready_file, 'w') as fp:
+ json.dump(list(self._ready), fp)
+
+ def save_ready(self, service_name):
+ """
+ Save an indicator that the given service is now data_ready.
+ """
+ self._load_ready_file()
+ self._ready.add(service_name)
+ self._save_ready_file()
+
+ def save_lost(self, service_name):
+ """
+ Save an indicator that the given service is no longer data_ready.
+ """
+ self._load_ready_file()
+ self._ready.discard(service_name)
+ self._save_ready_file()
+
+ def was_ready(self, service_name):
+ """
+ Determine if the given service was previously data_ready.
+ """
+ self._load_ready_file()
+ return service_name in self._ready
+
+
+class ManagerCallback(object):
+ """
+ Special case of a callback that takes the `ServiceManager` instance
+ in addition to the service name.
+
+ Subclasses should implement `__call__` which should accept three parameters:
+
+ * `manager` The `ServiceManager` instance
+ * `service_name` The name of the service it's being triggered for
+ * `event_name` The name of the event that this callback is handling
+ """
+ def __call__(self, manager, service_name, event_name):
+ raise NotImplementedError()
+
+
+class PortManagerCallback(ManagerCallback):
+ """
+ Callback class that will open or close ports, for use as either
+ a start or stop action.
+ """
+ def __call__(self, manager, service_name, event_name):
+ service = manager.get_service(service_name)
+ new_ports = service.get('ports', [])
+ port_file = os.path.join(hookenv.charm_dir(), '.{}.ports'.format(service_name))
+ if os.path.exists(port_file):
+ with open(port_file) as fp:
+ old_ports = fp.read().split(',')
+ for old_port in old_ports:
+ if bool(old_port):
+ old_port = int(old_port)
+ if old_port not in new_ports:
+ hookenv.close_port(old_port)
+ with open(port_file, 'w') as fp:
+ fp.write(','.join(str(port) for port in new_ports))
+ for port in new_ports:
+ if event_name == 'start':
+ hookenv.open_port(port)
+ elif event_name == 'stop':
+ hookenv.close_port(port)
+
+
+def service_stop(service_name):
+ """
+ Wrapper around host.service_stop to prevent spurious "unknown service"
+ messages in the logs.
+ """
+ if host.service_running(service_name):
+ host.service_stop(service_name)
+
+
+def service_restart(service_name):
+ """
+ Wrapper around host.service_restart to prevent spurious "unknown service"
+ messages in the logs.
+ """
+ if host.service_available(service_name):
+ if host.service_running(service_name):
+ host.service_restart(service_name)
+ else:
+ host.service_start(service_name)
+
+
+# Convenience aliases
+open_ports = close_ports = manage_ports = PortManagerCallback()
diff --git a/tests/charmhelpers/core/services/helpers.py b/tests/charmhelpers/core/services/helpers.py
new file mode 100644
index 0000000..3e6e30d
--- /dev/null
+++ b/tests/charmhelpers/core/services/helpers.py
@@ -0,0 +1,290 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import yaml
+
+from charmhelpers.core import hookenv
+from charmhelpers.core import host
+from charmhelpers.core import templating
+
+from charmhelpers.core.services.base import ManagerCallback
+
+
+__all__ = ['RelationContext', 'TemplateCallback',
+ 'render_template', 'template']
+
+
+class RelationContext(dict):
+ """
+ Base class for a context generator that gets relation data from juju.
+
+ Subclasses must provide the attributes `name`, which is the name of the
+ interface of interest, `interface`, which is the type of the interface of
+ interest, and `required_keys`, which is the set of keys required for the
+ relation to be considered complete. The data for all interfaces matching
+ the `name` attribute that are complete will used to populate the dictionary
+ values (see `get_data`, below).
+
+ The generated context will be namespaced under the relation :attr:`name`,
+ to prevent potential naming conflicts.
+
+ :param str name: Override the relation :attr:`name`, since it can vary from charm to charm
+ :param list additional_required_keys: Extend the list of :attr:`required_keys`
+ """
+ name = None
+ interface = None
+
+ def __init__(self, name=None, additional_required_keys=None):
+ if not hasattr(self, 'required_keys'):
+ self.required_keys = []
+
+ if name is not None:
+ self.name = name
+ if additional_required_keys:
+ self.required_keys.extend(additional_required_keys)
+ self.get_data()
+
+ def __bool__(self):
+ """
+ Returns True if all of the required_keys are available.
+ """
+ return self.is_ready()
+
+ __nonzero__ = __bool__
+
+ def __repr__(self):
+ return super(RelationContext, self).__repr__()
+
+ def is_ready(self):
+ """
+ Returns True if all of the `required_keys` are available from any units.
+ """
+ ready = len(self.get(self.name, [])) > 0
+ if not ready:
+ hookenv.log('Incomplete relation: {}'.format(self.__class__.__name__), hookenv.DEBUG)
+ return ready
+
+ def _is_ready(self, unit_data):
+ """
+ Helper method that tests a set of relation data and returns True if
+ all of the `required_keys` are present.
+ """
+ return set(unit_data.keys()).issuperset(set(self.required_keys))
+
+ def get_data(self):
+ """
+ Retrieve the relation data for each unit involved in a relation and,
+ if complete, store it in a list under `self[self.name]`. This
+ is automatically called when the RelationContext is instantiated.
+
+ The units are sorted lexographically first by the service ID, then by
+ the unit ID. Thus, if an interface has two other services, 'db:1'
+ and 'db:2', with 'db:1' having two units, 'wordpress/0' and 'wordpress/1',
+ and 'db:2' having one unit, 'mediawiki/0', all of which have a complete
+ set of data, the relation data for the units will be stored in the
+ order: 'wordpress/0', 'wordpress/1', 'mediawiki/0'.
+
+ If you only care about a single unit on the relation, you can just
+ access it as `{{ interface[0]['key'] }}`. However, if you can at all
+ support multiple units on a relation, you should iterate over the list,
+ like::
+
+ {% for unit in interface -%}
+ {{ unit['key'] }}{% if not loop.last %},{% endif %}
+ {%- endfor %}
+
+ Note that since all sets of relation data from all related services and
+ units are in a single list, if you need to know which service or unit a
+ set of data came from, you'll need to extend this class to preserve
+ that information.
+ """
+ if not hookenv.relation_ids(self.name):
+ return
+
+ ns = self.setdefault(self.name, [])
+ for rid in sorted(hookenv.relation_ids(self.name)):
+ for unit in sorted(hookenv.related_units(rid)):
+ reldata = hookenv.relation_get(rid=rid, unit=unit)
+ if self._is_ready(reldata):
+ ns.append(reldata)
+
+ def provide_data(self):
+ """
+ Return data to be relation_set for this interface.
+ """
+ return {}
+
+
+class MysqlRelation(RelationContext):
+ """
+ Relation context for the `mysql` interface.
+
+ :param str name: Override the relation :attr:`name`, since it can vary from charm to charm
+ :param list additional_required_keys: Extend the list of :attr:`required_keys`
+ """
+ name = 'db'
+ interface = 'mysql'
+
+ def __init__(self, *args, **kwargs):
+ self.required_keys = ['host', 'user', 'password', 'database']
+ RelationContext.__init__(self, *args, **kwargs)
+
+
+class HttpRelation(RelationContext):
+ """
+ Relation context for the `http` interface.
+
+ :param str name: Override the relation :attr:`name`, since it can vary from charm to charm
+ :param list additional_required_keys: Extend the list of :attr:`required_keys`
+ """
+ name = 'website'
+ interface = 'http'
+
+ def __init__(self, *args, **kwargs):
+ self.required_keys = ['host', 'port']
+ RelationContext.__init__(self, *args, **kwargs)
+
+ def provide_data(self):
+ return {
+ 'host': hookenv.unit_get('private-address'),
+ 'port': 80,
+ }
+
+
+class RequiredConfig(dict):
+ """
+ Data context that loads config options with one or more mandatory options.
+
+ Once the required options have been changed from their default values, all
+ config options will be available, namespaced under `config` to prevent
+ potential naming conflicts (for example, between a config option and a
+ relation property).
+
+ :param list *args: List of options that must be changed from their default values.
+ """
+
+ def __init__(self, *args):
+ self.required_options = args
+ self['config'] = hookenv.config()
+ with open(os.path.join(hookenv.charm_dir(), 'config.yaml')) as fp:
+ self.config = yaml.load(fp).get('options', {})
+
+ def __bool__(self):
+ for option in self.required_options:
+ if option not in self['config']:
+ return False
+ current_value = self['config'][option]
+ default_value = self.config[option].get('default')
+ if current_value == default_value:
+ return False
+ if current_value in (None, '') and default_value in (None, ''):
+ return False
+ return True
+
+ def __nonzero__(self):
+ return self.__bool__()
+
+
+class StoredContext(dict):
+ """
+ A data context that always returns the data that it was first created with.
+
+ This is useful to do a one-time generation of things like passwords, that
+ will thereafter use the same value that was originally generated, instead
+ of generating a new value each time it is run.
+ """
+ def __init__(self, file_name, config_data):
+ """
+ If the file exists, populate `self` with the data from the file.
+ Otherwise, populate with the given data and persist it to the file.
+ """
+ if os.path.exists(file_name):
+ self.update(self.read_context(file_name))
+ else:
+ self.store_context(file_name, config_data)
+ self.update(config_data)
+
+ def store_context(self, file_name, config_data):
+ if not os.path.isabs(file_name):
+ file_name = os.path.join(hookenv.charm_dir(), file_name)
+ with open(file_name, 'w') as file_stream:
+ os.fchmod(file_stream.fileno(), 0o600)
+ yaml.dump(config_data, file_stream)
+
+ def read_context(self, file_name):
+ if not os.path.isabs(file_name):
+ file_name = os.path.join(hookenv.charm_dir(), file_name)
+ with open(file_name, 'r') as file_stream:
+ data = yaml.load(file_stream)
+ if not data:
+ raise OSError("%s is empty" % file_name)
+ return data
+
+
+class TemplateCallback(ManagerCallback):
+ """
+ Callback class that will render a Jinja2 template, for use as a ready
+ action.
+
+ :param str source: The template source file, relative to
+ `$CHARM_DIR/templates`
+
+ :param str target: The target to write the rendered template to (or None)
+ :param str owner: The owner of the rendered file
+ :param str group: The group of the rendered file
+ :param int perms: The permissions of the rendered file
+ :param partial on_change_action: functools partial to be executed when
+ rendered file changes
+ :param jinja2 loader template_loader: A jinja2 template loader
+
+ :return str: The rendered template
+ """
+ def __init__(self, source, target,
+ owner='root', group='root', perms=0o444,
+ on_change_action=None, template_loader=None):
+ self.source = source
+ self.target = target
+ self.owner = owner
+ self.group = group
+ self.perms = perms
+ self.on_change_action = on_change_action
+ self.template_loader = template_loader
+
+ def __call__(self, manager, service_name, event_name):
+ pre_checksum = ''
+ if self.on_change_action and os.path.isfile(self.target):
+ pre_checksum = host.file_hash(self.target)
+ service = manager.get_service(service_name)
+ context = {'ctx': {}}
+ for ctx in service.get('required_data', []):
+ context.update(ctx)
+ context['ctx'].update(ctx)
+
+ result = templating.render(self.source, self.target, context,
+ self.owner, self.group, self.perms,
+ template_loader=self.template_loader)
+ if self.on_change_action:
+ if pre_checksum == host.file_hash(self.target):
+ hookenv.log(
+ 'No change detected: {}'.format(self.target),
+ hookenv.DEBUG)
+ else:
+ self.on_change_action()
+
+ return result
+
+
+# Convenience aliases for templates
+render_template = template = TemplateCallback
diff --git a/tests/charmhelpers/core/strutils.py b/tests/charmhelpers/core/strutils.py
new file mode 100644
index 0000000..e8df045
--- /dev/null
+++ b/tests/charmhelpers/core/strutils.py
@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import six
+import re
+
+
+def bool_from_string(value):
+ """Interpret string value as boolean.
+
+ Returns True if value translates to True otherwise False.
+ """
+ if isinstance(value, six.string_types):
+ value = six.text_type(value)
+ else:
+ msg = "Unable to interpret non-string value '%s' as boolean" % (value)
+ raise ValueError(msg)
+
+ value = value.strip().lower()
+
+ if value in ['y', 'yes', 'true', 't', 'on']:
+ return True
+ elif value in ['n', 'no', 'false', 'f', 'off']:
+ return False
+
+ msg = "Unable to interpret string value '%s' as boolean" % (value)
+ raise ValueError(msg)
+
+
+def bytes_from_string(value):
+ """Interpret human readable string value as bytes.
+
+ Returns int
+ """
+ BYTE_POWER = {
+ 'K': 1,
+ 'KB': 1,
+ 'M': 2,
+ 'MB': 2,
+ 'G': 3,
+ 'GB': 3,
+ 'T': 4,
+ 'TB': 4,
+ 'P': 5,
+ 'PB': 5,
+ }
+ if isinstance(value, six.string_types):
+ value = six.text_type(value)
+ else:
+ msg = "Unable to interpret non-string value '%s' as bytes" % (value)
+ raise ValueError(msg)
+ matches = re.match("([0-9]+)([a-zA-Z]+)", value)
+ if matches:
+ size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
+ else:
+ # Assume that value passed in is bytes
+ try:
+ size = int(value)
+ except ValueError:
+ msg = "Unable to interpret string value '%s' as bytes" % (value)
+ raise ValueError(msg)
+ return size
+
+
+class BasicStringComparator(object):
+ """Provides a class that will compare strings from an iterator type object.
+ Used to provide > and < comparisons on strings that may not necessarily be
+ alphanumerically ordered. e.g. OpenStack or Ubuntu releases AFTER the
+ z-wrap.
+ """
+
+ _list = None
+
+ def __init__(self, item):
+ if self._list is None:
+ raise Exception("Must define the _list in the class definition!")
+ try:
+ self.index = self._list.index(item)
+ except Exception:
+ raise KeyError("Item '{}' is not in list '{}'"
+ .format(item, self._list))
+
+ def __eq__(self, other):
+ assert isinstance(other, str) or isinstance(other, self.__class__)
+ return self.index == self._list.index(other)
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __lt__(self, other):
+ assert isinstance(other, str) or isinstance(other, self.__class__)
+ return self.index < self._list.index(other)
+
+ def __ge__(self, other):
+ return not self.__lt__(other)
+
+ def __gt__(self, other):
+ assert isinstance(other, str) or isinstance(other, self.__class__)
+ return self.index > self._list.index(other)
+
+ def __le__(self, other):
+ return not self.__gt__(other)
+
+ def __str__(self):
+ """Always give back the item at the index so it can be used in
+ comparisons like:
+
+ s_mitaka = CompareOpenStack('mitaka')
+ s_newton = CompareOpenstack('newton')
+
+ assert s_newton > s_mitaka
+
+ @returns:
+ """
+ return self._list[self.index]
diff --git a/tests/charmhelpers/core/sysctl.py b/tests/charmhelpers/core/sysctl.py
new file mode 100644
index 0000000..6e413e3
--- /dev/null
+++ b/tests/charmhelpers/core/sysctl.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import yaml
+
+from subprocess import check_call
+
+from charmhelpers.core.hookenv import (
+ log,
+ DEBUG,
+ ERROR,
+)
+
+__author__ = 'Jorge Niedbalski R. '
+
+
+def create(sysctl_dict, sysctl_file):
+ """Creates a sysctl.conf file from a YAML associative array
+
+ :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }"
+ :type sysctl_dict: str
+ :param sysctl_file: path to the sysctl file to be saved
+ :type sysctl_file: str or unicode
+ :returns: None
+ """
+ try:
+ sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
+ except yaml.YAMLError:
+ log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
+ level=ERROR)
+ return
+
+ with open(sysctl_file, "w") as fd:
+ for key, value in sysctl_dict_parsed.items():
+ fd.write("{}={}\n".format(key, value))
+
+ log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed),
+ level=DEBUG)
+
+ check_call(["sysctl", "-p", sysctl_file])
diff --git a/tests/charmhelpers/core/templating.py b/tests/charmhelpers/core/templating.py
new file mode 100644
index 0000000..7b801a3
--- /dev/null
+++ b/tests/charmhelpers/core/templating.py
@@ -0,0 +1,84 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+from charmhelpers.core import host
+from charmhelpers.core import hookenv
+
+
+def render(source, target, context, owner='root', group='root',
+ perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None):
+ """
+ Render a template.
+
+ The `source` path, if not absolute, is relative to the `templates_dir`.
+
+ The `target` path should be absolute. It can also be `None`, in which
+ case no file will be written.
+
+ The context should be a dict containing the values to be replaced in the
+ template.
+
+ The `owner`, `group`, and `perms` options will be passed to `write_file`.
+
+ If omitted, `templates_dir` defaults to the `templates` folder in the charm.
+
+ The rendered template will be written to the file as well as being returned
+ as a string.
+
+ Note: Using this requires python-jinja2 or python3-jinja2; if it is not
+ installed, calling this will attempt to use charmhelpers.fetch.apt_install
+ to install it.
+ """
+ try:
+ from jinja2 import FileSystemLoader, Environment, exceptions
+ except ImportError:
+ try:
+ from charmhelpers.fetch import apt_install
+ except ImportError:
+ hookenv.log('Could not import jinja2, and could not import '
+ 'charmhelpers.fetch to install it',
+ level=hookenv.ERROR)
+ raise
+ if sys.version_info.major == 2:
+ apt_install('python-jinja2', fatal=True)
+ else:
+ apt_install('python3-jinja2', fatal=True)
+ from jinja2 import FileSystemLoader, Environment, exceptions
+
+ if template_loader:
+ template_env = Environment(loader=template_loader)
+ else:
+ if templates_dir is None:
+ templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
+ template_env = Environment(loader=FileSystemLoader(templates_dir))
+ try:
+ source = source
+ template = template_env.get_template(source)
+ except exceptions.TemplateNotFound as e:
+ hookenv.log('Could not load template %s from %s.' %
+ (source, templates_dir),
+ level=hookenv.ERROR)
+ raise e
+ content = template.render(context)
+ if target is not None:
+ target_dir = os.path.dirname(target)
+ if not os.path.exists(target_dir):
+ # This is a terrible default directory permission, as the file
+ # or its siblings will often contain secrets.
+ host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
+ host.write_file(target, content.encode(encoding), owner, group, perms)
+ return content
diff --git a/tests/charmhelpers/core/unitdata.py b/tests/charmhelpers/core/unitdata.py
new file mode 100644
index 0000000..7af875c
--- /dev/null
+++ b/tests/charmhelpers/core/unitdata.py
@@ -0,0 +1,518 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# Kapil Thangavelu
+#
+"""
+Intro
+-----
+
+A simple way to store state in units. This provides a key value
+storage with support for versioned, transactional operation,
+and can calculate deltas from previous values to simplify unit logic
+when processing changes.
+
+
+Hook Integration
+----------------
+
+There are several extant frameworks for hook execution, including
+
+ - charmhelpers.core.hookenv.Hooks
+ - charmhelpers.core.services.ServiceManager
+
+The storage classes are framework agnostic, one simple integration is
+via the HookData contextmanager. It will record the current hook
+execution environment (including relation data, config data, etc.),
+setup a transaction and allow easy access to the changes from
+previously seen values. One consequence of the integration is the
+reservation of particular keys ('rels', 'unit', 'env', 'config',
+'charm_revisions') for their respective values.
+
+Here's a fully worked integration example using hookenv.Hooks::
+
+ from charmhelper.core import hookenv, unitdata
+
+ hook_data = unitdata.HookData()
+ db = unitdata.kv()
+ hooks = hookenv.Hooks()
+
+ @hooks.hook
+ def config_changed():
+ # Print all changes to configuration from previously seen
+ # values.
+ for changed, (prev, cur) in hook_data.conf.items():
+ print('config changed', changed,
+ 'previous value', prev,
+ 'current value', cur)
+
+ # Get some unit specific bookeeping
+ if not db.get('pkg_key'):
+ key = urllib.urlopen('https://example.com/pkg_key').read()
+ db.set('pkg_key', key)
+
+ # Directly access all charm config as a mapping.
+ conf = db.getrange('config', True)
+
+ # Directly access all relation data as a mapping
+ rels = db.getrange('rels', True)
+
+ if __name__ == '__main__':
+ with hook_data():
+ hook.execute()
+
+
+A more basic integration is via the hook_scope context manager which simply
+manages transaction scope (and records hook name, and timestamp)::
+
+ >>> from unitdata import kv
+ >>> db = kv()
+ >>> with db.hook_scope('install'):
+ ... # do work, in transactional scope.
+ ... db.set('x', 1)
+ >>> db.get('x')
+ 1
+
+
+Usage
+-----
+
+Values are automatically json de/serialized to preserve basic typing
+and complex data struct capabilities (dicts, lists, ints, booleans, etc).
+
+Individual values can be manipulated via get/set::
+
+ >>> kv.set('y', True)
+ >>> kv.get('y')
+ True
+
+ # We can set complex values (dicts, lists) as a single key.
+ >>> kv.set('config', {'a': 1, 'b': True'})
+
+ # Also supports returning dictionaries as a record which
+ # provides attribute access.
+ >>> config = kv.get('config', record=True)
+ >>> config.b
+ True
+
+
+Groups of keys can be manipulated with update/getrange::
+
+ >>> kv.update({'z': 1, 'y': 2}, prefix="gui.")
+ >>> kv.getrange('gui.', strip=True)
+ {'z': 1, 'y': 2}
+
+When updating values, its very helpful to understand which values
+have actually changed and how have they changed. The storage
+provides a delta method to provide for this::
+
+ >>> data = {'debug': True, 'option': 2}
+ >>> delta = kv.delta(data, 'config.')
+ >>> delta.debug.previous
+ None
+ >>> delta.debug.current
+ True
+ >>> delta
+ {'debug': (None, True), 'option': (None, 2)}
+
+Note the delta method does not persist the actual change, it needs to
+be explicitly saved via 'update' method::
+
+ >>> kv.update(data, 'config.')
+
+Values modified in the context of a hook scope retain historical values
+associated to the hookname.
+
+ >>> with db.hook_scope('config-changed'):
+ ... db.set('x', 42)
+ >>> db.gethistory('x')
+ [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'),
+ (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')]
+
+"""
+
+import collections
+import contextlib
+import datetime
+import itertools
+import json
+import os
+import pprint
+import sqlite3
+import sys
+
+__author__ = 'Kapil Thangavelu '
+
+
+class Storage(object):
+ """Simple key value database for local unit state within charms.
+
+ Modifications are not persisted unless :meth:`flush` is called.
+
+ To support dicts, lists, integer, floats, and booleans values
+ are automatically json encoded/decoded.
+ """
+ def __init__(self, path=None):
+ self.db_path = path
+ if path is None:
+ if 'UNIT_STATE_DB' in os.environ:
+ self.db_path = os.environ['UNIT_STATE_DB']
+ else:
+ self.db_path = os.path.join(
+ os.environ.get('CHARM_DIR', ''), '.unit-state.db')
+ self.conn = sqlite3.connect('%s' % self.db_path)
+ self.cursor = self.conn.cursor()
+ self.revision = None
+ self._closed = False
+ self._init()
+
+ def close(self):
+ if self._closed:
+ return
+ self.flush(False)
+ self.cursor.close()
+ self.conn.close()
+ self._closed = True
+
+ def get(self, key, default=None, record=False):
+ self.cursor.execute('select data from kv where key=?', [key])
+ result = self.cursor.fetchone()
+ if not result:
+ return default
+ if record:
+ return Record(json.loads(result[0]))
+ return json.loads(result[0])
+
+ def getrange(self, key_prefix, strip=False):
+ """
+ Get a range of keys starting with a common prefix as a mapping of
+ keys to values.
+
+ :param str key_prefix: Common prefix among all keys
+ :param bool strip: Optionally strip the common prefix from the key
+ names in the returned dict
+ :return dict: A (possibly empty) dict of key-value mappings
+ """
+ self.cursor.execute("select key, data from kv where key like ?",
+ ['%s%%' % key_prefix])
+ result = self.cursor.fetchall()
+
+ if not result:
+ return {}
+ if not strip:
+ key_prefix = ''
+ return dict([
+ (k[len(key_prefix):], json.loads(v)) for k, v in result])
+
+ def update(self, mapping, prefix=""):
+ """
+ Set the values of multiple keys at once.
+
+ :param dict mapping: Mapping of keys to values
+ :param str prefix: Optional prefix to apply to all keys in `mapping`
+ before setting
+ """
+ for k, v in mapping.items():
+ self.set("%s%s" % (prefix, k), v)
+
+ def unset(self, key):
+ """
+ Remove a key from the database entirely.
+ """
+ self.cursor.execute('delete from kv where key=?', [key])
+ if self.revision and self.cursor.rowcount:
+ self.cursor.execute(
+ 'insert into kv_revisions values (?, ?, ?)',
+ [key, self.revision, json.dumps('DELETED')])
+
+ def unsetrange(self, keys=None, prefix=""):
+ """
+ Remove a range of keys starting with a common prefix, from the database
+ entirely.
+
+ :param list keys: List of keys to remove.
+ :param str prefix: Optional prefix to apply to all keys in ``keys``
+ before removing.
+ """
+ if keys is not None:
+ keys = ['%s%s' % (prefix, key) for key in keys]
+ self.cursor.execute('delete from kv where key in (%s)' % ','.join(['?'] * len(keys)), keys)
+ if self.revision and self.cursor.rowcount:
+ self.cursor.execute(
+ 'insert into kv_revisions values %s' % ','.join(['(?, ?, ?)'] * len(keys)),
+ list(itertools.chain.from_iterable((key, self.revision, json.dumps('DELETED')) for key in keys)))
+ else:
+ self.cursor.execute('delete from kv where key like ?',
+ ['%s%%' % prefix])
+ if self.revision and self.cursor.rowcount:
+ self.cursor.execute(
+ 'insert into kv_revisions values (?, ?, ?)',
+ ['%s%%' % prefix, self.revision, json.dumps('DELETED')])
+
+ def set(self, key, value):
+ """
+ Set a value in the database.
+
+ :param str key: Key to set the value for
+ :param value: Any JSON-serializable value to be set
+ """
+ serialized = json.dumps(value)
+
+ self.cursor.execute('select data from kv where key=?', [key])
+ exists = self.cursor.fetchone()
+
+ # Skip mutations to the same value
+ if exists:
+ if exists[0] == serialized:
+ return value
+
+ if not exists:
+ self.cursor.execute(
+ 'insert into kv (key, data) values (?, ?)',
+ (key, serialized))
+ else:
+ self.cursor.execute('''
+ update kv
+ set data = ?
+ where key = ?''', [serialized, key])
+
+ # Save
+ if not self.revision:
+ return value
+
+ self.cursor.execute(
+ 'select 1 from kv_revisions where key=? and revision=?',
+ [key, self.revision])
+ exists = self.cursor.fetchone()
+
+ if not exists:
+ self.cursor.execute(
+ '''insert into kv_revisions (
+ revision, key, data) values (?, ?, ?)''',
+ (self.revision, key, serialized))
+ else:
+ self.cursor.execute(
+ '''
+ update kv_revisions
+ set data = ?
+ where key = ?
+ and revision = ?''',
+ [serialized, key, self.revision])
+
+ return value
+
+ def delta(self, mapping, prefix):
+ """
+ return a delta containing values that have changed.
+ """
+ previous = self.getrange(prefix, strip=True)
+ if not previous:
+ pk = set()
+ else:
+ pk = set(previous.keys())
+ ck = set(mapping.keys())
+ delta = DeltaSet()
+
+ # added
+ for k in ck.difference(pk):
+ delta[k] = Delta(None, mapping[k])
+
+ # removed
+ for k in pk.difference(ck):
+ delta[k] = Delta(previous[k], None)
+
+ # changed
+ for k in pk.intersection(ck):
+ c = mapping[k]
+ p = previous[k]
+ if c != p:
+ delta[k] = Delta(p, c)
+
+ return delta
+
+ @contextlib.contextmanager
+ def hook_scope(self, name=""):
+ """Scope all future interactions to the current hook execution
+ revision."""
+ assert not self.revision
+ self.cursor.execute(
+ 'insert into hooks (hook, date) values (?, ?)',
+ (name or sys.argv[0],
+ datetime.datetime.utcnow().isoformat()))
+ self.revision = self.cursor.lastrowid
+ try:
+ yield self.revision
+ self.revision = None
+ except Exception:
+ self.flush(False)
+ self.revision = None
+ raise
+ else:
+ self.flush()
+
+ def flush(self, save=True):
+ if save:
+ self.conn.commit()
+ elif self._closed:
+ return
+ else:
+ self.conn.rollback()
+
+ def _init(self):
+ self.cursor.execute('''
+ create table if not exists kv (
+ key text,
+ data text,
+ primary key (key)
+ )''')
+ self.cursor.execute('''
+ create table if not exists kv_revisions (
+ key text,
+ revision integer,
+ data text,
+ primary key (key, revision)
+ )''')
+ self.cursor.execute('''
+ create table if not exists hooks (
+ version integer primary key autoincrement,
+ hook text,
+ date text
+ )''')
+ self.conn.commit()
+
+ def gethistory(self, key, deserialize=False):
+ self.cursor.execute(
+ '''
+ select kv.revision, kv.key, kv.data, h.hook, h.date
+ from kv_revisions kv,
+ hooks h
+ where kv.key=?
+ and kv.revision = h.version
+ ''', [key])
+ if deserialize is False:
+ return self.cursor.fetchall()
+ return map(_parse_history, self.cursor.fetchall())
+
+ def debug(self, fh=sys.stderr):
+ self.cursor.execute('select * from kv')
+ pprint.pprint(self.cursor.fetchall(), stream=fh)
+ self.cursor.execute('select * from kv_revisions')
+ pprint.pprint(self.cursor.fetchall(), stream=fh)
+
+
+def _parse_history(d):
+ return (d[0], d[1], json.loads(d[2]), d[3],
+ datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f"))
+
+
+class HookData(object):
+ """Simple integration for existing hook exec frameworks.
+
+ Records all unit information, and stores deltas for processing
+ by the hook.
+
+ Sample::
+
+ from charmhelper.core import hookenv, unitdata
+
+ changes = unitdata.HookData()
+ db = unitdata.kv()
+ hooks = hookenv.Hooks()
+
+ @hooks.hook
+ def config_changed():
+ # View all changes to configuration
+ for changed, (prev, cur) in changes.conf.items():
+ print('config changed', changed,
+ 'previous value', prev,
+ 'current value', cur)
+
+ # Get some unit specific bookeeping
+ if not db.get('pkg_key'):
+ key = urllib.urlopen('https://example.com/pkg_key').read()
+ db.set('pkg_key', key)
+
+ if __name__ == '__main__':
+ with changes():
+ hook.execute()
+
+ """
+ def __init__(self):
+ self.kv = kv()
+ self.conf = None
+ self.rels = None
+
+ @contextlib.contextmanager
+ def __call__(self):
+ from charmhelpers.core import hookenv
+ hook_name = hookenv.hook_name()
+
+ with self.kv.hook_scope(hook_name):
+ self._record_charm_version(hookenv.charm_dir())
+ delta_config, delta_relation = self._record_hook(hookenv)
+ yield self.kv, delta_config, delta_relation
+
+ def _record_charm_version(self, charm_dir):
+ # Record revisions.. charm revisions are meaningless
+ # to charm authors as they don't control the revision.
+ # so logic dependnent on revision is not particularly
+ # useful, however it is useful for debugging analysis.
+ charm_rev = open(
+ os.path.join(charm_dir, 'revision')).read().strip()
+ charm_rev = charm_rev or '0'
+ revs = self.kv.get('charm_revisions', [])
+ if charm_rev not in revs:
+ revs.append(charm_rev.strip() or '0')
+ self.kv.set('charm_revisions', revs)
+
+ def _record_hook(self, hookenv):
+ data = hookenv.execution_environment()
+ self.conf = conf_delta = self.kv.delta(data['conf'], 'config')
+ self.rels = rels_delta = self.kv.delta(data['rels'], 'rels')
+ self.kv.set('env', dict(data['env']))
+ self.kv.set('unit', data['unit'])
+ self.kv.set('relid', data.get('relid'))
+ return conf_delta, rels_delta
+
+
+class Record(dict):
+
+ __slots__ = ()
+
+ def __getattr__(self, k):
+ if k in self:
+ return self[k]
+ raise AttributeError(k)
+
+
+class DeltaSet(Record):
+
+ __slots__ = ()
+
+
+Delta = collections.namedtuple('Delta', ['previous', 'current'])
+
+
+_KV = None
+
+
+def kv():
+ global _KV
+ if _KV is None:
+ _KV = Storage()
+ return _KV
diff --git a/tests/charmhelpers/fetch/__init__.py b/tests/charmhelpers/fetch/__init__.py
new file mode 100644
index 0000000..480a627
--- /dev/null
+++ b/tests/charmhelpers/fetch/__init__.py
@@ -0,0 +1,205 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+from charmhelpers.osplatform import get_platform
+from yaml import safe_load
+from charmhelpers.core.hookenv import (
+ config,
+ log,
+)
+
+import six
+if six.PY3:
+ from urllib.parse import urlparse, urlunparse
+else:
+ from urlparse import urlparse, urlunparse
+
+
+# The order of this list is very important. Handlers should be listed in from
+# least- to most-specific URL matching.
+FETCH_HANDLERS = (
+ 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
+ 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
+ 'charmhelpers.fetch.giturl.GitUrlFetchHandler',
+)
+
+
+class SourceConfigError(Exception):
+ pass
+
+
+class UnhandledSource(Exception):
+ pass
+
+
+class AptLockError(Exception):
+ pass
+
+
+class GPGKeyError(Exception):
+ """Exception occurs when a GPG key cannot be fetched or used. The message
+ indicates what the problem is.
+ """
+ pass
+
+
+class BaseFetchHandler(object):
+
+ """Base class for FetchHandler implementations in fetch plugins"""
+
+ def can_handle(self, source):
+ """Returns True if the source can be handled. Otherwise returns
+ a string explaining why it cannot"""
+ return "Wrong source type"
+
+ def install(self, source):
+ """Try to download and unpack the source. Return the path to the
+ unpacked files or raise UnhandledSource."""
+ raise UnhandledSource("Wrong source type {}".format(source))
+
+ def parse_url(self, url):
+ return urlparse(url)
+
+ def base_url(self, url):
+ """Return url without querystring or fragment"""
+ parts = list(self.parse_url(url))
+ parts[4:] = ['' for i in parts[4:]]
+ return urlunparse(parts)
+
+
+__platform__ = get_platform()
+module = "charmhelpers.fetch.%s" % __platform__
+fetch = importlib.import_module(module)
+
+filter_installed_packages = fetch.filter_installed_packages
+install = fetch.apt_install
+upgrade = fetch.apt_upgrade
+update = _fetch_update = fetch.apt_update
+purge = fetch.apt_purge
+add_source = fetch.add_source
+
+if __platform__ == "ubuntu":
+ apt_cache = fetch.apt_cache
+ apt_install = fetch.apt_install
+ apt_update = fetch.apt_update
+ apt_upgrade = fetch.apt_upgrade
+ apt_purge = fetch.apt_purge
+ apt_mark = fetch.apt_mark
+ apt_hold = fetch.apt_hold
+ apt_unhold = fetch.apt_unhold
+ import_key = fetch.import_key
+ get_upstream_version = fetch.get_upstream_version
+elif __platform__ == "centos":
+ yum_search = fetch.yum_search
+
+
+def configure_sources(update=False,
+ sources_var='install_sources',
+ keys_var='install_keys'):
+ """Configure multiple sources from charm configuration.
+
+ The lists are encoded as yaml fragments in the configuration.
+ The fragment needs to be included as a string. Sources and their
+ corresponding keys are of the types supported by add_source().
+
+ Example config:
+ install_sources: |
+ - "ppa:foo"
+ - "http://example.com/repo precise main"
+ install_keys: |
+ - null
+ - "a1b2c3d4"
+
+ Note that 'null' (a.k.a. None) should not be quoted.
+ """
+ sources = safe_load((config(sources_var) or '').strip()) or []
+ keys = safe_load((config(keys_var) or '').strip()) or None
+
+ if isinstance(sources, six.string_types):
+ sources = [sources]
+
+ if keys is None:
+ for source in sources:
+ add_source(source, None)
+ else:
+ if isinstance(keys, six.string_types):
+ keys = [keys]
+
+ if len(sources) != len(keys):
+ raise SourceConfigError(
+ 'Install sources and keys lists are different lengths')
+ for source, key in zip(sources, keys):
+ add_source(source, key)
+ if update:
+ _fetch_update(fatal=True)
+
+
+def install_remote(source, *args, **kwargs):
+ """Install a file tree from a remote source.
+
+ The specified source should be a url of the form:
+ scheme://[host]/path[#[option=value][&...]]
+
+ Schemes supported are based on this modules submodules.
+ Options supported are submodule-specific.
+ Additional arguments are passed through to the submodule.
+
+ For example::
+
+ dest = install_remote('http://example.com/archive.tgz',
+ checksum='deadbeef',
+ hash_type='sha1')
+
+ This will download `archive.tgz`, validate it using SHA1 and, if
+ the file is ok, extract it and return the directory in which it
+ was extracted. If the checksum fails, it will raise
+ :class:`charmhelpers.core.host.ChecksumError`.
+ """
+ # We ONLY check for True here because can_handle may return a string
+ # explaining why it can't handle a given source.
+ handlers = [h for h in plugins() if h.can_handle(source) is True]
+ for handler in handlers:
+ try:
+ return handler.install(source, *args, **kwargs)
+ except UnhandledSource as e:
+ log('Install source attempt unsuccessful: {}'.format(e),
+ level='WARNING')
+ raise UnhandledSource("No handler found for source {}".format(source))
+
+
+def install_from_config(config_var_name):
+ """Install a file from config."""
+ charm_config = config()
+ source = charm_config[config_var_name]
+ return install_remote(source)
+
+
+def plugins(fetch_handlers=None):
+ if not fetch_handlers:
+ fetch_handlers = FETCH_HANDLERS
+ plugin_list = []
+ for handler_name in fetch_handlers:
+ package, classname = handler_name.rsplit('.', 1)
+ try:
+ handler_class = getattr(
+ importlib.import_module(package),
+ classname)
+ plugin_list.append(handler_class())
+ except NotImplementedError:
+ # Skip missing plugins so that they can be ommitted from
+ # installation if desired
+ log("FetchHandler {} not found, skipping plugin".format(
+ handler_name))
+ return plugin_list
diff --git a/tests/charmhelpers/fetch/archiveurl.py b/tests/charmhelpers/fetch/archiveurl.py
new file mode 100644
index 0000000..dd24f9e
--- /dev/null
+++ b/tests/charmhelpers/fetch/archiveurl.py
@@ -0,0 +1,165 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import hashlib
+import re
+
+from charmhelpers.fetch import (
+ BaseFetchHandler,
+ UnhandledSource
+)
+from charmhelpers.payload.archive import (
+ get_archive_handler,
+ extract,
+)
+from charmhelpers.core.host import mkdir, check_hash
+
+import six
+if six.PY3:
+ from urllib.request import (
+ build_opener, install_opener, urlopen, urlretrieve,
+ HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
+ )
+ from urllib.parse import urlparse, urlunparse, parse_qs
+ from urllib.error import URLError
+else:
+ from urllib import urlretrieve
+ from urllib2 import (
+ build_opener, install_opener, urlopen,
+ HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
+ URLError
+ )
+ from urlparse import urlparse, urlunparse, parse_qs
+
+
+def splituser(host):
+ '''urllib.splituser(), but six's support of this seems broken'''
+ _userprog = re.compile('^(.*)@(.*)$')
+ match = _userprog.match(host)
+ if match:
+ return match.group(1, 2)
+ return None, host
+
+
+def splitpasswd(user):
+ '''urllib.splitpasswd(), but six's support of this is missing'''
+ _passwdprog = re.compile('^([^:]*):(.*)$', re.S)
+ match = _passwdprog.match(user)
+ if match:
+ return match.group(1, 2)
+ return user, None
+
+
+class ArchiveUrlFetchHandler(BaseFetchHandler):
+ """
+ Handler to download archive files from arbitrary URLs.
+
+ Can fetch from http, https, ftp, and file URLs.
+
+ Can install either tarballs (.tar, .tgz, .tbz2, etc) or zip files.
+
+ Installs the contents of the archive in $CHARM_DIR/fetched/.
+ """
+ def can_handle(self, source):
+ url_parts = self.parse_url(source)
+ if url_parts.scheme not in ('http', 'https', 'ftp', 'file'):
+ # XXX: Why is this returning a boolean and a string? It's
+ # doomed to fail since "bool(can_handle('foo://'))" will be True.
+ return "Wrong source type"
+ if get_archive_handler(self.base_url(source)):
+ return True
+ return False
+
+ def download(self, source, dest):
+ """
+ Download an archive file.
+
+ :param str source: URL pointing to an archive file.
+ :param str dest: Local path location to download archive file to.
+ """
+ # propogate all exceptions
+ # URLError, OSError, etc
+ proto, netloc, path, params, query, fragment = urlparse(source)
+ if proto in ('http', 'https'):
+ auth, barehost = splituser(netloc)
+ if auth is not None:
+ source = urlunparse((proto, barehost, path, params, query, fragment))
+ username, password = splitpasswd(auth)
+ passman = HTTPPasswordMgrWithDefaultRealm()
+ # Realm is set to None in add_password to force the username and password
+ # to be used whatever the realm
+ passman.add_password(None, source, username, password)
+ authhandler = HTTPBasicAuthHandler(passman)
+ opener = build_opener(authhandler)
+ install_opener(opener)
+ response = urlopen(source)
+ try:
+ with open(dest, 'wb') as dest_file:
+ dest_file.write(response.read())
+ except Exception as e:
+ if os.path.isfile(dest):
+ os.unlink(dest)
+ raise e
+
+ # Mandatory file validation via Sha1 or MD5 hashing.
+ def download_and_validate(self, url, hashsum, validate="sha1"):
+ tempfile, headers = urlretrieve(url)
+ check_hash(tempfile, hashsum, validate)
+ return tempfile
+
+ def install(self, source, dest=None, checksum=None, hash_type='sha1'):
+ """
+ Download and install an archive file, with optional checksum validation.
+
+ The checksum can also be given on the `source` URL's fragment.
+ For example::
+
+ handler.install('http://example.com/file.tgz#sha1=deadbeef')
+
+ :param str source: URL pointing to an archive file.
+ :param str dest: Local destination path to install to. If not given,
+ installs to `$CHARM_DIR/archives/archive_file_name`.
+ :param str checksum: If given, validate the archive file after download.
+ :param str hash_type: Algorithm used to generate `checksum`.
+ Can be any hash alrgorithm supported by :mod:`hashlib`,
+ such as md5, sha1, sha256, sha512, etc.
+
+ """
+ url_parts = self.parse_url(source)
+ dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
+ if not os.path.exists(dest_dir):
+ mkdir(dest_dir, perms=0o755)
+ dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path))
+ try:
+ self.download(source, dld_file)
+ except URLError as e:
+ raise UnhandledSource(e.reason)
+ except OSError as e:
+ raise UnhandledSource(e.strerror)
+ options = parse_qs(url_parts.fragment)
+ for key, value in options.items():
+ if not six.PY3:
+ algorithms = hashlib.algorithms
+ else:
+ algorithms = hashlib.algorithms_available
+ if key in algorithms:
+ if len(value) != 1:
+ raise TypeError(
+ "Expected 1 hash value, not %d" % len(value))
+ expected = value[0]
+ check_hash(dld_file, expected, key)
+ if checksum:
+ check_hash(dld_file, checksum, hash_type)
+ return extract(dld_file, dest)
diff --git a/tests/charmhelpers/fetch/bzrurl.py b/tests/charmhelpers/fetch/bzrurl.py
new file mode 100644
index 0000000..07cd029
--- /dev/null
+++ b/tests/charmhelpers/fetch/bzrurl.py
@@ -0,0 +1,76 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from subprocess import check_call
+from charmhelpers.fetch import (
+ BaseFetchHandler,
+ UnhandledSource,
+ filter_installed_packages,
+ install,
+)
+from charmhelpers.core.host import mkdir
+
+
+if filter_installed_packages(['bzr']) != []:
+ install(['bzr'])
+ if filter_installed_packages(['bzr']) != []:
+ raise NotImplementedError('Unable to install bzr')
+
+
+class BzrUrlFetchHandler(BaseFetchHandler):
+ """Handler for bazaar branches via generic and lp URLs."""
+
+ def can_handle(self, source):
+ url_parts = self.parse_url(source)
+ if url_parts.scheme not in ('bzr+ssh', 'lp', ''):
+ return False
+ elif not url_parts.scheme:
+ return os.path.exists(os.path.join(source, '.bzr'))
+ else:
+ return True
+
+ def branch(self, source, dest, revno=None):
+ if not self.can_handle(source):
+ raise UnhandledSource("Cannot handle {}".format(source))
+ cmd_opts = []
+ if revno:
+ cmd_opts += ['-r', str(revno)]
+ if os.path.exists(dest):
+ cmd = ['bzr', 'pull']
+ cmd += cmd_opts
+ cmd += ['--overwrite', '-d', dest, source]
+ else:
+ cmd = ['bzr', 'branch']
+ cmd += cmd_opts
+ cmd += [source, dest]
+ check_call(cmd)
+
+ def install(self, source, dest=None, revno=None):
+ url_parts = self.parse_url(source)
+ branch_name = url_parts.path.strip("/").split("/")[-1]
+ if dest:
+ dest_dir = os.path.join(dest, branch_name)
+ else:
+ dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
+ branch_name)
+
+ if dest and not os.path.exists(dest):
+ mkdir(dest, perms=0o755)
+
+ try:
+ self.branch(source, dest_dir, revno)
+ except OSError as e:
+ raise UnhandledSource(e.strerror)
+ return dest_dir
diff --git a/tests/charmhelpers/fetch/centos.py b/tests/charmhelpers/fetch/centos.py
new file mode 100644
index 0000000..a91dcff
--- /dev/null
+++ b/tests/charmhelpers/fetch/centos.py
@@ -0,0 +1,171 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+import os
+import time
+import six
+import yum
+
+from tempfile import NamedTemporaryFile
+from charmhelpers.core.hookenv import log
+
+YUM_NO_LOCK = 1 # The return code for "couldn't acquire lock" in YUM.
+YUM_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks.
+YUM_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
+
+
+def filter_installed_packages(packages):
+ """Return a list of packages that require installation."""
+ yb = yum.YumBase()
+ package_list = yb.doPackageLists()
+ temp_cache = {p.base_package_name: 1 for p in package_list['installed']}
+
+ _pkgs = [p for p in packages if not temp_cache.get(p, False)]
+ return _pkgs
+
+
+def install(packages, options=None, fatal=False):
+ """Install one or more packages."""
+ cmd = ['yum', '--assumeyes']
+ if options is not None:
+ cmd.extend(options)
+ cmd.append('install')
+ if isinstance(packages, six.string_types):
+ cmd.append(packages)
+ else:
+ cmd.extend(packages)
+ log("Installing {} with options: {}".format(packages,
+ options))
+ _run_yum_command(cmd, fatal)
+
+
+def upgrade(options=None, fatal=False, dist=False):
+ """Upgrade all packages."""
+ cmd = ['yum', '--assumeyes']
+ if options is not None:
+ cmd.extend(options)
+ cmd.append('upgrade')
+ log("Upgrading with options: {}".format(options))
+ _run_yum_command(cmd, fatal)
+
+
+def update(fatal=False):
+ """Update local yum cache."""
+ cmd = ['yum', '--assumeyes', 'update']
+ log("Update with fatal: {}".format(fatal))
+ _run_yum_command(cmd, fatal)
+
+
+def purge(packages, fatal=False):
+ """Purge one or more packages."""
+ cmd = ['yum', '--assumeyes', 'remove']
+ if isinstance(packages, six.string_types):
+ cmd.append(packages)
+ else:
+ cmd.extend(packages)
+ log("Purging {}".format(packages))
+ _run_yum_command(cmd, fatal)
+
+
+def yum_search(packages):
+ """Search for a package."""
+ output = {}
+ cmd = ['yum', 'search']
+ if isinstance(packages, six.string_types):
+ cmd.append(packages)
+ else:
+ cmd.extend(packages)
+ log("Searching for {}".format(packages))
+ result = subprocess.check_output(cmd)
+ for package in list(packages):
+ output[package] = package in result
+ return output
+
+
+def add_source(source, key=None):
+ """Add a package source to this system.
+
+ @param source: a URL with a rpm package
+
+ @param key: A key to be added to the system's keyring and used
+ to verify the signatures on packages. Ideally, this should be an
+ ASCII format GPG public key including the block headers. A GPG key
+ id may also be used, but be aware that only insecure protocols are
+ available to retrieve the actual public key from a public keyserver
+ placing your Juju environment at risk.
+ """
+ if source is None:
+ log('Source is not present. Skipping')
+ return
+
+ if source.startswith('http'):
+ directory = '/etc/yum.repos.d/'
+ for filename in os.listdir(directory):
+ with open(directory + filename, 'r') as rpm_file:
+ if source in rpm_file.read():
+ break
+ else:
+ log("Add source: {!r}".format(source))
+ # write in the charms.repo
+ with open(directory + 'Charms.repo', 'a') as rpm_file:
+ rpm_file.write('[%s]\n' % source[7:].replace('/', '_'))
+ rpm_file.write('name=%s\n' % source[7:])
+ rpm_file.write('baseurl=%s\n\n' % source)
+ else:
+ log("Unknown source: {!r}".format(source))
+
+ if key:
+ if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
+ with NamedTemporaryFile('w+') as key_file:
+ key_file.write(key)
+ key_file.flush()
+ key_file.seek(0)
+ subprocess.check_call(['rpm', '--import', key_file.name])
+ else:
+ subprocess.check_call(['rpm', '--import', key])
+
+
+def _run_yum_command(cmd, fatal=False):
+ """Run an YUM command.
+
+ Checks the output and retry if the fatal flag is set to True.
+
+ :param: cmd: str: The yum command to run.
+ :param: fatal: bool: Whether the command's output should be checked and
+ retried.
+ """
+ env = os.environ.copy()
+
+ if fatal:
+ retry_count = 0
+ result = None
+
+ # If the command is considered "fatal", we need to retry if the yum
+ # lock was not acquired.
+
+ while result is None or result == YUM_NO_LOCK:
+ try:
+ result = subprocess.check_call(cmd, env=env)
+ except subprocess.CalledProcessError as e:
+ retry_count = retry_count + 1
+ if retry_count > YUM_NO_LOCK_RETRY_COUNT:
+ raise
+ result = e.returncode
+ log("Couldn't acquire YUM lock. Will retry in {} seconds."
+ "".format(YUM_NO_LOCK_RETRY_DELAY))
+ time.sleep(YUM_NO_LOCK_RETRY_DELAY)
+
+ else:
+ subprocess.call(cmd, env=env)
diff --git a/tests/charmhelpers/fetch/giturl.py b/tests/charmhelpers/fetch/giturl.py
new file mode 100644
index 0000000..4cf21bc
--- /dev/null
+++ b/tests/charmhelpers/fetch/giturl.py
@@ -0,0 +1,69 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from subprocess import check_call, CalledProcessError
+from charmhelpers.fetch import (
+ BaseFetchHandler,
+ UnhandledSource,
+ filter_installed_packages,
+ install,
+)
+
+if filter_installed_packages(['git']) != []:
+ install(['git'])
+ if filter_installed_packages(['git']) != []:
+ raise NotImplementedError('Unable to install git')
+
+
+class GitUrlFetchHandler(BaseFetchHandler):
+ """Handler for git branches via generic and github URLs."""
+
+ def can_handle(self, source):
+ url_parts = self.parse_url(source)
+ # TODO (mattyw) no support for ssh git@ yet
+ if url_parts.scheme not in ('http', 'https', 'git', ''):
+ return False
+ elif not url_parts.scheme:
+ return os.path.exists(os.path.join(source, '.git'))
+ else:
+ return True
+
+ def clone(self, source, dest, branch="master", depth=None):
+ if not self.can_handle(source):
+ raise UnhandledSource("Cannot handle {}".format(source))
+
+ if os.path.exists(dest):
+ cmd = ['git', '-C', dest, 'pull', source, branch]
+ else:
+ cmd = ['git', 'clone', source, dest, '--branch', branch]
+ if depth:
+ cmd.extend(['--depth', depth])
+ check_call(cmd)
+
+ def install(self, source, branch="master", dest=None, depth=None):
+ url_parts = self.parse_url(source)
+ branch_name = url_parts.path.strip("/").split("/")[-1]
+ if dest:
+ dest_dir = os.path.join(dest, branch_name)
+ else:
+ dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
+ branch_name)
+ try:
+ self.clone(source, dest_dir, branch, depth)
+ except CalledProcessError as e:
+ raise UnhandledSource(e)
+ except OSError as e:
+ raise UnhandledSource(e.strerror)
+ return dest_dir
diff --git a/tests/charmhelpers/fetch/snap.py b/tests/charmhelpers/fetch/snap.py
new file mode 100644
index 0000000..395836c
--- /dev/null
+++ b/tests/charmhelpers/fetch/snap.py
@@ -0,0 +1,150 @@
+# Copyright 2014-2017 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Charm helpers snap for classic charms.
+
+If writing reactive charms, use the snap layer:
+https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html
+"""
+import subprocess
+import os
+from time import sleep
+from charmhelpers.core.hookenv import log
+
+__author__ = 'Joseph Borg '
+
+# The return code for "couldn't acquire lock" in Snap
+# (hopefully this will be improved).
+SNAP_NO_LOCK = 1
+SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks.
+SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
+SNAP_CHANNELS = [
+ 'edge',
+ 'beta',
+ 'candidate',
+ 'stable',
+]
+
+
+class CouldNotAcquireLockException(Exception):
+ pass
+
+
+class InvalidSnapChannel(Exception):
+ pass
+
+
+def _snap_exec(commands):
+ """
+ Execute snap commands.
+
+ :param commands: List commands
+ :return: Integer exit code
+ """
+ assert type(commands) == list
+
+ retry_count = 0
+ return_code = None
+
+ while return_code is None or return_code == SNAP_NO_LOCK:
+ try:
+ return_code = subprocess.check_call(['snap'] + commands,
+ env=os.environ)
+ except subprocess.CalledProcessError as e:
+ retry_count += + 1
+ if retry_count > SNAP_NO_LOCK_RETRY_COUNT:
+ raise CouldNotAcquireLockException(
+ 'Could not aquire lock after {} attempts'
+ .format(SNAP_NO_LOCK_RETRY_COUNT))
+ return_code = e.returncode
+ log('Snap failed to acquire lock, trying again in {} seconds.'
+ .format(SNAP_NO_LOCK_RETRY_DELAY, level='WARN'))
+ sleep(SNAP_NO_LOCK_RETRY_DELAY)
+
+ return return_code
+
+
+def snap_install(packages, *flags):
+ """
+ Install a snap package.
+
+ :param packages: String or List String package name
+ :param flags: List String flags to pass to install command
+ :return: Integer return code from snap
+ """
+ if type(packages) is not list:
+ packages = [packages]
+
+ flags = list(flags)
+
+ message = 'Installing snap(s) "%s"' % ', '.join(packages)
+ if flags:
+ message += ' with option(s) "%s"' % ', '.join(flags)
+
+ log(message, level='INFO')
+ return _snap_exec(['install'] + flags + packages)
+
+
+def snap_remove(packages, *flags):
+ """
+ Remove a snap package.
+
+ :param packages: String or List String package name
+ :param flags: List String flags to pass to remove command
+ :return: Integer return code from snap
+ """
+ if type(packages) is not list:
+ packages = [packages]
+
+ flags = list(flags)
+
+ message = 'Removing snap(s) "%s"' % ', '.join(packages)
+ if flags:
+ message += ' with options "%s"' % ', '.join(flags)
+
+ log(message, level='INFO')
+ return _snap_exec(['remove'] + flags + packages)
+
+
+def snap_refresh(packages, *flags):
+ """
+ Refresh / Update snap package.
+
+ :param packages: String or List String package name
+ :param flags: List String flags to pass to refresh command
+ :return: Integer return code from snap
+ """
+ if type(packages) is not list:
+ packages = [packages]
+
+ flags = list(flags)
+
+ message = 'Refreshing snap(s) "%s"' % ', '.join(packages)
+ if flags:
+ message += ' with options "%s"' % ', '.join(flags)
+
+ log(message, level='INFO')
+ return _snap_exec(['refresh'] + flags + packages)
+
+
+def valid_snap_channel(channel):
+ """ Validate snap channel exists
+
+ :raises InvalidSnapChannel: When channel does not exist
+ :return: Boolean
+ """
+ if channel.lower() in SNAP_CHANNELS:
+ return True
+ else:
+ raise InvalidSnapChannel("Invalid Snap Channel: {}".format(channel))
diff --git a/tests/charmhelpers/fetch/ubuntu.py b/tests/charmhelpers/fetch/ubuntu.py
new file mode 100644
index 0000000..910e96a
--- /dev/null
+++ b/tests/charmhelpers/fetch/ubuntu.py
@@ -0,0 +1,583 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import OrderedDict
+import os
+import platform
+import re
+import six
+import time
+import subprocess
+from tempfile import NamedTemporaryFile
+
+from charmhelpers.core.host import (
+ lsb_release
+)
+from charmhelpers.core.hookenv import (
+ log,
+ DEBUG,
+ WARNING,
+)
+from charmhelpers.fetch import SourceConfigError, GPGKeyError
+
+PROPOSED_POCKET = (
+ "# Proposed\n"
+ "deb http://archive.ubuntu.com/ubuntu {}-proposed main universe "
+ "multiverse restricted\n")
+PROPOSED_PORTS_POCKET = (
+ "# Proposed\n"
+ "deb http://ports.ubuntu.com/ubuntu-ports {}-proposed main universe "
+ "multiverse restricted\n")
+# Only supports 64bit and ppc64 at the moment.
+ARCH_TO_PROPOSED_POCKET = {
+ 'x86_64': PROPOSED_POCKET,
+ 'ppc64le': PROPOSED_PORTS_POCKET,
+ 'aarch64': PROPOSED_PORTS_POCKET,
+}
+CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu"
+CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
+CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
+deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
+"""
+CLOUD_ARCHIVE_POCKETS = {
+ # Folsom
+ 'folsom': 'precise-updates/folsom',
+ 'folsom/updates': 'precise-updates/folsom',
+ 'precise-folsom': 'precise-updates/folsom',
+ 'precise-folsom/updates': 'precise-updates/folsom',
+ 'precise-updates/folsom': 'precise-updates/folsom',
+ 'folsom/proposed': 'precise-proposed/folsom',
+ 'precise-folsom/proposed': 'precise-proposed/folsom',
+ 'precise-proposed/folsom': 'precise-proposed/folsom',
+ # Grizzly
+ 'grizzly': 'precise-updates/grizzly',
+ 'grizzly/updates': 'precise-updates/grizzly',
+ 'precise-grizzly': 'precise-updates/grizzly',
+ 'precise-grizzly/updates': 'precise-updates/grizzly',
+ 'precise-updates/grizzly': 'precise-updates/grizzly',
+ 'grizzly/proposed': 'precise-proposed/grizzly',
+ 'precise-grizzly/proposed': 'precise-proposed/grizzly',
+ 'precise-proposed/grizzly': 'precise-proposed/grizzly',
+ # Havana
+ 'havana': 'precise-updates/havana',
+ 'havana/updates': 'precise-updates/havana',
+ 'precise-havana': 'precise-updates/havana',
+ 'precise-havana/updates': 'precise-updates/havana',
+ 'precise-updates/havana': 'precise-updates/havana',
+ 'havana/proposed': 'precise-proposed/havana',
+ 'precise-havana/proposed': 'precise-proposed/havana',
+ 'precise-proposed/havana': 'precise-proposed/havana',
+ # Icehouse
+ 'icehouse': 'precise-updates/icehouse',
+ 'icehouse/updates': 'precise-updates/icehouse',
+ 'precise-icehouse': 'precise-updates/icehouse',
+ 'precise-icehouse/updates': 'precise-updates/icehouse',
+ 'precise-updates/icehouse': 'precise-updates/icehouse',
+ 'icehouse/proposed': 'precise-proposed/icehouse',
+ 'precise-icehouse/proposed': 'precise-proposed/icehouse',
+ 'precise-proposed/icehouse': 'precise-proposed/icehouse',
+ # Juno
+ 'juno': 'trusty-updates/juno',
+ 'juno/updates': 'trusty-updates/juno',
+ 'trusty-juno': 'trusty-updates/juno',
+ 'trusty-juno/updates': 'trusty-updates/juno',
+ 'trusty-updates/juno': 'trusty-updates/juno',
+ 'juno/proposed': 'trusty-proposed/juno',
+ 'trusty-juno/proposed': 'trusty-proposed/juno',
+ 'trusty-proposed/juno': 'trusty-proposed/juno',
+ # Kilo
+ 'kilo': 'trusty-updates/kilo',
+ 'kilo/updates': 'trusty-updates/kilo',
+ 'trusty-kilo': 'trusty-updates/kilo',
+ 'trusty-kilo/updates': 'trusty-updates/kilo',
+ 'trusty-updates/kilo': 'trusty-updates/kilo',
+ 'kilo/proposed': 'trusty-proposed/kilo',
+ 'trusty-kilo/proposed': 'trusty-proposed/kilo',
+ 'trusty-proposed/kilo': 'trusty-proposed/kilo',
+ # Liberty
+ 'liberty': 'trusty-updates/liberty',
+ 'liberty/updates': 'trusty-updates/liberty',
+ 'trusty-liberty': 'trusty-updates/liberty',
+ 'trusty-liberty/updates': 'trusty-updates/liberty',
+ 'trusty-updates/liberty': 'trusty-updates/liberty',
+ 'liberty/proposed': 'trusty-proposed/liberty',
+ 'trusty-liberty/proposed': 'trusty-proposed/liberty',
+ 'trusty-proposed/liberty': 'trusty-proposed/liberty',
+ # Mitaka
+ 'mitaka': 'trusty-updates/mitaka',
+ 'mitaka/updates': 'trusty-updates/mitaka',
+ 'trusty-mitaka': 'trusty-updates/mitaka',
+ 'trusty-mitaka/updates': 'trusty-updates/mitaka',
+ 'trusty-updates/mitaka': 'trusty-updates/mitaka',
+ 'mitaka/proposed': 'trusty-proposed/mitaka',
+ 'trusty-mitaka/proposed': 'trusty-proposed/mitaka',
+ 'trusty-proposed/mitaka': 'trusty-proposed/mitaka',
+ # Newton
+ 'newton': 'xenial-updates/newton',
+ 'newton/updates': 'xenial-updates/newton',
+ 'xenial-newton': 'xenial-updates/newton',
+ 'xenial-newton/updates': 'xenial-updates/newton',
+ 'xenial-updates/newton': 'xenial-updates/newton',
+ 'newton/proposed': 'xenial-proposed/newton',
+ 'xenial-newton/proposed': 'xenial-proposed/newton',
+ 'xenial-proposed/newton': 'xenial-proposed/newton',
+ # Ocata
+ 'ocata': 'xenial-updates/ocata',
+ 'ocata/updates': 'xenial-updates/ocata',
+ 'xenial-ocata': 'xenial-updates/ocata',
+ 'xenial-ocata/updates': 'xenial-updates/ocata',
+ 'xenial-updates/ocata': 'xenial-updates/ocata',
+ 'ocata/proposed': 'xenial-proposed/ocata',
+ 'xenial-ocata/proposed': 'xenial-proposed/ocata',
+ 'xenial-proposed/ocata': 'xenial-proposed/ocata',
+ # Pike
+ 'pike': 'xenial-updates/pike',
+ 'xenial-pike': 'xenial-updates/pike',
+ 'xenial-pike/updates': 'xenial-updates/pike',
+ 'xenial-updates/pike': 'xenial-updates/pike',
+ 'pike/proposed': 'xenial-proposed/pike',
+ 'xenial-pike/proposed': 'xenial-proposed/pike',
+ 'xenial-proposed/pike': 'xenial-proposed/pike',
+ # Queens
+ 'queens': 'xenial-updates/queens',
+ 'xenial-queens': 'xenial-updates/queens',
+ 'xenial-queens/updates': 'xenial-updates/queens',
+ 'xenial-updates/queens': 'xenial-updates/queens',
+ 'queens/proposed': 'xenial-proposed/queens',
+ 'xenial-queens/proposed': 'xenial-proposed/queens',
+ 'xenial-proposed/queens': 'xenial-proposed/queens',
+}
+
+
+APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
+CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries.
+CMD_RETRY_COUNT = 3 # Retry a failing fatal command X times.
+
+
+def filter_installed_packages(packages):
+ """Return a list of packages that require installation."""
+ cache = apt_cache()
+ _pkgs = []
+ for package in packages:
+ try:
+ p = cache[package]
+ p.current_ver or _pkgs.append(package)
+ except KeyError:
+ log('Package {} has no installation candidate.'.format(package),
+ level='WARNING')
+ _pkgs.append(package)
+ return _pkgs
+
+
+def apt_cache(in_memory=True, progress=None):
+ """Build and return an apt cache."""
+ from apt import apt_pkg
+ apt_pkg.init()
+ if in_memory:
+ apt_pkg.config.set("Dir::Cache::pkgcache", "")
+ apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
+ return apt_pkg.Cache(progress)
+
+
+def apt_install(packages, options=None, fatal=False):
+ """Install one or more packages."""
+ if options is None:
+ options = ['--option=Dpkg::Options::=--force-confold']
+
+ cmd = ['apt-get', '--assume-yes']
+ cmd.extend(options)
+ cmd.append('install')
+ if isinstance(packages, six.string_types):
+ cmd.append(packages)
+ else:
+ cmd.extend(packages)
+ log("Installing {} with options: {}".format(packages,
+ options))
+ _run_apt_command(cmd, fatal)
+
+
+def apt_upgrade(options=None, fatal=False, dist=False):
+ """Upgrade all packages."""
+ if options is None:
+ options = ['--option=Dpkg::Options::=--force-confold']
+
+ cmd = ['apt-get', '--assume-yes']
+ cmd.extend(options)
+ if dist:
+ cmd.append('dist-upgrade')
+ else:
+ cmd.append('upgrade')
+ log("Upgrading with options: {}".format(options))
+ _run_apt_command(cmd, fatal)
+
+
+def apt_update(fatal=False):
+ """Update local apt cache."""
+ cmd = ['apt-get', 'update']
+ _run_apt_command(cmd, fatal)
+
+
+def apt_purge(packages, fatal=False):
+ """Purge one or more packages."""
+ cmd = ['apt-get', '--assume-yes', 'purge']
+ if isinstance(packages, six.string_types):
+ cmd.append(packages)
+ else:
+ cmd.extend(packages)
+ log("Purging {}".format(packages))
+ _run_apt_command(cmd, fatal)
+
+
+def apt_mark(packages, mark, fatal=False):
+ """Flag one or more packages using apt-mark."""
+ log("Marking {} as {}".format(packages, mark))
+ cmd = ['apt-mark', mark]
+ if isinstance(packages, six.string_types):
+ cmd.append(packages)
+ else:
+ cmd.extend(packages)
+
+ if fatal:
+ subprocess.check_call(cmd, universal_newlines=True)
+ else:
+ subprocess.call(cmd, universal_newlines=True)
+
+
+def apt_hold(packages, fatal=False):
+ return apt_mark(packages, 'hold', fatal=fatal)
+
+
+def apt_unhold(packages, fatal=False):
+ return apt_mark(packages, 'unhold', fatal=fatal)
+
+
+def import_key(key):
+ """Import an ASCII Armor key.
+
+ /!\ A Radix64 format keyid is also supported for backwards
+ compatibility, but should never be used; the key retrieval
+ mechanism is insecure and subject to man-in-the-middle attacks
+ voiding all signature checks using that key.
+
+ :param keyid: The key in ASCII armor format,
+ including BEGIN and END markers.
+ :raises: GPGKeyError if the key could not be imported
+ """
+ key = key.strip()
+ if '-' in key or '\n' in key:
+ # Send everything not obviously a keyid to GPG to import, as
+ # we trust its validation better than our own. eg. handling
+ # comments before the key.
+ log("PGP key found (looks like ASCII Armor format)", level=DEBUG)
+ if ('-----BEGIN PGP PUBLIC KEY BLOCK-----' in key and
+ '-----END PGP PUBLIC KEY BLOCK-----' in key):
+ log("Importing ASCII Armor PGP key", level=DEBUG)
+ with NamedTemporaryFile() as keyfile:
+ with open(keyfile.name, 'w') as fd:
+ fd.write(key)
+ fd.write("\n")
+ cmd = ['apt-key', 'add', keyfile.name]
+ try:
+ subprocess.check_call(cmd)
+ except subprocess.CalledProcessError:
+ error = "Error importing PGP key '{}'".format(key)
+ log(error)
+ raise GPGKeyError(error)
+ else:
+ raise GPGKeyError("ASCII armor markers missing from GPG key")
+ else:
+ # We should only send things obviously not a keyid offsite
+ # via this unsecured protocol, as it may be a secret or part
+ # of one.
+ log("PGP key found (looks like Radix64 format)", level=WARNING)
+ log("INSECURLY importing PGP key from keyserver; "
+ "full key not provided.", level=WARNING)
+ cmd = ['apt-key', 'adv', '--keyserver',
+ 'hkp://keyserver.ubuntu.com:80', '--recv-keys', key]
+ try:
+ subprocess.check_call(cmd)
+ except subprocess.CalledProcessError:
+ error = "Error importing PGP key '{}'".format(key)
+ log(error)
+ raise GPGKeyError(error)
+
+
+def add_source(source, key=None, fail_invalid=False):
+ """Add a package source to this system.
+
+ @param source: a URL or sources.list entry, as supported by
+ add-apt-repository(1). Examples::
+
+ ppa:charmers/example
+ deb https://stub:key@private.example.com/ubuntu trusty main
+
+ In addition:
+ 'proposed:' may be used to enable the standard 'proposed'
+ pocket for the release.
+ 'cloud:' may be used to activate official cloud archive pockets,
+ such as 'cloud:icehouse'
+ 'distro' may be used as a noop
+
+ Full list of source specifications supported by the function are:
+
+ 'distro': A NOP; i.e. it has no effect.
+ 'proposed': the proposed deb spec [2] is wrtten to
+ /etc/apt/sources.list/proposed
+ 'distro-proposed': adds -proposed to the debs [2]
+ 'ppa:': add-apt-repository --yes
+ 'deb ': add-apt-repository --yes deb
+ 'http://....': add-apt-repository --yes http://...
+ 'cloud-archive:': add-apt-repository -yes cloud-archive:
+ 'cloud:[-staging]': specify a Cloud Archive pocket with
+ optional staging version. If staging is used then the staging PPA [2]
+ with be used. If staging is NOT used then the cloud archive [3] will be
+ added, and the 'ubuntu-cloud-keyring' package will be added for the
+ current distro.
+
+ Otherwise the source is not recognised and this is logged to the juju log.
+ However, no error is raised, unless sys_error_on_exit is True.
+
+ [1] deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
+ where {} is replaced with the derived pocket name.
+ [2] deb http://archive.ubuntu.com/ubuntu {}-proposed \
+ main universe multiverse restricted
+ where {} is replaced with the lsb_release codename (e.g. xenial)
+ [3] deb http://ubuntu-cloud.archive.canonical.com/ubuntu
+ to /etc/apt/sources.list.d/cloud-archive-list
+
+ @param key: A key to be added to the system's APT keyring and used
+ to verify the signatures on packages. Ideally, this should be an
+ ASCII format GPG public key including the block headers. A GPG key
+ id may also be used, but be aware that only insecure protocols are
+ available to retrieve the actual public key from a public keyserver
+ placing your Juju environment at risk. ppa and cloud archive keys
+ are securely added automtically, so sould not be provided.
+
+ @param fail_invalid: (boolean) if True, then the function raises a
+ SourceConfigError is there is no matching installation source.
+
+ @raises SourceConfigError() if for cloud:, the is not a
+ valid pocket in CLOUD_ARCHIVE_POCKETS
+ """
+ _mapping = OrderedDict([
+ (r"^distro$", lambda: None), # This is a NOP
+ (r"^(?:proposed|distro-proposed)$", _add_proposed),
+ (r"^cloud-archive:(.*)$", _add_apt_repository),
+ (r"^((?:deb |http:|https:|ppa:).*)$", _add_apt_repository),
+ (r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging),
+ (r"^cloud:(.*)-(.*)$", _add_cloud_distro_check),
+ (r"^cloud:(.*)$", _add_cloud_pocket),
+ (r"^snap:.*-(.*)-(.*)$", _add_cloud_distro_check),
+ ])
+ if source is None:
+ source = ''
+ for r, fn in six.iteritems(_mapping):
+ m = re.match(r, source)
+ if m:
+ # call the assoicated function with the captured groups
+ # raises SourceConfigError on error.
+ fn(*m.groups())
+ if key:
+ try:
+ import_key(key)
+ except GPGKeyError as e:
+ raise SourceConfigError(str(e))
+ break
+ else:
+ # nothing matched. log an error and maybe sys.exit
+ err = "Unknown source: {!r}".format(source)
+ log(err)
+ if fail_invalid:
+ raise SourceConfigError(err)
+
+
+def _add_proposed():
+ """Add the PROPOSED_POCKET as /etc/apt/source.list.d/proposed.list
+
+ Uses lsb_release()['DISTRIB_CODENAME'] to determine the correct staza for
+ the deb line.
+
+ For intel architecutres PROPOSED_POCKET is used for the release, but for
+ other architectures PROPOSED_PORTS_POCKET is used for the release.
+ """
+ release = lsb_release()['DISTRIB_CODENAME']
+ arch = platform.machine()
+ if arch not in six.iterkeys(ARCH_TO_PROPOSED_POCKET):
+ raise SourceConfigError("Arch {} not supported for (distro-)proposed"
+ .format(arch))
+ with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
+ apt.write(ARCH_TO_PROPOSED_POCKET[arch].format(release))
+
+
+def _add_apt_repository(spec):
+ """Add the spec using add_apt_repository
+
+ :param spec: the parameter to pass to add_apt_repository
+ """
+ _run_with_retries(['add-apt-repository', '--yes', spec])
+
+
+def _add_cloud_pocket(pocket):
+ """Add a cloud pocket as /etc/apt/sources.d/cloud-archive.list
+
+ Note that this overwrites the existing file if there is one.
+
+ This function also converts the simple pocket in to the actual pocket using
+ the CLOUD_ARCHIVE_POCKETS mapping.
+
+ :param pocket: string representing the pocket to add a deb spec for.
+ :raises: SourceConfigError if the cloud pocket doesn't exist or the
+ requested release doesn't match the current distro version.
+ """
+ apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
+ fatal=True)
+ if pocket not in CLOUD_ARCHIVE_POCKETS:
+ raise SourceConfigError(
+ 'Unsupported cloud: source option %s' %
+ pocket)
+ actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
+ with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
+ apt.write(CLOUD_ARCHIVE.format(actual_pocket))
+
+
+def _add_cloud_staging(cloud_archive_release, openstack_release):
+ """Add the cloud staging repository which is in
+ ppa:ubuntu-cloud-archive/-staging
+
+ This function checks that the cloud_archive_release matches the current
+ codename for the distro that charm is being installed on.
+
+ :param cloud_archive_release: string, codename for the release.
+ :param openstack_release: String, codename for the openstack release.
+ :raises: SourceConfigError if the cloud_archive_release doesn't match the
+ current version of the os.
+ """
+ _verify_is_ubuntu_rel(cloud_archive_release, openstack_release)
+ ppa = 'ppa:ubuntu-cloud-archive/{}-staging'.format(openstack_release)
+ cmd = 'add-apt-repository -y {}'.format(ppa)
+ _run_with_retries(cmd.split(' '))
+
+
+def _add_cloud_distro_check(cloud_archive_release, openstack_release):
+ """Add the cloud pocket, but also check the cloud_archive_release against
+ the current distro, and use the openstack_release as the full lookup.
+
+ This just calls _add_cloud_pocket() with the openstack_release as pocket
+ to get the correct cloud-archive.list for dpkg to work with.
+
+ :param cloud_archive_release:String, codename for the distro release.
+ :param openstack_release: String, spec for the release to look up in the
+ CLOUD_ARCHIVE_POCKETS
+ :raises: SourceConfigError if this is the wrong distro, or the pocket spec
+ doesn't exist.
+ """
+ _verify_is_ubuntu_rel(cloud_archive_release, openstack_release)
+ _add_cloud_pocket("{}-{}".format(cloud_archive_release, openstack_release))
+
+
+def _verify_is_ubuntu_rel(release, os_release):
+ """Verify that the release is in the same as the current ubuntu release.
+
+ :param release: String, lowercase for the release.
+ :param os_release: String, the os_release being asked for
+ :raises: SourceConfigError if the release is not the same as the ubuntu
+ release.
+ """
+ ubuntu_rel = lsb_release()['DISTRIB_CODENAME']
+ if release != ubuntu_rel:
+ raise SourceConfigError(
+ 'Invalid Cloud Archive release specified: {}-{} on this Ubuntu'
+ 'version ({})'.format(release, os_release, ubuntu_rel))
+
+
+def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,),
+ retry_message="", cmd_env=None):
+ """Run a command and retry until success or max_retries is reached.
+
+ :param: cmd: str: The apt command to run.
+ :param: max_retries: int: The number of retries to attempt on a fatal
+ command. Defaults to CMD_RETRY_COUNT.
+ :param: retry_exitcodes: tuple: Optional additional exit codes to retry.
+ Defaults to retry on exit code 1.
+ :param: retry_message: str: Optional log prefix emitted during retries.
+ :param: cmd_env: dict: Environment variables to add to the command run.
+ """
+
+ env = None
+ kwargs = {}
+ if cmd_env:
+ env = os.environ.copy()
+ env.update(cmd_env)
+ kwargs['env'] = env
+
+ if not retry_message:
+ retry_message = "Failed executing '{}'".format(" ".join(cmd))
+ retry_message += ". Will retry in {} seconds".format(CMD_RETRY_DELAY)
+
+ retry_count = 0
+ result = None
+
+ retry_results = (None,) + retry_exitcodes
+ while result in retry_results:
+ try:
+ # result = subprocess.check_call(cmd, env=env)
+ result = subprocess.check_call(cmd, **kwargs)
+ except subprocess.CalledProcessError as e:
+ retry_count = retry_count + 1
+ if retry_count > max_retries:
+ raise
+ result = e.returncode
+ log(retry_message)
+ time.sleep(CMD_RETRY_DELAY)
+
+
+def _run_apt_command(cmd, fatal=False):
+ """Run an apt command with optional retries.
+
+ :param: cmd: str: The apt command to run.
+ :param: fatal: bool: Whether the command's output should be checked and
+ retried.
+ """
+ # Provide DEBIAN_FRONTEND=noninteractive if not present in the environment.
+ cmd_env = {
+ 'DEBIAN_FRONTEND': os.environ.get('DEBIAN_FRONTEND', 'noninteractive')}
+
+ if fatal:
+ _run_with_retries(
+ cmd, cmd_env=cmd_env, retry_exitcodes=(1, APT_NO_LOCK,),
+ retry_message="Couldn't acquire DPKG lock")
+ else:
+ env = os.environ.copy()
+ env.update(cmd_env)
+ subprocess.call(cmd, env=env)
+
+
+def get_upstream_version(package):
+ """Determine upstream version based on installed package
+
+ @returns None (if not installed) or the upstream version
+ """
+ import apt_pkg
+ cache = apt_cache()
+ try:
+ pkg = cache[package]
+ except Exception:
+ # the package is unknown to the current apt cache.
+ return None
+
+ if not pkg.current_ver:
+ # package is known, but no version is currently installed.
+ return None
+
+ return apt_pkg.upstream_version(pkg.current_ver.ver_str)
diff --git a/tests/charmhelpers/osplatform.py b/tests/charmhelpers/osplatform.py
new file mode 100644
index 0000000..d9a4d5c
--- /dev/null
+++ b/tests/charmhelpers/osplatform.py
@@ -0,0 +1,25 @@
+import platform
+
+
+def get_platform():
+ """Return the current OS platform.
+
+ For example: if current os platform is Ubuntu then a string "ubuntu"
+ will be returned (which is the name of the module).
+ This string is used to decide which platform module should be imported.
+ """
+ # linux_distribution is deprecated and will be removed in Python 3.7
+ # Warings *not* disabled, as we certainly need to fix this.
+ tuple_platform = platform.linux_distribution()
+ current_platform = tuple_platform[0]
+ if "Ubuntu" in current_platform:
+ return "ubuntu"
+ elif "CentOS" in current_platform:
+ return "centos"
+ elif "debian" in current_platform:
+ # Stock Python does not detect Ubuntu and instead returns debian.
+ # Or at least it does in some build environments like Travis CI
+ return "ubuntu"
+ else:
+ raise RuntimeError("This module is not supported on {}."
+ .format(current_platform))
diff --git a/tests/gate-basic-artful-pike b/tests/gate-basic-artful-pike
new file mode 100644
index 0000000..f850488
--- /dev/null
+++ b/tests/gate-basic-artful-pike
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Amulet tests on a basic rabbitmq-server deployment on artful-pike."""
+
+from basic_deployment import RmqBasicDeployment
+
+if __name__ == '__main__':
+ deployment = RmqBasicDeployment(series='artful')
+ deployment.run_tests()
+
+# NOTE(admcleod): Artful target disabled, pending bug:
+# https://bugs.launchpad.net/charm-percona-cluster/+bug/1728132
diff --git a/tests/gate-basic-xenial-pike b/tests/gate-basic-xenial-pike
new file mode 100755
index 0000000..20e286f
--- /dev/null
+++ b/tests/gate-basic-xenial-pike
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Amulet tests on a basic nova compute deployment on xenial-pike."""
+
+from basic_deployment import NovaBasicDeployment
+
+if __name__ == '__main__':
+ deployment = NovaBasicDeployment(series='xenial',
+ openstack='cloud:xenial-pike',
+ source='cloud:xenial-updates/pike')
+ deployment.run_tests()
diff --git a/tox.ini b/tox.ini
index 7c2936e..6d44f4b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -60,7 +60,7 @@ basepython = python2.7
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
commands =
- bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-mitaka --no-destroy
+ bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-pike --no-destroy
[testenv:func27-dfs]
# Charm Functional Test