diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
index 0cfeaa4..b01e6cb 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -15,6 +15,7 @@
# along with charm-helpers. If not, see .
import six
+from collections import OrderedDict
from charmhelpers.contrib.amulet.deployment import (
AmuletDeployment
)
@@ -43,17 +44,24 @@ class OpenStackAmuletDeployment(AmuletDeployment):
Determine if the local branch being tested is derived from its
stable or next (dev) branch, and based on this, use the corresonding
stable or next branches for the other_services."""
- base_charms = ['mysql', 'mongodb', 'rabbitmq-server']
+ base_charms = ['mysql', 'mongodb']
+
+ if self.series in ['precise', 'trusty']:
+ base_series = self.series
+ else:
+ base_series = self.current_next
if self.stable:
for svc in other_services:
- temp = 'lp:charms/{}'
- svc['location'] = temp.format(svc['name'])
+ temp = 'lp:charms/{}/{}'
+ svc['location'] = temp.format(base_series,
+ svc['name'])
else:
for svc in other_services:
if svc['name'] in base_charms:
- temp = 'lp:charms/{}'
- svc['location'] = temp.format(svc['name'])
+ temp = 'lp:charms/{}/{}'
+ svc['location'] = temp.format(base_series,
+ svc['name'])
else:
temp = 'lp:~openstack-charmers/charms/{}/{}/next'
svc['location'] = temp.format(self.current_next,
@@ -71,9 +79,9 @@ class OpenStackAmuletDeployment(AmuletDeployment):
services.append(this_service)
use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
'ceph-osd', 'ceph-radosgw']
- # Openstack subordinate charms do not expose an origin option as that
- # is controlled by the principle
- ignore = ['neutron-openvswitch']
+ # Most OpenStack subordinate charms do not expose an origin option
+ # as that is controlled by the principle.
+ ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch']
if self.openstack:
for svc in services:
@@ -98,14 +106,78 @@ class OpenStackAmuletDeployment(AmuletDeployment):
Return an integer representing the enum value of the openstack
release.
"""
+ # Must be ordered by OpenStack release (not by Ubuntu release):
(self.precise_essex, self.precise_folsom, self.precise_grizzly,
self.precise_havana, self.precise_icehouse,
- self.trusty_icehouse) = range(6)
+ self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
+ self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
+ self.wily_liberty) = range(12)
+
releases = {
('precise', None): self.precise_essex,
('precise', 'cloud:precise-folsom'): self.precise_folsom,
('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
('precise', 'cloud:precise-havana'): self.precise_havana,
('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
- ('trusty', None): self.trusty_icehouse}
+ ('trusty', None): self.trusty_icehouse,
+ ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
+ ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
+ ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
+ ('utopic', None): self.utopic_juno,
+ ('vivid', None): self.vivid_kilo,
+ ('wily', None): self.wily_liberty}
return releases[(self.series, self.openstack)]
+
+ def _get_openstack_release_string(self):
+ """Get openstack release string.
+
+ Return a string representing the openstack release.
+ """
+ releases = OrderedDict([
+ ('precise', 'essex'),
+ ('quantal', 'folsom'),
+ ('raring', 'grizzly'),
+ ('saucy', 'havana'),
+ ('trusty', 'icehouse'),
+ ('utopic', 'juno'),
+ ('vivid', 'kilo'),
+ ('wily', 'liberty'),
+ ])
+ if self.openstack:
+ os_origin = self.openstack.split(':')[1]
+ return os_origin.split('%s-' % self.series)[1].split('/')[0]
+ else:
+ return releases[self.series]
+
+ def get_ceph_expected_pools(self, radosgw=False):
+ """Return a list of expected ceph pools in a ceph + cinder + glance
+ test scenario, based on OpenStack release and whether ceph radosgw
+ is flagged as present or not."""
+
+ if self._get_openstack_release() >= self.trusty_kilo:
+ # Kilo or later
+ pools = [
+ 'rbd',
+ 'cinder',
+ 'glance'
+ ]
+ else:
+ # Juno or earlier
+ pools = [
+ 'data',
+ 'metadata',
+ 'rbd',
+ 'cinder',
+ 'glance'
+ ]
+
+ if radosgw:
+ pools.extend([
+ '.rgw.root',
+ '.rgw.control',
+ '.rgw',
+ '.rgw.gc',
+ '.users.uid'
+ ])
+
+ return pools
diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py
index 9c3d918..03f7927 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py
@@ -14,16 +14,20 @@
# You should have received a copy of the GNU Lesser General Public License
# along with charm-helpers. If not, see .
+import amulet
+import json
import logging
import os
+import six
import time
import urllib
+import cinderclient.v1.client as cinder_client
import glanceclient.v1.client as glance_client
+import heatclient.v1.client as heat_client
import keystoneclient.v2_0 as keystone_client
import novaclient.v1_1.client as nova_client
-
-import six
+import swiftclient
from charmhelpers.contrib.amulet.utils import (
AmuletUtils
@@ -37,7 +41,7 @@ class OpenStackAmuletUtils(AmuletUtils):
"""OpenStack amulet utilities.
This class inherits from AmuletUtils and has additional support
- that is specifically for use by OpenStack charms.
+ that is specifically for use by OpenStack charm tests.
"""
def __init__(self, log_level=ERROR):
@@ -51,6 +55,8 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate actual endpoint data vs expected endpoint data. The ports
are used to find the matching endpoint.
"""
+ self.log.debug('Validating endpoint data...')
+ self.log.debug('actual: {}'.format(repr(endpoints)))
found = False
for ep in endpoints:
self.log.debug('endpoint: {}'.format(repr(ep)))
@@ -77,6 +83,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual service catalog endpoints vs a list of
expected service catalog endpoints.
"""
+ self.log.debug('Validating service catalog endpoint data...')
self.log.debug('actual: {}'.format(repr(actual)))
for k, v in six.iteritems(expected):
if k in actual:
@@ -93,6 +100,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual tenant data vs list of expected tenant
data.
"""
+ self.log.debug('Validating tenant data...')
self.log.debug('actual: {}'.format(repr(actual)))
for e in expected:
found = False
@@ -114,6 +122,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual role data vs a list of expected role
data.
"""
+ self.log.debug('Validating role data...')
self.log.debug('actual: {}'.format(repr(actual)))
for e in expected:
found = False
@@ -134,6 +143,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual user data vs a list of expected user
data.
"""
+ self.log.debug('Validating user data...')
self.log.debug('actual: {}'.format(repr(actual)))
for e in expected:
found = False
@@ -155,17 +165,30 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual flavors vs a list of expected flavors.
"""
+ self.log.debug('Validating flavor data...')
self.log.debug('actual: {}'.format(repr(actual)))
act = [a.name for a in actual]
return self._validate_list_data(expected, act)
def tenant_exists(self, keystone, tenant):
"""Return True if tenant exists."""
+ self.log.debug('Checking if tenant exists ({})...'.format(tenant))
return tenant in [t.name for t in keystone.tenants.list()]
+ def authenticate_cinder_admin(self, keystone_sentry, username,
+ password, tenant):
+ """Authenticates admin user with cinder."""
+ # NOTE(beisner): cinder python client doesn't accept tokens.
+ service_ip = \
+ keystone_sentry.relation('shared-db',
+ 'mysql:shared-db')['private-address']
+ ept = "http://{}:5000/v2.0".format(service_ip.strip().decode('utf-8'))
+ return cinder_client.Client(username, password, tenant, ept)
+
def authenticate_keystone_admin(self, keystone_sentry, user, password,
tenant):
"""Authenticates admin user with the keystone admin endpoint."""
+ self.log.debug('Authenticating keystone admin...')
unit = keystone_sentry
service_ip = unit.relation('shared-db',
'mysql:shared-db')['private-address']
@@ -175,6 +198,7 @@ class OpenStackAmuletUtils(AmuletUtils):
def authenticate_keystone_user(self, keystone, user, password, tenant):
"""Authenticates a regular user with the keystone public endpoint."""
+ self.log.debug('Authenticating keystone user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
return keystone_client.Client(username=user, password=password,
@@ -182,19 +206,49 @@ class OpenStackAmuletUtils(AmuletUtils):
def authenticate_glance_admin(self, keystone):
"""Authenticates admin user with glance."""
+ self.log.debug('Authenticating glance admin...')
ep = keystone.service_catalog.url_for(service_type='image',
endpoint_type='adminURL')
return glance_client.Client(ep, token=keystone.auth_token)
+ def authenticate_heat_admin(self, keystone):
+ """Authenticates the admin user with heat."""
+ self.log.debug('Authenticating heat admin...')
+ ep = keystone.service_catalog.url_for(service_type='orchestration',
+ endpoint_type='publicURL')
+ return heat_client.Client(endpoint=ep, token=keystone.auth_token)
+
def authenticate_nova_user(self, keystone, user, password, tenant):
"""Authenticates a regular user with nova-api."""
+ self.log.debug('Authenticating nova user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
return nova_client.Client(username=user, api_key=password,
project_id=tenant, auth_url=ep)
+ def authenticate_swift_user(self, keystone, user, password, tenant):
+ """Authenticates a regular user with swift api."""
+ self.log.debug('Authenticating swift user ({})...'.format(user))
+ ep = keystone.service_catalog.url_for(service_type='identity',
+ endpoint_type='publicURL')
+ return swiftclient.Connection(authurl=ep,
+ user=user,
+ key=password,
+ tenant_name=tenant,
+ auth_version='2.0')
+
def create_cirros_image(self, glance, image_name):
- """Download the latest cirros image and upload it to glance."""
+ """Download the latest cirros image and upload it to glance,
+ validate and return a resource pointer.
+
+ :param glance: pointer to authenticated glance connection
+ :param image_name: display name for new image
+ :returns: glance image pointer
+ """
+ self.log.debug('Creating glance cirros image '
+ '({})...'.format(image_name))
+
+ # Download cirros image
http_proxy = os.getenv('AMULET_HTTP_PROXY')
self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
if http_proxy:
@@ -203,57 +257,67 @@ class OpenStackAmuletUtils(AmuletUtils):
else:
opener = urllib.FancyURLopener()
- f = opener.open("http://download.cirros-cloud.net/version/released")
+ f = opener.open('http://download.cirros-cloud.net/version/released')
version = f.read().strip()
- cirros_img = "cirros-{}-x86_64-disk.img".format(version)
+ cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
local_path = os.path.join('tests', cirros_img)
if not os.path.exists(local_path):
- cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net",
+ cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
version, cirros_img)
opener.retrieve(cirros_url, local_path)
f.close()
+ # Create glance image
with open(local_path) as f:
image = glance.images.create(name=image_name, is_public=True,
disk_format='qcow2',
container_format='bare', data=f)
- count = 1
- status = image.status
- while status != 'active' and count < 10:
- time.sleep(3)
- image = glance.images.get(image.id)
- status = image.status
- self.log.debug('image status: {}'.format(status))
- count += 1
- if status != 'active':
- self.log.error('image creation timed out')
- return None
+ # Wait for image to reach active status
+ img_id = image.id
+ ret = self.resource_reaches_status(glance.images, img_id,
+ expected_stat='active',
+ msg='Image status wait')
+ if not ret:
+ msg = 'Glance image failed to reach expected state.'
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ # Re-validate new image
+ self.log.debug('Validating image attributes...')
+ val_img_name = glance.images.get(img_id).name
+ val_img_stat = glance.images.get(img_id).status
+ val_img_pub = glance.images.get(img_id).is_public
+ val_img_cfmt = glance.images.get(img_id).container_format
+ val_img_dfmt = glance.images.get(img_id).disk_format
+ msg_attr = ('Image attributes - name:{} public:{} id:{} stat:{} '
+ 'container fmt:{} disk fmt:{}'.format(
+ val_img_name, val_img_pub, img_id,
+ val_img_stat, val_img_cfmt, val_img_dfmt))
+
+ if val_img_name == image_name and val_img_stat == 'active' \
+ and val_img_pub is True and val_img_cfmt == 'bare' \
+ and val_img_dfmt == 'qcow2':
+ self.log.debug(msg_attr)
+ else:
+ msg = ('Volume validation failed, {}'.format(msg_attr))
+ amulet.raise_status(amulet.FAIL, msg=msg)
return image
def delete_image(self, glance, image):
"""Delete the specified image."""
- num_before = len(list(glance.images.list()))
- glance.images.delete(image)
- count = 1
- num_after = len(list(glance.images.list()))
- while num_after != (num_before - 1) and count < 10:
- time.sleep(3)
- num_after = len(list(glance.images.list()))
- self.log.debug('number of images: {}'.format(num_after))
- count += 1
-
- if num_after != (num_before - 1):
- self.log.error('image deletion timed out')
- return False
-
- return True
+ # /!\ DEPRECATION WARNING
+ self.log.warn('/!\\ DEPRECATION WARNING: use '
+ 'delete_resource instead of delete_image.')
+ self.log.debug('Deleting glance image ({})...'.format(image))
+ return self.delete_resource(glance.images, image, msg='glance image')
def create_instance(self, nova, image_name, instance_name, flavor):
"""Create the specified instance."""
+ self.log.debug('Creating instance '
+ '({}|{}|{})'.format(instance_name, image_name, flavor))
image = nova.images.find(name=image_name)
flavor = nova.flavors.find(name=flavor)
instance = nova.servers.create(name=instance_name, image=image,
@@ -276,19 +340,265 @@ class OpenStackAmuletUtils(AmuletUtils):
def delete_instance(self, nova, instance):
"""Delete the specified instance."""
- num_before = len(list(nova.servers.list()))
- nova.servers.delete(instance)
- count = 1
- num_after = len(list(nova.servers.list()))
- while num_after != (num_before - 1) and count < 10:
- time.sleep(3)
- num_after = len(list(nova.servers.list()))
- self.log.debug('number of instances: {}'.format(num_after))
- count += 1
+ # /!\ DEPRECATION WARNING
+ self.log.warn('/!\\ DEPRECATION WARNING: use '
+ 'delete_resource instead of delete_instance.')
+ self.log.debug('Deleting instance ({})...'.format(instance))
+ return self.delete_resource(nova.servers, instance,
+ msg='nova instance')
- if num_after != (num_before - 1):
- self.log.error('instance deletion timed out')
+ def create_or_get_keypair(self, nova, keypair_name="testkey"):
+ """Create a new keypair, or return pointer if it already exists."""
+ try:
+ _keypair = nova.keypairs.get(keypair_name)
+ self.log.debug('Keypair ({}) already exists, '
+ 'using it.'.format(keypair_name))
+ return _keypair
+ except:
+ self.log.debug('Keypair ({}) does not exist, '
+ 'creating it.'.format(keypair_name))
+
+ _keypair = nova.keypairs.create(name=keypair_name)
+ return _keypair
+
+ def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1,
+ img_id=None, src_vol_id=None, snap_id=None):
+ """Create cinder volume, optionally from a glance image, OR
+ optionally as a clone of an existing volume, OR optionally
+ from a snapshot. Wait for the new volume status to reach
+ the expected status, validate and return a resource pointer.
+
+ :param vol_name: cinder volume display name
+ :param vol_size: size in gigabytes
+ :param img_id: optional glance image id
+ :param src_vol_id: optional source volume id to clone
+ :param snap_id: optional snapshot id to use
+ :returns: cinder volume pointer
+ """
+ # Handle parameter input and avoid impossible combinations
+ if img_id and not src_vol_id and not snap_id:
+ # Create volume from image
+ self.log.debug('Creating cinder volume from glance image...')
+ bootable = 'true'
+ elif src_vol_id and not img_id and not snap_id:
+ # Clone an existing volume
+ self.log.debug('Cloning cinder volume...')
+ bootable = cinder.volumes.get(src_vol_id).bootable
+ elif snap_id and not src_vol_id and not img_id:
+ # Create volume from snapshot
+ self.log.debug('Creating cinder volume from snapshot...')
+ snap = cinder.volume_snapshots.find(id=snap_id)
+ vol_size = snap.size
+ snap_vol_id = cinder.volume_snapshots.get(snap_id).volume_id
+ bootable = cinder.volumes.get(snap_vol_id).bootable
+ elif not img_id and not src_vol_id and not snap_id:
+ # Create volume
+ self.log.debug('Creating cinder volume...')
+ bootable = 'false'
+ else:
+ # Impossible combination of parameters
+ msg = ('Invalid method use - name:{} size:{} img_id:{} '
+ 'src_vol_id:{} snap_id:{}'.format(vol_name, vol_size,
+ img_id, src_vol_id,
+ snap_id))
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ # Create new volume
+ try:
+ vol_new = cinder.volumes.create(display_name=vol_name,
+ imageRef=img_id,
+ size=vol_size,
+ source_volid=src_vol_id,
+ snapshot_id=snap_id)
+ vol_id = vol_new.id
+ except Exception as e:
+ msg = 'Failed to create volume: {}'.format(e)
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ # Wait for volume to reach available status
+ ret = self.resource_reaches_status(cinder.volumes, vol_id,
+ expected_stat="available",
+ msg="Volume status wait")
+ if not ret:
+ msg = 'Cinder volume failed to reach expected state.'
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ # Re-validate new volume
+ self.log.debug('Validating volume attributes...')
+ val_vol_name = cinder.volumes.get(vol_id).display_name
+ val_vol_boot = cinder.volumes.get(vol_id).bootable
+ val_vol_stat = cinder.volumes.get(vol_id).status
+ val_vol_size = cinder.volumes.get(vol_id).size
+ msg_attr = ('Volume attributes - name:{} id:{} stat:{} boot:'
+ '{} size:{}'.format(val_vol_name, vol_id,
+ val_vol_stat, val_vol_boot,
+ val_vol_size))
+
+ if val_vol_boot == bootable and val_vol_stat == 'available' \
+ and val_vol_name == vol_name and val_vol_size == vol_size:
+ self.log.debug(msg_attr)
+ else:
+ msg = ('Volume validation failed, {}'.format(msg_attr))
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ return vol_new
+
+ def delete_resource(self, resource, resource_id,
+ msg="resource", max_wait=120):
+ """Delete one openstack resource, such as one instance, keypair,
+ image, volume, stack, etc., and confirm deletion within max wait time.
+
+ :param resource: pointer to os resource type, ex:glance_client.images
+ :param resource_id: unique name or id for the openstack resource
+ :param msg: text to identify purpose in logging
+ :param max_wait: maximum wait time in seconds
+ :returns: True if successful, otherwise False
+ """
+ self.log.debug('Deleting OpenStack resource '
+ '{} ({})'.format(resource_id, msg))
+ num_before = len(list(resource.list()))
+ resource.delete(resource_id)
+
+ tries = 0
+ num_after = len(list(resource.list()))
+ while num_after != (num_before - 1) and tries < (max_wait / 4):
+ self.log.debug('{} delete check: '
+ '{} [{}:{}] {}'.format(msg, tries,
+ num_before,
+ num_after,
+ resource_id))
+ time.sleep(4)
+ num_after = len(list(resource.list()))
+ tries += 1
+
+ self.log.debug('{}: expected, actual count = {}, '
+ '{}'.format(msg, num_before - 1, num_after))
+
+ if num_after == (num_before - 1):
+ return True
+ else:
+ self.log.error('{} delete timed out'.format(msg))
return False
- return True
+ def resource_reaches_status(self, resource, resource_id,
+ expected_stat='available',
+ msg='resource', max_wait=120):
+ """Wait for an openstack resources status to reach an
+ expected status within a specified time. Useful to confirm that
+ nova instances, cinder vols, snapshots, glance images, heat stacks
+ and other resources eventually reach the expected status.
+
+ :param resource: pointer to os resource type, ex: heat_client.stacks
+ :param resource_id: unique id for the openstack resource
+ :param expected_stat: status to expect resource to reach
+ :param msg: text to identify purpose in logging
+ :param max_wait: maximum wait time in seconds
+ :returns: True if successful, False if status is not reached
+ """
+
+ tries = 0
+ resource_stat = resource.get(resource_id).status
+ while resource_stat != expected_stat and tries < (max_wait / 4):
+ self.log.debug('{} status check: '
+ '{} [{}:{}] {}'.format(msg, tries,
+ resource_stat,
+ expected_stat,
+ resource_id))
+ time.sleep(4)
+ resource_stat = resource.get(resource_id).status
+ tries += 1
+
+ self.log.debug('{}: expected, actual status = {}, '
+ '{}'.format(msg, resource_stat, expected_stat))
+
+ if resource_stat == expected_stat:
+ return True
+ else:
+ self.log.debug('{} never reached expected status: '
+ '{}'.format(resource_id, expected_stat))
+ return False
+
+ def get_ceph_osd_id_cmd(self, index):
+ """Produce a shell command that will return a ceph-osd id."""
+ return ("`initctl list | grep 'ceph-osd ' | "
+ "awk 'NR=={} {{ print $2 }}' | "
+ "grep -o '[0-9]*'`".format(index + 1))
+
+ def get_ceph_pools(self, sentry_unit):
+ """Return a dict of ceph pools from a single ceph unit, with
+ pool name as keys, pool id as vals."""
+ pools = {}
+ cmd = 'sudo ceph osd lspools'
+ output, code = sentry_unit.run(cmd)
+ if code != 0:
+ msg = ('{} `{}` returned {} '
+ '{}'.format(sentry_unit.info['unit_name'],
+ cmd, code, output))
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance,
+ for pool in str(output).split(','):
+ pool_id_name = pool.split(' ')
+ if len(pool_id_name) == 2:
+ pool_id = pool_id_name[0]
+ pool_name = pool_id_name[1]
+ pools[pool_name] = int(pool_id)
+
+ self.log.debug('Pools on {}: {}'.format(sentry_unit.info['unit_name'],
+ pools))
+ return pools
+
+ def get_ceph_df(self, sentry_unit):
+ """Return dict of ceph df json output, including ceph pool state.
+
+ :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+ :returns: Dict of ceph df output
+ """
+ cmd = 'sudo ceph df --format=json'
+ output, code = sentry_unit.run(cmd)
+ if code != 0:
+ msg = ('{} `{}` returned {} '
+ '{}'.format(sentry_unit.info['unit_name'],
+ cmd, code, output))
+ amulet.raise_status(amulet.FAIL, msg=msg)
+ return json.loads(output)
+
+ def get_ceph_pool_sample(self, sentry_unit, pool_id=0):
+ """Take a sample of attributes of a ceph pool, returning ceph
+ pool name, object count and disk space used for the specified
+ pool ID number.
+
+ :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+ :param pool_id: Ceph pool ID
+ :returns: List of pool name, object count, kb disk space used
+ """
+ df = self.get_ceph_df(sentry_unit)
+ pool_name = df['pools'][pool_id]['name']
+ obj_count = df['pools'][pool_id]['stats']['objects']
+ kb_used = df['pools'][pool_id]['stats']['kb_used']
+ self.log.debug('Ceph {} pool (ID {}): {} objects, '
+ '{} kb used'.format(pool_name, pool_id,
+ obj_count, kb_used))
+ return pool_name, obj_count, kb_used
+
+ def validate_ceph_pool_samples(self, samples, sample_type="resource pool"):
+ """Validate ceph pool samples taken over time, such as pool
+ object counts or pool kb used, before adding, after adding, and
+ after deleting items which affect those pool attributes. The
+ 2nd element is expected to be greater than the 1st; 3rd is expected
+ to be less than the 2nd.
+
+ :param samples: List containing 3 data samples
+ :param sample_type: String for logging and usage context
+ :returns: None if successful, Failure message otherwise
+ """
+ original, created, deleted = range(3)
+ if samples[created] <= samples[original] or \
+ samples[deleted] >= samples[created]:
+ return ('Ceph {} samples ({}) '
+ 'unexpected.'.format(sample_type, samples))
+ else:
+ self.log.debug('Ceph {} samples (OK): '
+ '{}'.format(sample_type, samples))
+ return None
diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py
index c7c4cd4..8f3f1b1 100644
--- a/hooks/charmhelpers/contrib/openstack/context.py
+++ b/hooks/charmhelpers/contrib/openstack/context.py
@@ -16,11 +16,13 @@
import json
import os
+import re
import time
from base64 import b64decode
from subprocess import check_call
import six
+import yaml
from charmhelpers.fetch import (
apt_install,
@@ -45,8 +47,11 @@ from charmhelpers.core.hookenv import (
)
from charmhelpers.core.sysctl import create as sysctl_create
+from charmhelpers.core.strutils import bool_from_string
from charmhelpers.core.host import (
+ list_nics,
+ get_nic_hwaddr,
mkdir,
write_file,
)
@@ -63,16 +68,22 @@ from charmhelpers.contrib.hahelpers.apache import (
)
from charmhelpers.contrib.openstack.neutron import (
neutron_plugin_attribute,
+ parse_data_port_mappings,
+)
+from charmhelpers.contrib.openstack.ip import (
+ resolve_address,
+ INTERNAL,
)
from charmhelpers.contrib.network.ip import (
get_address_in_network,
+ get_ipv4_addr,
get_ipv6_addr,
get_netmask_for_address,
format_ipv6_addr,
is_address_in_network,
+ is_bridge_member,
)
from charmhelpers.contrib.openstack.utils import get_host_ip
-
CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
ADDRESS_TYPES = ['admin', 'internal', 'public']
@@ -104,9 +115,44 @@ def context_complete(ctxt):
def config_flags_parser(config_flags):
"""Parses config flags string into dict.
+ This parsing method supports a few different formats for the config
+ flag values to be parsed:
+
+ 1. A string in the simple format of key=value pairs, with the possibility
+ of specifying multiple key value pairs within the same string. For
+ example, a string in the format of 'key1=value1, key2=value2' will
+ return a dict of:
+
+ {'key1': 'value1',
+ 'key2': 'value2'}.
+
+ 2. A string in the above format, but supporting a comma-delimited list
+ of values for the same key. For example, a string in the format of
+ 'key1=value1, key2=value3,value4,value5' will return a dict of:
+
+ {'key1', 'value1',
+ 'key2', 'value2,value3,value4'}
+
+ 3. A string containing a colon character (:) prior to an equal
+ character (=) will be treated as yaml and parsed as such. This can be
+ used to specify more complex key value pairs. For example,
+ a string in the format of 'key1: subkey1=value1, subkey2=value2' will
+ return a dict of:
+
+ {'key1', 'subkey1=value1, subkey2=value2'}
+
The provided config_flags string may be a list of comma-separated values
which themselves may be comma-separated list of values.
"""
+ # If we find a colon before an equals sign then treat it as yaml.
+ # Note: limit it to finding the colon first since this indicates assignment
+ # for inline yaml.
+ colon = config_flags.find(':')
+ equals = config_flags.find('=')
+ if colon > 0:
+ if colon < equals or equals < 0:
+ return yaml.safe_load(config_flags)
+
if config_flags.find('==') >= 0:
log("config_flags is not in expected format (key=value)", level=ERROR)
raise OSContextError
@@ -191,13 +237,13 @@ class SharedDBContext(OSContextGenerator):
unit=local_unit())
if set_hostname != access_hostname:
relation_set(relation_settings={hostname_key: access_hostname})
- return ctxt # Defer any further hook execution for now....
+ return None # Defer any further hook execution for now....
password_setting = 'password'
if self.relation_prefix:
password_setting = self.relation_prefix + '_password'
- for rid in relation_ids('shared-db'):
+ for rid in relation_ids(self.interfaces[0]):
for unit in related_units(rid):
rdata = relation_get(rid=rid, unit=unit)
host = rdata.get('db_host')
@@ -277,12 +323,29 @@ def db_ssl(rdata, ctxt, ssl_dir):
class IdentityServiceContext(OSContextGenerator):
- interfaces = ['identity-service']
+
+ def __init__(self, service=None, service_user=None, rel_name='identity-service'):
+ self.service = service
+ self.service_user = service_user
+ self.rel_name = rel_name
+ self.interfaces = [self.rel_name]
def __call__(self):
- log('Generating template context for identity-service', level=DEBUG)
+ log('Generating template context for ' + self.rel_name, level=DEBUG)
ctxt = {}
- for rid in relation_ids('identity-service'):
+
+ if self.service and self.service_user:
+ # This is required for pki token signing if we don't want /tmp to
+ # be used.
+ cachedir = '/var/cache/%s' % (self.service)
+ if not os.path.isdir(cachedir):
+ log("Creating service cache dir %s" % (cachedir), level=DEBUG)
+ mkdir(path=cachedir, owner=self.service_user,
+ group=self.service_user, perms=0o700)
+
+ ctxt['signing_dir'] = cachedir
+
+ for rid in relation_ids(self.rel_name):
for unit in related_units(rid):
rdata = relation_get(rid=rid, unit=unit)
serv_host = rdata.get('service_host')
@@ -291,15 +354,16 @@ class IdentityServiceContext(OSContextGenerator):
auth_host = format_ipv6_addr(auth_host) or auth_host
svc_protocol = rdata.get('service_protocol') or 'http'
auth_protocol = rdata.get('auth_protocol') or 'http'
- ctxt = {'service_port': rdata.get('service_port'),
- 'service_host': serv_host,
- 'auth_host': auth_host,
- 'auth_port': rdata.get('auth_port'),
- 'admin_tenant_name': rdata.get('service_tenant'),
- 'admin_user': rdata.get('service_username'),
- 'admin_password': rdata.get('service_password'),
- 'service_protocol': svc_protocol,
- 'auth_protocol': auth_protocol}
+ ctxt.update({'service_port': rdata.get('service_port'),
+ 'service_host': serv_host,
+ 'auth_host': auth_host,
+ 'auth_port': rdata.get('auth_port'),
+ 'admin_tenant_name': rdata.get('service_tenant'),
+ 'admin_user': rdata.get('service_username'),
+ 'admin_password': rdata.get('service_password'),
+ 'service_protocol': svc_protocol,
+ 'auth_protocol': auth_protocol})
+
if context_complete(ctxt):
# NOTE(jamespage) this is required for >= icehouse
# so a missing value just indicates keystone needs
@@ -398,6 +462,11 @@ class AMQPContext(OSContextGenerator):
ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts))
+ oslo_messaging_flags = conf.get('oslo-messaging-flags', None)
+ if oslo_messaging_flags:
+ ctxt['oslo_messaging_flags'] = config_flags_parser(
+ oslo_messaging_flags)
+
if not context_complete(ctxt):
return {}
@@ -677,7 +746,14 @@ class ApacheSSLContext(OSContextGenerator):
'endpoints': [],
'ext_ports': []}
- for cn in self.canonical_names():
+ cns = self.canonical_names()
+ if cns:
+ for cn in cns:
+ self.configure_cert(cn)
+ else:
+ # Expect cert/key provided in config (currently assumed that ca
+ # uses ip for cn)
+ cn = resolve_address(endpoint_type=INTERNAL)
self.configure_cert(cn)
addresses = self.get_network_addresses()
@@ -740,6 +816,19 @@ class NeutronContext(OSContextGenerator):
return ovs_ctxt
+ def nuage_ctxt(self):
+ driver = neutron_plugin_attribute(self.plugin, 'driver',
+ self.network_manager)
+ config = neutron_plugin_attribute(self.plugin, 'config',
+ self.network_manager)
+ nuage_ctxt = {'core_plugin': driver,
+ 'neutron_plugin': 'vsp',
+ 'neutron_security_groups': self.neutron_security_groups,
+ 'local_ip': unit_private_ip(),
+ 'config': config}
+
+ return nuage_ctxt
+
def nvp_ctxt(self):
driver = neutron_plugin_attribute(self.plugin, 'driver',
self.network_manager)
@@ -805,8 +894,6 @@ class NeutronContext(OSContextGenerator):
return ctxt
def __call__(self):
- self._ensure_packages()
-
if self.network_manager not in ['quantum', 'neutron']:
return {}
@@ -823,6 +910,8 @@ class NeutronContext(OSContextGenerator):
ctxt.update(self.n1kv_ctxt())
elif self.plugin == 'Calico':
ctxt.update(self.calico_ctxt())
+ elif self.plugin == 'vsp':
+ ctxt.update(self.nuage_ctxt())
alchemy_flags = config('neutron-alchemy-flags')
if alchemy_flags:
@@ -833,6 +922,48 @@ class NeutronContext(OSContextGenerator):
return ctxt
+class NeutronPortContext(OSContextGenerator):
+ NIC_PREFIXES = ['eth', 'bond']
+
+ def resolve_ports(self, ports):
+ """Resolve NICs not yet bound to bridge(s)
+
+ If hwaddress provided then returns resolved hwaddress otherwise NIC.
+ """
+ if not ports:
+ return None
+
+ hwaddr_to_nic = {}
+ hwaddr_to_ip = {}
+ for nic in list_nics(self.NIC_PREFIXES):
+ hwaddr = get_nic_hwaddr(nic)
+ hwaddr_to_nic[hwaddr] = nic
+ addresses = get_ipv4_addr(nic, fatal=False)
+ addresses += get_ipv6_addr(iface=nic, fatal=False)
+ hwaddr_to_ip[hwaddr] = addresses
+
+ resolved = []
+ mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I)
+ for entry in ports:
+ if re.match(mac_regex, entry):
+ # NIC is in known NICs and does NOT hace an IP address
+ if entry in hwaddr_to_nic and not hwaddr_to_ip[entry]:
+ # If the nic is part of a bridge then don't use it
+ if is_bridge_member(hwaddr_to_nic[entry]):
+ continue
+
+ # Entry is a MAC address for a valid interface that doesn't
+ # have an IP address assigned yet.
+ resolved.append(hwaddr_to_nic[entry])
+ else:
+ # If the passed entry is not a MAC address, assume it's a valid
+ # interface, and that the user put it there on purpose (we can
+ # trust it to be the real external network).
+ resolved.append(entry)
+
+ return resolved
+
+
class OSConfigFlagContext(OSContextGenerator):
"""Provides support for user-defined config flags.
@@ -1021,6 +1152,8 @@ class ZeroMQContext(OSContextGenerator):
for unit in related_units(rid):
ctxt['zmq_nonce'] = relation_get('nonce', unit, rid)
ctxt['zmq_host'] = relation_get('host', unit, rid)
+ ctxt['zmq_redis_address'] = relation_get(
+ 'zmq_redis_address', unit, rid)
return ctxt
@@ -1052,3 +1185,145 @@ class SysctlContext(OSContextGenerator):
sysctl_create(sysctl_dict,
'/etc/sysctl.d/50-{0}.conf'.format(charm_name()))
return {'sysctl': sysctl_dict}
+
+
+class NeutronAPIContext(OSContextGenerator):
+ '''
+ Inspects current neutron-plugin-api relation for neutron settings. Return
+ defaults if it is not present.
+ '''
+ interfaces = ['neutron-plugin-api']
+
+ def __call__(self):
+ self.neutron_defaults = {
+ 'l2_population': {
+ 'rel_key': 'l2-population',
+ 'default': False,
+ },
+ 'overlay_network_type': {
+ 'rel_key': 'overlay-network-type',
+ 'default': 'gre',
+ },
+ 'neutron_security_groups': {
+ 'rel_key': 'neutron-security-groups',
+ 'default': False,
+ },
+ 'network_device_mtu': {
+ 'rel_key': 'network-device-mtu',
+ 'default': None,
+ },
+ 'enable_dvr': {
+ 'rel_key': 'enable-dvr',
+ 'default': False,
+ },
+ 'enable_l3ha': {
+ 'rel_key': 'enable-l3ha',
+ 'default': False,
+ },
+ }
+ ctxt = self.get_neutron_options({})
+ for rid in relation_ids('neutron-plugin-api'):
+ for unit in related_units(rid):
+ rdata = relation_get(rid=rid, unit=unit)
+ if 'l2-population' in rdata:
+ ctxt.update(self.get_neutron_options(rdata))
+
+ return ctxt
+
+ def get_neutron_options(self, rdata):
+ settings = {}
+ for nkey in self.neutron_defaults.keys():
+ defv = self.neutron_defaults[nkey]['default']
+ rkey = self.neutron_defaults[nkey]['rel_key']
+ if rkey in rdata.keys():
+ if type(defv) is bool:
+ settings[nkey] = bool_from_string(rdata[rkey])
+ else:
+ settings[nkey] = rdata[rkey]
+ else:
+ settings[nkey] = defv
+ return settings
+
+
+class ExternalPortContext(NeutronPortContext):
+
+ def __call__(self):
+ ctxt = {}
+ ports = config('ext-port')
+ if ports:
+ ports = [p.strip() for p in ports.split()]
+ ports = self.resolve_ports(ports)
+ if ports:
+ ctxt = {"ext_port": ports[0]}
+ napi_settings = NeutronAPIContext()()
+ mtu = napi_settings.get('network_device_mtu')
+ if mtu:
+ ctxt['ext_port_mtu'] = mtu
+
+ return ctxt
+
+
+class DataPortContext(NeutronPortContext):
+
+ def __call__(self):
+ ports = config('data-port')
+ if ports:
+ portmap = parse_data_port_mappings(ports)
+ ports = portmap.values()
+ resolved = self.resolve_ports(ports)
+ normalized = {get_nic_hwaddr(port): port for port in resolved
+ if port not in ports}
+ normalized.update({port: port for port in resolved
+ if port in ports})
+ if resolved:
+ return {bridge: normalized[port] for bridge, port in
+ six.iteritems(portmap) if port in normalized.keys()}
+
+ return None
+
+
+class PhyNICMTUContext(DataPortContext):
+
+ def __call__(self):
+ ctxt = {}
+ mappings = super(PhyNICMTUContext, self).__call__()
+ if mappings and mappings.values():
+ ports = mappings.values()
+ napi_settings = NeutronAPIContext()()
+ mtu = napi_settings.get('network_device_mtu')
+ if mtu:
+ ctxt["devs"] = '\\n'.join(ports)
+ ctxt['mtu'] = mtu
+
+ return ctxt
+
+
+class NetworkServiceContext(OSContextGenerator):
+
+ def __init__(self, rel_name='quantum-network-service'):
+ self.rel_name = rel_name
+ self.interfaces = [rel_name]
+
+ def __call__(self):
+ for rid in relation_ids(self.rel_name):
+ for unit in related_units(rid):
+ rdata = relation_get(rid=rid, unit=unit)
+ ctxt = {
+ 'keystone_host': rdata.get('keystone_host'),
+ 'service_port': rdata.get('service_port'),
+ 'auth_port': rdata.get('auth_port'),
+ 'service_tenant': rdata.get('service_tenant'),
+ 'service_username': rdata.get('service_username'),
+ 'service_password': rdata.get('service_password'),
+ 'quantum_host': rdata.get('quantum_host'),
+ 'quantum_port': rdata.get('quantum_port'),
+ 'quantum_url': rdata.get('quantum_url'),
+ 'region': rdata.get('region'),
+ 'service_protocol':
+ rdata.get('service_protocol') or 'http',
+ 'auth_protocol':
+ rdata.get('auth_protocol') or 'http',
+ }
+ if context_complete(ctxt):
+ return ctxt
+ return {}
diff --git a/hooks/charmhelpers/contrib/openstack/files/__init__.py b/hooks/charmhelpers/contrib/openstack/files/__init__.py
new file mode 100644
index 0000000..7587679
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/files/__init__.py
@@ -0,0 +1,18 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers. If not, see .
+
+# dummy __init__.py to fool syncer into thinking this is a syncable python
+# module
diff --git a/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh b/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh
new file mode 100755
index 0000000..eb8527f
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#--------------------------------------------
+# This file is managed by Juju
+#--------------------------------------------
+#
+# Copyright 2009,2012 Canonical Ltd.
+# Author: Tom Haddon
+
+CRITICAL=0
+NOTACTIVE=''
+LOGFILE=/var/log/nagios/check_haproxy.log
+AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}')
+
+for appserver in $(grep ' server' /etc/haproxy/haproxy.cfg | awk '{print $2'});
+do
+ output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 --regex="class=\"(active|backup)(2|3).*${appserver}" -e ' 200 OK')
+ if [ $? != 0 ]; then
+ date >> $LOGFILE
+ echo $output >> $LOGFILE
+ /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -v | grep $appserver >> $LOGFILE 2>&1
+ CRITICAL=1
+ NOTACTIVE="${NOTACTIVE} $appserver"
+ fi
+done
+
+if [ $CRITICAL = 1 ]; then
+ echo "CRITICAL:${NOTACTIVE}"
+ exit 2
+fi
+
+echo "OK: All haproxy instances looking good"
+exit 0
diff --git a/hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh b/hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh
new file mode 100755
index 0000000..3ebb532
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+#--------------------------------------------
+# This file is managed by Juju
+#--------------------------------------------
+#
+# Copyright 2009,2012 Canonical Ltd.
+# Author: Tom Haddon
+
+# These should be config options at some stage
+CURRQthrsh=0
+MAXQthrsh=100
+
+AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}')
+
+HAPROXYSTATS=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' -v)
+
+for BACKEND in $(echo $HAPROXYSTATS| xargs -n1 | grep BACKEND | awk -F , '{print $1}')
+do
+ CURRQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 3)
+ MAXQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 4)
+
+ if [[ $CURRQ -gt $CURRQthrsh || $MAXQ -gt $MAXQthrsh ]] ; then
+ echo "CRITICAL: queue depth for $BACKEND - CURRENT:$CURRQ MAX:$MAXQ"
+ exit 2
+ fi
+done
+
+echo "OK: All haproxy queue depths looking good"
+exit 0
+
diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py
index 9eabed7..3dca6dc 100644
--- a/hooks/charmhelpers/contrib/openstack/ip.py
+++ b/hooks/charmhelpers/contrib/openstack/ip.py
@@ -17,6 +17,7 @@
from charmhelpers.core.hookenv import (
config,
unit_get,
+ service_name,
)
from charmhelpers.contrib.network.ip import (
get_address_in_network,
@@ -33,15 +34,18 @@ ADMIN = 'admin'
ADDRESS_MAP = {
PUBLIC: {
'config': 'os-public-network',
- 'fallback': 'public-address'
+ 'fallback': 'public-address',
+ 'override': 'os-public-hostname',
},
INTERNAL: {
'config': 'os-internal-network',
- 'fallback': 'private-address'
+ 'fallback': 'private-address',
+ 'override': 'os-internal-hostname',
},
ADMIN: {
'config': 'os-admin-network',
- 'fallback': 'private-address'
+ 'fallback': 'private-address',
+ 'override': 'os-admin-hostname',
}
}
@@ -55,15 +59,50 @@ def canonical_url(configs, endpoint_type=PUBLIC):
:param endpoint_type: str endpoint type to resolve.
:param returns: str base URL for services on the current service unit.
"""
- scheme = 'http'
- if 'https' in configs.complete_contexts():
- scheme = 'https'
+ scheme = _get_scheme(configs)
+
address = resolve_address(endpoint_type)
if is_ipv6(address):
address = "[{}]".format(address)
+
return '%s://%s' % (scheme, address)
+def _get_scheme(configs):
+ """Returns the scheme to use for the url (either http or https)
+ depending upon whether https is in the configs value.
+
+ :param configs: OSTemplateRenderer config templating object to inspect
+ for a complete https context.
+ :returns: either 'http' or 'https' depending on whether https is
+ configured within the configs context.
+ """
+ scheme = 'http'
+ if configs and 'https' in configs.complete_contexts():
+ scheme = 'https'
+ return scheme
+
+
+def _get_address_override(endpoint_type=PUBLIC):
+ """Returns any address overrides that the user has defined based on the
+ endpoint type.
+
+ Note: this function allows for the service name to be inserted into the
+ address if the user specifies {service_name}.somehost.org.
+
+ :param endpoint_type: the type of endpoint to retrieve the override
+ value for.
+ :returns: any endpoint address or hostname that the user has overridden
+ or None if an override is not present.
+ """
+ override_key = ADDRESS_MAP[endpoint_type]['override']
+ addr_override = config(override_key)
+ if not addr_override:
+ return None
+ else:
+ return addr_override.format(service_name=service_name())
+
+
def resolve_address(endpoint_type=PUBLIC):
"""Return unit address depending on net config.
@@ -75,7 +114,10 @@ def resolve_address(endpoint_type=PUBLIC):
:param endpoint_type: Network endpoing type
"""
- resolved_address = None
+ resolved_address = _get_address_override(endpoint_type)
+ if resolved_address:
+ return resolved_address
+
vips = config('vip')
if vips:
vips = vips.split()
diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py
index 902757f..f7b7235 100644
--- a/hooks/charmhelpers/contrib/openstack/neutron.py
+++ b/hooks/charmhelpers/contrib/openstack/neutron.py
@@ -16,6 +16,7 @@
# Various utilies for dealing with Neutron and the renaming from Quantum.
+import six
from subprocess import check_output
from charmhelpers.core.hookenv import (
@@ -171,13 +172,28 @@ def neutron_plugins():
'services': ['calico-felix',
'bird',
'neutron-dhcp-agent',
- 'nova-api-metadata'],
+ 'nova-api-metadata',
+ 'etcd'],
'packages': [[headers_package()] + determine_dkms_package(),
['calico-compute',
'bird',
'neutron-dhcp-agent',
- 'nova-api-metadata']],
- 'server_packages': ['neutron-server', 'calico-control'],
+ 'nova-api-metadata',
+ 'etcd']],
+ 'server_packages': ['neutron-server', 'calico-control', 'etcd'],
+ 'server_services': ['neutron-server', 'etcd']
+ },
+ 'vsp': {
+ 'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini',
+ 'driver': 'neutron.plugins.nuage.plugin.NuagePlugin',
+ 'contexts': [
+ context.SharedDBContext(user=config('neutron-database-user'),
+ database=config('neutron-database'),
+ relation_prefix='neutron',
+ ssl_dir=NEUTRON_CONF_DIR)],
+ 'services': [],
+ 'packages': [],
+ 'server_packages': ['neutron-server', 'neutron-plugin-nuage'],
'server_services': ['neutron-server']
}
}
@@ -237,3 +253,77 @@ def network_manager():
else:
# ensure accurate naming for all releases post-H
return 'neutron'
+
+
+def parse_mappings(mappings):
+ parsed = {}
+ if mappings:
+ mappings = mappings.split()
+ for m in mappings:
+ p = m.partition(':')
+ key = p[0].strip()
+ if p[1]:
+ parsed[key] = p[2].strip()
+ else:
+ parsed[key] = ''
+
+ return parsed
+
+
+def parse_bridge_mappings(mappings):
+ """Parse bridge mappings.
+
+ Mappings must be a space-delimited list of provider:bridge mappings.
+
+ Returns dict of the form {provider:bridge}.
+ """
+ return parse_mappings(mappings)
+
+
+def parse_data_port_mappings(mappings, default_bridge='br-data'):
+ """Parse data port mappings.
+
+ Mappings must be a space-delimited list of bridge:port mappings.
+
+ Returns dict of the form {bridge:port}.
+ """
+ _mappings = parse_mappings(mappings)
+ if not _mappings or list(_mappings.values()) == ['']:
+ if not mappings:
+ return {}
+
+ # For backwards-compatibility we need to support port-only provided in
+ # config.
+ _mappings = {default_bridge: mappings.split()[0]}
+
+ bridges = _mappings.keys()
+ ports = _mappings.values()
+ if len(set(bridges)) != len(bridges):
+ raise Exception("It is not allowed to have more than one port "
+ "configured on the same bridge")
+
+ if len(set(ports)) != len(ports):
+ raise Exception("It is not allowed to have the same port configured "
+ "on more than one bridge")
+
+ return _mappings
+
+
+def parse_vlan_range_mappings(mappings):
+ """Parse vlan range mappings.
+
+ Mappings must be a space-delimited list of provider:start:end mappings.
+
+ The start:end range is optional and may be omitted.
+
+ Returns dict of the form {provider: (start, end)}.
+ """
+ _mappings = parse_mappings(mappings)
+ if not _mappings:
+ return {}
+
+ mappings = {}
+ for p, r in six.iteritems(_mappings):
+ mappings[p] = tuple(r.split(':'))
+
+ return mappings
diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf
index 81a9719..b99851c 100644
--- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf
+++ b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf
@@ -5,11 +5,11 @@
###############################################################################
[global]
{% if auth -%}
- auth_supported = {{ auth }}
- keyring = /etc/ceph/$cluster.$name.keyring
- mon host = {{ mon_hosts }}
+auth_supported = {{ auth }}
+keyring = /etc/ceph/$cluster.$name.keyring
+mon host = {{ mon_hosts }}
{% endif -%}
- log to syslog = {{ use_syslog }}
- err to syslog = {{ use_syslog }}
- clog to syslog = {{ use_syslog }}
+log to syslog = {{ use_syslog }}
+err to syslog = {{ use_syslog }}
+clog to syslog = {{ use_syslog }}
diff --git a/hooks/charmhelpers/contrib/openstack/templates/git.upstart b/hooks/charmhelpers/contrib/openstack/templates/git.upstart
new file mode 100644
index 0000000..4bed404
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/templates/git.upstart
@@ -0,0 +1,17 @@
+description "{{ service_description }}"
+author "Juju {{ service_name }} Charm "
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+
+exec start-stop-daemon --start --chuid {{ user_name }} \
+ --chdir {{ start_dir }} --name {{ process_name }} \
+ --exec {{ executable_name }} -- \
+ {% for config_file in config_files -%}
+ --config-file={{ config_file }} \
+ {% endfor -%}
+ {% if log_file -%}
+ --log-file={{ log_file }}
+ {% endif -%}
diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken
new file mode 100644
index 0000000..2a37edd
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken
@@ -0,0 +1,9 @@
+{% if auth_host -%}
+[keystone_authtoken]
+identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/{{ auth_admin_prefix }}
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }}
+admin_tenant_name = {{ admin_tenant_name }}
+admin_user = {{ admin_user }}
+admin_password = {{ admin_password }}
+signing_dir = {{ signing_dir }}
+{% endif -%}
diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo b/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo
new file mode 100644
index 0000000..b444c9c
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo
@@ -0,0 +1,22 @@
+{% if rabbitmq_host or rabbitmq_hosts -%}
+[oslo_messaging_rabbit]
+rabbit_userid = {{ rabbitmq_user }}
+rabbit_virtual_host = {{ rabbitmq_virtual_host }}
+rabbit_password = {{ rabbitmq_password }}
+{% if rabbitmq_hosts -%}
+rabbit_hosts = {{ rabbitmq_hosts }}
+{% if rabbitmq_ha_queues -%}
+rabbit_ha_queues = True
+rabbit_durable_queues = False
+{% endif -%}
+{% else -%}
+rabbit_host = {{ rabbitmq_host }}
+{% endif -%}
+{% if rabbit_ssl_port -%}
+rabbit_use_ssl = True
+rabbit_port = {{ rabbit_ssl_port }}
+{% if rabbit_ssl_ca -%}
+kombu_ssl_ca_certs = {{ rabbit_ssl_ca }}
+{% endif -%}
+{% endif -%}
+{% endif -%}
diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-zeromq b/hooks/charmhelpers/contrib/openstack/templates/section-zeromq
new file mode 100644
index 0000000..95f1a76
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/templates/section-zeromq
@@ -0,0 +1,14 @@
+{% if zmq_host -%}
+# ZeroMQ configuration (restart-nonce: {{ zmq_nonce }})
+rpc_backend = zmq
+rpc_zmq_host = {{ zmq_host }}
+{% if zmq_redis_address -%}
+rpc_zmq_matchmaker = redis
+matchmaker_heartbeat_freq = 15
+matchmaker_heartbeat_ttl = 30
+[matchmaker_redis]
+host = {{ zmq_redis_address }}
+{% else -%}
+rpc_zmq_matchmaker = ring
+{% endif -%}
+{% endif -%}
diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index 26259a0..4dd000c 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -23,12 +23,17 @@ from functools import wraps
import subprocess
import json
import os
-import socket
import sys
import six
import yaml
+from charmhelpers.contrib.network import ip
+
+from charmhelpers.core import (
+ unitdata,
+)
+
from charmhelpers.core.hookenv import (
config,
log as juju_log,
@@ -48,9 +53,13 @@ from charmhelpers.contrib.network.ip import (
get_ipv6_addr
)
+from charmhelpers.contrib.python.packages import (
+ pip_create_virtualenv,
+ pip_install,
+)
+
from charmhelpers.core.host import lsb_release, mounts, umount
from charmhelpers.fetch import apt_install, apt_cache, install_remote
-from charmhelpers.contrib.python.packages import pip_install
from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
@@ -70,6 +79,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
('trusty', 'icehouse'),
('utopic', 'juno'),
('vivid', 'kilo'),
+ ('wily', 'liberty'),
])
@@ -82,6 +92,7 @@ OPENSTACK_CODENAMES = OrderedDict([
('2014.1', 'icehouse'),
('2014.2', 'juno'),
('2015.1', 'kilo'),
+ ('2015.2', 'liberty'),
])
# The ugly duckling
@@ -103,6 +114,8 @@ SWIFT_CODENAMES = OrderedDict([
('2.1.0', 'juno'),
('2.2.0', 'juno'),
('2.2.1', 'kilo'),
+ ('2.2.2', 'kilo'),
+ ('2.3.0', 'liberty'),
])
DEFAULT_LOOPBACK_SIZE = '5G'
@@ -311,6 +324,9 @@ def configure_installation_source(rel):
'kilo': 'trusty-updates/kilo',
'kilo/updates': 'trusty-updates/kilo',
'kilo/proposed': 'trusty-proposed/kilo',
+ 'liberty': 'trusty-updates/liberty',
+ 'liberty/updates': 'trusty-updates/liberty',
+ 'liberty/proposed': 'trusty-proposed/liberty',
}
try:
@@ -328,6 +344,21 @@ def configure_installation_source(rel):
error_out("Invalid openstack-release specified: %s" % rel)
+def config_value_changed(option):
+ """
+ Determine if config value changed since last call to this function.
+ """
+ hook_data = unitdata.HookData()
+ with hook_data():
+ db = unitdata.kv()
+ current = config(option)
+ saved = db.get(option)
+ db.set(option, current)
+ if saved is None:
+ return False
+ return current != saved
+
+
def save_script_rc(script_path="scripts/scriptrc", **env_vars):
"""
Write an rc file in the charm-delivered directory containing
@@ -420,77 +451,10 @@ def clean_storage(block_device):
else:
zap_disk(block_device)
-
-def is_ip(address):
- """
- Returns True if address is a valid IP address.
- """
- try:
- # Test to see if already an IPv4 address
- socket.inet_aton(address)
- return True
- except socket.error:
- return False
-
-
-def ns_query(address):
- try:
- import dns.resolver
- except ImportError:
- apt_install('python-dnspython')
- import dns.resolver
-
- if isinstance(address, dns.name.Name):
- rtype = 'PTR'
- elif isinstance(address, six.string_types):
- rtype = 'A'
- else:
- return None
-
- answers = dns.resolver.query(address, rtype)
- if answers:
- return str(answers[0])
- return None
-
-
-def get_host_ip(hostname):
- """
- Resolves the IP for a given hostname, or returns
- the input if it is already an IP.
- """
- if is_ip(hostname):
- return hostname
-
- return ns_query(hostname)
-
-
-def get_hostname(address, fqdn=True):
- """
- Resolves hostname for given IP, or returns the input
- if it is already a hostname.
- """
- if is_ip(address):
- try:
- import dns.reversename
- except ImportError:
- apt_install('python-dnspython')
- import dns.reversename
-
- rev = dns.reversename.from_address(address)
- result = ns_query(rev)
- if not result:
- return None
- else:
- result = address
-
- if fqdn:
- # strip trailing .
- if result.endswith('.'):
- return result[:-1]
- else:
- return result
- else:
- return result.split('.')[0]
+is_ip = ip.is_ip
+ns_query = ip.ns_query
+get_host_ip = ip.get_host_ip
+get_hostname = ip.get_hostname
def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'):
@@ -534,108 +498,209 @@ def os_requires_version(ostack_release, pkg):
def git_install_requested():
- """Returns true if openstack-origin-git is specified."""
- return config('openstack-origin-git') != "None"
+ """
+ Returns true if openstack-origin-git is specified.
+ """
+ return config('openstack-origin-git') is not None
requirements_dir = None
-def git_clone_and_install(file_name, core_project):
- """Clone/install all OpenStack repos specified in yaml config file."""
+def _git_yaml_load(projects_yaml):
+ """
+ Load the specified yaml into a dictionary.
+ """
+ if not projects_yaml:
+ return None
+
+ return yaml.load(projects_yaml)
+
+
+def git_clone_and_install(projects_yaml, core_project, depth=1):
+ """
+ Clone/install all specified OpenStack repositories.
+
+ The expected format of projects_yaml is:
+
+ repositories:
+ - {name: keystone,
+ repository: 'git://git.openstack.org/openstack/keystone.git',
+ branch: 'stable/icehouse'}
+ - {name: requirements,
+ repository: 'git://git.openstack.org/openstack/requirements.git',
+ branch: 'stable/icehouse'}
+
+ directory: /mnt/openstack-git
+ http_proxy: squid-proxy-url
+ https_proxy: squid-proxy-url
+
+ The directory, http_proxy, and https_proxy keys are optional.
+
+ """
global requirements_dir
+ parent_dir = '/mnt/openstack-git'
+ http_proxy = None
- if file_name == "None":
- return
+ projects = _git_yaml_load(projects_yaml)
+ _git_validate_projects_yaml(projects, core_project)
- yaml_file = os.path.join(charm_dir(), file_name)
+ old_environ = dict(os.environ)
- # clone/install the requirements project first
- installed = _git_clone_and_install_subset(yaml_file,
- whitelist=['requirements'])
- if 'requirements' not in installed:
- error_out('requirements git repository must be specified')
+ if 'http_proxy' in projects.keys():
+ http_proxy = projects['http_proxy']
+ os.environ['http_proxy'] = projects['http_proxy']
+ if 'https_proxy' in projects.keys():
+ os.environ['https_proxy'] = projects['https_proxy']
- # clone/install all other projects except requirements and the core project
- blacklist = ['requirements', core_project]
- _git_clone_and_install_subset(yaml_file, blacklist=blacklist,
- update_requirements=True)
+ if 'directory' in projects.keys():
+ parent_dir = projects['directory']
- # clone/install the core project
- whitelist = [core_project]
- installed = _git_clone_and_install_subset(yaml_file, whitelist=whitelist,
- update_requirements=True)
- if core_project not in installed:
- error_out('{} git repository must be specified'.format(core_project))
+ pip_create_virtualenv(os.path.join(parent_dir, 'venv'))
+
+ # Upgrade setuptools and pip from default virtualenv versions. The default
+ # versions in trusty break master OpenStack branch deployments.
+ for p in ['pip', 'setuptools']:
+ pip_install(p, upgrade=True, proxy=http_proxy,
+ venv=os.path.join(parent_dir, 'venv'))
+
+ for p in projects['repositories']:
+ repo = p['repository']
+ branch = p['branch']
+ if p['name'] == 'requirements':
+ repo_dir = _git_clone_and_install_single(repo, branch, depth,
+ parent_dir, http_proxy,
+ update_requirements=False)
+ requirements_dir = repo_dir
+ else:
+ repo_dir = _git_clone_and_install_single(repo, branch, depth,
+ parent_dir, http_proxy,
+ update_requirements=True)
+
+ os.environ = old_environ
-def _git_clone_and_install_subset(yaml_file, whitelist=[], blacklist=[],
- update_requirements=False):
- """Clone/install subset of OpenStack repos specified in yaml config file."""
- global requirements_dir
- installed = []
+def _git_validate_projects_yaml(projects, core_project):
+ """
+ Validate the projects yaml.
+ """
+ _git_ensure_key_exists('repositories', projects)
- with open(yaml_file, 'r') as fd:
- projects = yaml.load(fd)
- for proj, val in projects.items():
- # The project subset is chosen based on the following 3 rules:
- # 1) If project is in blacklist, we don't clone/install it, period.
- # 2) If whitelist is empty, we clone/install everything else.
- # 3) If whitelist is not empty, we clone/install everything in the
- # whitelist.
- if proj in blacklist:
- continue
- if whitelist and proj not in whitelist:
- continue
- repo = val['repository']
- branch = val['branch']
- repo_dir = _git_clone_and_install_single(repo, branch,
- update_requirements)
- if proj == 'requirements':
- requirements_dir = repo_dir
- installed.append(proj)
- return installed
+ for project in projects['repositories']:
+ _git_ensure_key_exists('name', project.keys())
+ _git_ensure_key_exists('repository', project.keys())
+ _git_ensure_key_exists('branch', project.keys())
+
+ if projects['repositories'][0]['name'] != 'requirements':
+ error_out('{} git repo must be specified first'.format('requirements'))
+
+ if projects['repositories'][-1]['name'] != core_project:
+ error_out('{} git repo must be specified last'.format(core_project))
-def _git_clone_and_install_single(repo, branch, update_requirements=False):
- """Clone and install a single git repository."""
- dest_parent_dir = "/mnt/openstack-git/"
- dest_dir = os.path.join(dest_parent_dir, os.path.basename(repo))
+def _git_ensure_key_exists(key, keys):
+ """
+ Ensure that key exists in keys.
+ """
+ if key not in keys:
+ error_out('openstack-origin-git key \'{}\' is missing'.format(key))
- if not os.path.exists(dest_parent_dir):
- juju_log('Host dir not mounted at {}. '
- 'Creating directory there instead.'.format(dest_parent_dir))
- os.mkdir(dest_parent_dir)
+
+def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
+ update_requirements):
+ """
+ Clone and install a single git repository.
+ """
+ dest_dir = os.path.join(parent_dir, os.path.basename(repo))
+
+ if not os.path.exists(parent_dir):
+ juju_log('Directory already exists at {}. '
+ 'No need to create directory.'.format(parent_dir))
+ os.mkdir(parent_dir)
if not os.path.exists(dest_dir):
juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
- repo_dir = install_remote(repo, dest=dest_parent_dir, branch=branch)
+ repo_dir = install_remote(repo, dest=parent_dir, branch=branch,
+ depth=depth)
else:
repo_dir = dest_dir
+ venv = os.path.join(parent_dir, 'venv')
+
if update_requirements:
if not requirements_dir:
error_out('requirements repo must be cloned before '
'updating from global requirements.')
- _git_update_requirements(repo_dir, requirements_dir)
+ _git_update_requirements(venv, repo_dir, requirements_dir)
juju_log('Installing git repo from dir: {}'.format(repo_dir))
- pip_install(repo_dir)
+ if http_proxy:
+ pip_install(repo_dir, proxy=http_proxy, venv=venv)
+ else:
+ pip_install(repo_dir, venv=venv)
return repo_dir
-def _git_update_requirements(package_dir, reqs_dir):
- """Update from global requirements.
+def _git_update_requirements(venv, package_dir, reqs_dir):
+ """
+ Update from global requirements.
- Update an OpenStack git directory's requirements.txt and
- test-requirements.txt from global-requirements.txt."""
+ Update an OpenStack git directory's requirements.txt and
+ test-requirements.txt from global-requirements.txt.
+ """
orig_dir = os.getcwd()
os.chdir(reqs_dir)
- cmd = "python update.py {}".format(package_dir)
+ python = os.path.join(venv, 'bin/python')
+ cmd = [python, 'update.py', package_dir]
try:
- subprocess.check_call(cmd.split(' '))
+ subprocess.check_call(cmd)
except subprocess.CalledProcessError:
package = os.path.basename(package_dir)
- error_out("Error updating {} from global-requirements.txt".format(package))
+ error_out("Error updating {} from "
+ "global-requirements.txt".format(package))
os.chdir(orig_dir)
+
+
+def git_pip_venv_dir(projects_yaml):
+ """
+ Return the pip virtualenv path.
+ """
+ parent_dir = '/mnt/openstack-git'
+
+ projects = _git_yaml_load(projects_yaml)
+
+ if 'directory' in projects.keys():
+ parent_dir = projects['directory']
+
+ return os.path.join(parent_dir, 'venv')
+
+
+def git_src_dir(projects_yaml, project):
+ """
+ Return the directory where the specified project's source is located.
+ """
+ parent_dir = '/mnt/openstack-git'
+
+ projects = _git_yaml_load(projects_yaml)
+
+ if 'directory' in projects.keys():
+ parent_dir = projects['directory']
+
+ for p in projects['repositories']:
+ if p['name'] == project:
+ return os.path.join(parent_dir, os.path.basename(p['repository']))
+
+ return None
+
+
+def git_yaml_value(projects_yaml, key):
+ """
+ Return the value in projects_yaml for the specified key.
+ """
+ projects = _git_yaml_load(projects_yaml)
+
+ if key in projects.keys():
+ return projects[key]
+
+ return None
diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py
index 31ea7f9..00dbffb 100644
--- a/hooks/charmhelpers/contrib/storage/linux/ceph.py
+++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py
@@ -60,12 +60,12 @@ KEYRING = '/etc/ceph/ceph.client.{}.keyring'
KEYFILE = '/etc/ceph/ceph.client.{}.key'
CEPH_CONF = """[global]
- auth supported = {auth}
- keyring = {keyring}
- mon host = {mon_hosts}
- log to syslog = {use_syslog}
- err to syslog = {use_syslog}
- clog to syslog = {use_syslog}
+auth supported = {auth}
+keyring = {keyring}
+mon host = {mon_hosts}
+log to syslog = {use_syslog}
+err to syslog = {use_syslog}
+clog to syslog = {use_syslog}
"""
diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py
index 9cdcc88..3056fba 100644
--- a/hooks/charmhelpers/core/fstab.py
+++ b/hooks/charmhelpers/core/fstab.py
@@ -77,7 +77,7 @@ class Fstab(io.FileIO):
for line in self.readlines():
line = line.decode('us-ascii')
try:
- if line.strip() and not line.startswith("#"):
+ if line.strip() and not line.strip().startswith("#"):
yield self._hydrate_entry(line)
except ValueError:
pass
@@ -104,7 +104,7 @@ class Fstab(io.FileIO):
found = False
for index, line in enumerate(lines):
- if not line.startswith("#"):
+ if line.strip() and not line.strip().startswith("#"):
if self._hydrate_entry(line) == entry:
found = True
break
diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
index cf552b3..dd8def9 100644
--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -20,11 +20,17 @@
# Authors:
# Charm Helpers Developers
+from __future__ import print_function
+from distutils.version import LooseVersion
+from functools import wraps
+import glob
import os
import json
import yaml
import subprocess
import sys
+import errno
+import tempfile
from subprocess import CalledProcessError
import six
@@ -56,15 +62,17 @@ def cached(func):
will cache the result of unit_get + 'test' for future calls.
"""
+ @wraps(func)
def wrapper(*args, **kwargs):
global cache
key = str((func, args, kwargs))
try:
return cache[key]
except KeyError:
- res = func(*args, **kwargs)
- cache[key] = res
- return res
+ pass # Drop out of the exception handler scope.
+ res = func(*args, **kwargs)
+ cache[key] = res
+ return res
return wrapper
@@ -87,7 +95,18 @@ def log(message, level=None):
if not isinstance(message, six.string_types):
message = repr(message)
command += [message]
- subprocess.call(command)
+ # Missing juju-log should not cause failures in unit tests
+ # Send log output to stderr
+ try:
+ subprocess.call(command)
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ if level:
+ message = "{}: {}".format(level, message)
+ message = "juju-log: {}".format(message)
+ print(message, file=sys.stderr)
+ else:
+ raise
class Serializable(UserDict):
@@ -165,7 +184,7 @@ def local_unit():
def remote_unit():
"""The remote unit for the current relation hook"""
- return os.environ['JUJU_REMOTE_UNIT']
+ return os.environ.get('JUJU_REMOTE_UNIT', None)
def service_name():
@@ -225,23 +244,7 @@ class Config(dict):
self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
if os.path.exists(self.path):
self.load_previous()
-
- def __getitem__(self, key):
- """For regular dict lookups, check the current juju config first,
- then the previous (saved) copy. This ensures that user-saved values
- will be returned by a dict lookup.
-
- """
- try:
- return dict.__getitem__(self, key)
- except KeyError:
- return (self._prev_dict or {})[key]
-
- def keys(self):
- prev_keys = []
- if self._prev_dict is not None:
- prev_keys = self._prev_dict.keys()
- return list(set(prev_keys + list(dict.keys(self))))
+ atexit(self._implicit_save)
def load_previous(self, path=None):
"""Load previous copy of config from disk.
@@ -260,6 +263,9 @@ class Config(dict):
self.path = path or self.path
with open(self.path) as f:
self._prev_dict = json.load(f)
+ for k, v in self._prev_dict.items():
+ if k not in self:
+ self[k] = v
def changed(self, key):
"""Return True if the current value for this key is different from
@@ -291,13 +297,13 @@ class Config(dict):
instance.
"""
- if self._prev_dict:
- for k, v in six.iteritems(self._prev_dict):
- if k not in self:
- self[k] = v
with open(self.path, 'w') as f:
json.dump(self, f)
+ def _implicit_save(self):
+ if self.implicit_save:
+ self.save()
+
@cached
def config(scope=None):
@@ -340,18 +346,49 @@ def relation_set(relation_id=None, relation_settings=None, **kwargs):
"""Set relation information for the current unit"""
relation_settings = relation_settings if relation_settings else {}
relation_cmd_line = ['relation-set']
+ accepts_file = "--file" in subprocess.check_output(
+ relation_cmd_line + ["--help"], universal_newlines=True)
if relation_id is not None:
relation_cmd_line.extend(('-r', relation_id))
- for k, v in (list(relation_settings.items()) + list(kwargs.items())):
- if v is None:
- relation_cmd_line.append('{}='.format(k))
- else:
- relation_cmd_line.append('{}={}'.format(k, v))
- subprocess.check_call(relation_cmd_line)
+ settings = relation_settings.copy()
+ settings.update(kwargs)
+ for key, value in settings.items():
+ # Force value to be a string: it always should, but some call
+ # sites pass in things like dicts or numbers.
+ if value is not None:
+ settings[key] = "{}".format(value)
+ if accepts_file:
+ # --file was introduced in Juju 1.23.2. Use it by default if
+ # available, since otherwise we'll break if the relation data is
+ # too big. Ideally we should tell relation-set to read the data from
+ # stdin, but that feature is broken in 1.23.2: Bug #1454678.
+ with tempfile.NamedTemporaryFile(delete=False) as settings_file:
+ settings_file.write(yaml.safe_dump(settings).encode("utf-8"))
+ subprocess.check_call(
+ relation_cmd_line + ["--file", settings_file.name])
+ os.remove(settings_file.name)
+ else:
+ for key, value in settings.items():
+ if value is None:
+ relation_cmd_line.append('{}='.format(key))
+ else:
+ relation_cmd_line.append('{}={}'.format(key, value))
+ subprocess.check_call(relation_cmd_line)
# Flush cache of any relation-gets for local unit
flush(local_unit())
+def relation_clear(r_id=None):
+ ''' Clears any relation data already set on relation r_id '''
+ settings = relation_get(rid=r_id,
+ unit=local_unit())
+ for setting in settings:
+ if setting not in ['public-address', 'private-address']:
+ settings[setting] = None
+ relation_set(relation_id=r_id,
+ **settings)
+
+
@cached
def relation_ids(reltype=None):
"""A list of relation_ids"""
@@ -496,6 +533,11 @@ def unit_get(attribute):
return None
+def unit_public_ip():
+ """Get this unit's public IP address"""
+ return unit_get('public-address')
+
+
def unit_private_ip():
"""Get this unit's private IP address"""
return unit_get('private-address')
@@ -528,10 +570,14 @@ class Hooks(object):
hooks.execute(sys.argv)
"""
- def __init__(self, config_save=True):
+ def __init__(self, config_save=None):
super(Hooks, self).__init__()
self._hooks = {}
- self._config_save = config_save
+
+ # For unknown reasons, we allow the Hooks constructor to override
+ # config().implicit_save.
+ if config_save is not None:
+ config().implicit_save = config_save
def register(self, name, function):
"""Register a hook"""
@@ -539,13 +585,16 @@ class Hooks(object):
def execute(self, args):
"""Execute a registered hook based on args[0]"""
+ _run_atstart()
hook_name = os.path.basename(args[0])
if hook_name in self._hooks:
- self._hooks[hook_name]()
- if self._config_save:
- cfg = config()
- if cfg.implicit_save:
- cfg.save()
+ try:
+ self._hooks[hook_name]()
+ except SystemExit as x:
+ if x.code is None or x.code == 0:
+ _run_atexit()
+ raise
+ _run_atexit()
else:
raise UnregisteredHookError(hook_name)
@@ -566,3 +615,187 @@ class Hooks(object):
def charm_dir():
"""Return the root directory of the current charm"""
return os.environ.get('CHARM_DIR')
+
+
+@cached
+def action_get(key=None):
+ """Gets the value of an action parameter, or all key/value param pairs"""
+ cmd = ['action-get']
+ if key is not None:
+ cmd.append(key)
+ cmd.append('--format=json')
+ action_data = json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+ return action_data
+
+
+def action_set(values):
+ """Sets the values to be returned after the action finishes"""
+ cmd = ['action-set']
+ for k, v in list(values.items()):
+ cmd.append('{}={}'.format(k, v))
+ subprocess.check_call(cmd)
+
+
+def action_fail(message):
+ """Sets the action status to failed and sets the error message.
+
+ The results set by action_set are preserved."""
+ subprocess.check_call(['action-fail', message])
+
+
+def status_set(workload_state, message):
+ """Set the workload state with a message
+
+ Use status-set to set the workload state with a message which is visible
+ to the user via juju status. If the status-set command is not found then
+ assume this is juju < 1.23 and juju-log the message unstead.
+
+ workload_state -- valid juju workload state.
+ message -- status update message
+ """
+ valid_states = ['maintenance', 'blocked', 'waiting', 'active']
+ if workload_state not in valid_states:
+ raise ValueError(
+ '{!r} is not a valid workload state'.format(workload_state)
+ )
+ cmd = ['status-set', workload_state, message]
+ try:
+ ret = subprocess.call(cmd)
+ if ret == 0:
+ return
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ log_message = 'status-set failed: {} {}'.format(workload_state,
+ message)
+ log(log_message, level='INFO')
+
+
+def status_get():
+ """Retrieve the previously set juju workload state
+
+ If the status-set command is not found then assume this is juju < 1.23 and
+ return 'unknown'
+ """
+ cmd = ['status-get']
+ try:
+ raw_status = subprocess.check_output(cmd, universal_newlines=True)
+ status = raw_status.rstrip()
+ return status
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ return 'unknown'
+ else:
+ raise
+
+
+def translate_exc(from_exc, to_exc):
+ def inner_translate_exc1(f):
+ def inner_translate_exc2(*args, **kwargs):
+ try:
+ return f(*args, **kwargs)
+ except from_exc:
+ raise to_exc
+
+ return inner_translate_exc2
+
+ return inner_translate_exc1
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def is_leader():
+ """Does the current unit hold the juju leadership
+
+ Uses juju to determine whether the current unit is the leader of its peers
+ """
+ cmd = ['is-leader', '--format=json']
+ return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_get(attribute=None):
+ """Juju leader get value(s)"""
+ cmd = ['leader-get', '--format=json'] + [attribute or '-']
+ return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_set(settings=None, **kwargs):
+ """Juju leader set value(s)"""
+ # Don't log secrets.
+ # log("Juju leader-set '%s'" % (settings), level=DEBUG)
+ cmd = ['leader-set']
+ settings = settings or {}
+ settings.update(kwargs)
+ for k, v in settings.items():
+ if v is None:
+ cmd.append('{}='.format(k))
+ else:
+ cmd.append('{}={}'.format(k, v))
+ subprocess.check_call(cmd)
+
+
+@cached
+def juju_version():
+ """Full version string (eg. '1.23.3.1-trusty-amd64')"""
+ # Per https://bugs.launchpad.net/juju-core/+bug/1455368/comments/1
+ jujud = glob.glob('/var/lib/juju/tools/machine-*/jujud')[0]
+ return subprocess.check_output([jujud, 'version'],
+ universal_newlines=True).strip()
+
+
+@cached
+def has_juju_version(minimum_version):
+ """Return True if the Juju version is at least the provided version"""
+ return LooseVersion(juju_version()) >= LooseVersion(minimum_version)
+
+
+_atexit = []
+_atstart = []
+
+
+def atstart(callback, *args, **kwargs):
+ '''Schedule a callback to run before the main hook.
+
+ Callbacks are run in the order they were added.
+
+ This is useful for modules and classes to perform initialization
+ and inject behavior. In particular:
+
+ - Run common code before all of your hooks, such as logging
+ the hook name or interesting relation data.
+ - Defer object or module initialization that requires a hook
+ context until we know there actually is a hook context,
+ making testing easier.
+ - Rather than requiring charm authors to include boilerplate to
+ invoke your helper's behavior, have it run automatically if
+ your object is instantiated or module imported.
+
+ This is not at all useful after your hook framework as been launched.
+ '''
+ global _atstart
+ _atstart.append((callback, args, kwargs))
+
+
+def atexit(callback, *args, **kwargs):
+ '''Schedule a callback to run on successful hook completion.
+
+ Callbacks are run in the reverse order that they were added.'''
+ _atexit.append((callback, args, kwargs))
+
+
+def _run_atstart():
+ '''Hook frameworks must invoke this before running the main hook body.'''
+ global _atstart
+ for callback, args, kwargs in _atstart:
+ callback(*args, **kwargs)
+ del _atstart[:]
+
+
+def _run_atexit():
+ '''Hook frameworks must invoke this after the main hook body has
+ successfully completed. Do not invoke it if the hook fails.'''
+ global _atexit
+ for callback, args, kwargs in reversed(_atexit):
+ callback(*args, **kwargs)
+ del _atexit[:]
diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py
index b771c61..8ae8ef8 100644
--- a/hooks/charmhelpers/core/host.py
+++ b/hooks/charmhelpers/core/host.py
@@ -24,6 +24,7 @@
import os
import re
import pwd
+import glob
import grp
import random
import string
@@ -62,6 +63,36 @@ def service_reload(service_name, restart_on_failure=False):
return service_result
+def service_pause(service_name, init_dir=None):
+ """Pause a system service.
+
+ Stop it, and prevent it from starting again at boot."""
+ if init_dir is None:
+ init_dir = "/etc/init"
+ stopped = service_stop(service_name)
+ # XXX: Support systemd too
+ override_path = os.path.join(
+ init_dir, '{}.conf.override'.format(service_name))
+ with open(override_path, 'w') as fh:
+ fh.write("manual\n")
+ return stopped
+
+
+def service_resume(service_name, init_dir=None):
+ """Resume a system service.
+
+ Reenable starting again at boot. Start the service"""
+ # XXX: Support systemd too
+ if init_dir is None:
+ init_dir = "/etc/init"
+ override_path = os.path.join(
+ init_dir, '{}.conf.override'.format(service_name))
+ if os.path.exists(override_path):
+ os.unlink(override_path)
+ started = service_start(service_name)
+ return started
+
+
def service(action, service_name):
"""Control a system service"""
cmd = ['service', service_name, action]
@@ -90,7 +121,7 @@ def service_available(service_name):
['service', service_name, 'status'],
stderr=subprocess.STDOUT).decode('UTF-8')
except subprocess.CalledProcessError as e:
- return 'unrecognized service' not in e.output
+ return b'unrecognized service' not in e.output
else:
return True
@@ -139,11 +170,7 @@ def add_group(group_name, system_group=False):
def add_user_to_group(username, group):
"""Add a user to a group"""
- cmd = [
- 'gpasswd', '-a',
- username,
- group
- ]
+ cmd = ['gpasswd', '-a', username, group]
log("Adding user {} to group {}".format(username, group))
subprocess.check_call(cmd)
@@ -269,6 +296,21 @@ def file_hash(path, hash_type='md5'):
return None
+def path_hash(path):
+ """
+ Generate a hash checksum of all files matching 'path'. Standard wildcards
+ like '*' and '?' are supported, see documentation for the 'glob' module for
+ more information.
+
+ :return: dict: A { filename: hash } dictionary for all matched files.
+ Empty if none found.
+ """
+ return {
+ filename: file_hash(filename)
+ for filename in glob.iglob(path)
+ }
+
+
def check_hash(path, checksum, hash_type='md5'):
"""
Validate a file using a cryptographic checksum.
@@ -296,23 +338,25 @@ def restart_on_change(restart_map, stopstart=False):
@restart_on_change({
'/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ]
+ '/etc/apache/sites-enabled/*': [ 'apache2' ]
})
- def ceph_client_changed():
+ def config_changed():
pass # your code here
In this example, the cinder-api and cinder-volume services
would be restarted if /etc/ceph/ceph.conf is changed by the
- ceph_client_changed function.
+ ceph_client_changed function. The apache2 service would be
+ restarted if any file matching the pattern got changed, created
+ or removed. Standard wildcards are supported, see documentation
+ for the 'glob' module for more information.
"""
def wrap(f):
def wrapped_f(*args, **kwargs):
- checksums = {}
- for path in restart_map:
- checksums[path] = file_hash(path)
+ checksums = {path: path_hash(path) for path in restart_map}
f(*args, **kwargs)
restarts = []
for path in restart_map:
- if checksums[path] != file_hash(path):
+ if path_hash(path) != checksums[path]:
restarts += restart_map[path]
services_list = list(OrderedDict.fromkeys(restarts))
if not stopstart:
@@ -339,12 +383,16 @@ def lsb_release():
def pwgen(length=None):
"""Generate a random pasword."""
if length is None:
+ # A random length is ok to use a weak PRNG
length = random.choice(range(35, 45))
alphanumeric_chars = [
l for l in (string.ascii_letters + string.digits)
if l not in 'l0QD1vAEIOUaeiou']
+ # Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the
+ # actual password
+ random_generator = random.SystemRandom()
random_chars = [
- random.choice(alphanumeric_chars) for _ in range(length)]
+ random_generator.choice(alphanumeric_chars) for _ in range(length)]
return(''.join(random_chars))
diff --git a/hooks/charmhelpers/core/services/base.py b/hooks/charmhelpers/core/services/base.py
index c5534e4..a42660c 100644
--- a/hooks/charmhelpers/core/services/base.py
+++ b/hooks/charmhelpers/core/services/base.py
@@ -15,9 +15,9 @@
# along with charm-helpers. If not, see .
import os
-import re
import json
-from collections import Iterable
+from inspect import getargspec
+from collections import Iterable, OrderedDict
from charmhelpers.core import host
from charmhelpers.core import hookenv
@@ -119,7 +119,7 @@ class ServiceManager(object):
"""
self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json')
self._ready = None
- self.services = {}
+ self.services = OrderedDict()
for service in services or []:
service_name = service['service']
self.services[service_name] = service
@@ -128,15 +128,18 @@ class ServiceManager(object):
"""
Handle the current hook by doing The Right Thing with the registered services.
"""
- hook_name = hookenv.hook_name()
- if hook_name == 'stop':
- self.stop_services()
- else:
- self.provide_data()
- self.reconfigure_services()
- cfg = hookenv.config()
- if cfg.implicit_save:
- cfg.save()
+ hookenv._run_atstart()
+ try:
+ hook_name = hookenv.hook_name()
+ if hook_name == 'stop':
+ self.stop_services()
+ else:
+ self.reconfigure_services()
+ self.provide_data()
+ except SystemExit as x:
+ if x.code is None or x.code == 0:
+ hookenv._run_atexit()
+ hookenv._run_atexit()
def provide_data(self):
"""
@@ -145,15 +148,36 @@ class ServiceManager(object):
A provider must have a `name` attribute, which indicates which relation
to set data on, and a `provide_data()` method, which returns a dict of
data to set.
+
+ The `provide_data()` method can optionally accept two parameters:
+
+ * ``remote_service`` The name of the remote service that the data will
+ be provided to. The `provide_data()` method will be called once
+ for each connected service (not unit). This allows the method to
+ tailor its data to the given service.
+ * ``service_ready`` Whether or not the service definition had all of
+ its requirements met, and thus the ``data_ready`` callbacks run.
+
+ Note that the ``provided_data`` methods are now called **after** the
+ ``data_ready`` callbacks are run. This gives the ``data_ready`` callbacks
+ a chance to generate any data necessary for the providing to the remote
+ services.
"""
- hook_name = hookenv.hook_name()
- for service in self.services.values():
+ for service_name, service in self.services.items():
+ service_ready = self.is_ready(service_name)
for provider in service.get('provided_data', []):
- if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name):
- data = provider.provide_data()
- _ready = provider._is_ready(data) if hasattr(provider, '_is_ready') else data
- if _ready:
- hookenv.relation_set(None, data)
+ for relid in hookenv.relation_ids(provider.name):
+ units = hookenv.related_units(relid)
+ if not units:
+ continue
+ remote_service = units[0].split('/')[0]
+ argspec = getargspec(provider.provide_data)
+ if len(argspec.args) > 1:
+ data = provider.provide_data(remote_service, service_ready)
+ else:
+ data = provider.provide_data()
+ if data:
+ hookenv.relation_set(relid, data)
def reconfigure_services(self, *service_names):
"""
diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py
index 5e3af9d..8005c41 100644
--- a/hooks/charmhelpers/core/services/helpers.py
+++ b/hooks/charmhelpers/core/services/helpers.py
@@ -45,12 +45,14 @@ class RelationContext(dict):
"""
name = None
interface = None
- required_keys = []
def __init__(self, name=None, additional_required_keys=None):
+ if not hasattr(self, 'required_keys'):
+ self.required_keys = []
+
if name is not None:
self.name = name
- if additional_required_keys is not None:
+ if additional_required_keys:
self.required_keys.extend(additional_required_keys)
self.get_data()
@@ -134,7 +136,10 @@ class MysqlRelation(RelationContext):
"""
name = 'db'
interface = 'mysql'
- required_keys = ['host', 'user', 'password', 'database']
+
+ def __init__(self, *args, **kwargs):
+ self.required_keys = ['host', 'user', 'password', 'database']
+ RelationContext.__init__(self, *args, **kwargs)
class HttpRelation(RelationContext):
@@ -146,7 +151,10 @@ class HttpRelation(RelationContext):
"""
name = 'website'
interface = 'http'
- required_keys = ['host', 'port']
+
+ def __init__(self, *args, **kwargs):
+ self.required_keys = ['host', 'port']
+ RelationContext.__init__(self, *args, **kwargs)
def provide_data(self):
return {
@@ -231,12 +239,12 @@ class TemplateCallback(ManagerCallback):
action.
:param str source: The template source file, relative to
- `$CHARM_DIR/templates`
-
+ `$CHARM_DIR/templates`
:param str target: The target to write the rendered template to
:param str owner: The owner of the rendered file
:param str group: The group of the rendered file
:param int perms: The permissions of the rendered file
+
"""
def __init__(self, source, target,
owner='root', group='root', perms=0o444):
diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py
index efc4402..a2a784a 100644
--- a/hooks/charmhelpers/core/strutils.py
+++ b/hooks/charmhelpers/core/strutils.py
@@ -33,9 +33,9 @@ def bool_from_string(value):
value = value.strip().lower()
- if value in ['y', 'yes', 'true', 't']:
+ if value in ['y', 'yes', 'true', 't', 'on']:
return True
- elif value in ['n', 'no', 'false', 'f']:
+ elif value in ['n', 'no', 'false', 'f', 'off']:
return False
msg = "Unable to interpret string value '%s' as boolean" % (value)
diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py
index 3000134..406a35c 100644
--- a/hooks/charmhelpers/core/unitdata.py
+++ b/hooks/charmhelpers/core/unitdata.py
@@ -443,7 +443,7 @@ class HookData(object):
data = hookenv.execution_environment()
self.conf = conf_delta = self.kv.delta(data['conf'], 'config')
self.rels = rels_delta = self.kv.delta(data['rels'], 'rels')
- self.kv.set('env', data['env'])
+ self.kv.set('env', dict(data['env']))
self.kv.set('unit', data['unit'])
self.kv.set('relid', data.get('relid'))
return conf_delta, rels_delta
diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py
index 792e629..0a3bb96 100644
--- a/hooks/charmhelpers/fetch/__init__.py
+++ b/hooks/charmhelpers/fetch/__init__.py
@@ -158,7 +158,7 @@ def filter_installed_packages(packages):
def apt_cache(in_memory=True):
"""Build and return an apt cache"""
- import apt_pkg
+ from apt import apt_pkg
apt_pkg.init()
if in_memory:
apt_pkg.config.set("Dir::Cache::pkgcache", "")
@@ -215,9 +215,9 @@ def apt_purge(packages, fatal=False):
_run_apt_command(cmd, fatal)
-def apt_hold(packages, fatal=False):
- """Hold one or more packages"""
- cmd = ['apt-mark', 'hold']
+def apt_mark(packages, mark, fatal=False):
+ """Flag one or more packages using apt-mark"""
+ cmd = ['apt-mark', mark]
if isinstance(packages, six.string_types):
cmd.append(packages)
else:
@@ -225,9 +225,17 @@ def apt_hold(packages, fatal=False):
log("Holding {}".format(packages))
if fatal:
- subprocess.check_call(cmd)
+ subprocess.check_call(cmd, universal_newlines=True)
else:
- subprocess.call(cmd)
+ subprocess.call(cmd, universal_newlines=True)
+
+
+def apt_hold(packages, fatal=False):
+ return apt_mark(packages, 'hold', fatal=fatal)
+
+
+def apt_unhold(packages, fatal=False):
+ return apt_mark(packages, 'unhold', fatal=fatal)
def add_source(source, key=None):
@@ -370,8 +378,9 @@ def install_remote(source, *args, **kwargs):
for handler in handlers:
try:
installed_to = handler.install(source, *args, **kwargs)
- except UnhandledSource:
- pass
+ except UnhandledSource as e:
+ log('Install source attempt unsuccessful: {}'.format(e),
+ level='WARNING')
if not installed_to:
raise UnhandledSource("No handler found for source {}".format(source))
return installed_to
diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py
index 8dfce50..efd7f9f 100644
--- a/hooks/charmhelpers/fetch/archiveurl.py
+++ b/hooks/charmhelpers/fetch/archiveurl.py
@@ -77,6 +77,8 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
def can_handle(self, source):
url_parts = self.parse_url(source)
if url_parts.scheme not in ('http', 'https', 'ftp', 'file'):
+ # XXX: Why is this returning a boolean and a string? It's
+ # doomed to fail since "bool(can_handle('foo://'))" will be True.
return "Wrong source type"
if get_archive_handler(self.base_url(source)):
return True
@@ -155,7 +157,11 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
else:
algorithms = hashlib.algorithms_available
if key in algorithms:
- check_hash(dld_file, value, key)
+ if len(value) != 1:
+ raise TypeError(
+ "Expected 1 hash value, not %d" % len(value))
+ expected = value[0]
+ check_hash(dld_file, expected, key)
if checksum:
check_hash(dld_file, checksum, hash_type)
return extract(dld_file, dest)
diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py
index 93aae87..f023b26 100644
--- a/hooks/charmhelpers/fetch/giturl.py
+++ b/hooks/charmhelpers/fetch/giturl.py
@@ -45,14 +45,16 @@ class GitUrlFetchHandler(BaseFetchHandler):
else:
return True
- def clone(self, source, dest, branch):
+ def clone(self, source, dest, branch, depth=None):
if not self.can_handle(source):
raise UnhandledSource("Cannot handle {}".format(source))
- repo = Repo.clone_from(source, dest)
- repo.git.checkout(branch)
+ if depth:
+ Repo.clone_from(source, dest, branch=branch, depth=depth)
+ else:
+ Repo.clone_from(source, dest, branch=branch)
- def install(self, source, branch="master", dest=None):
+ def install(self, source, branch="master", dest=None, depth=None):
url_parts = self.parse_url(source)
branch_name = url_parts.path.strip("/").split("/")[-1]
if dest:
@@ -63,9 +65,9 @@ class GitUrlFetchHandler(BaseFetchHandler):
if not os.path.exists(dest_dir):
mkdir(dest_dir, perms=0o755)
try:
- self.clone(source, dest_dir, branch)
+ self.clone(source, dest_dir, branch, depth)
except GitCommandError as e:
- raise UnhandledSource(e.message)
+ raise UnhandledSource(e)
except OSError as e:
raise UnhandledSource(e.strerror)
return dest_dir