Sync charm-helpers

Change-Id: I734acea202178926689a43ab413b56cb9b948b9a
This commit is contained in:
Corey Bryant 2019-01-08 15:16:18 +00:00
parent b7fde9532f
commit 3ee1cf4b6c
14 changed files with 522 additions and 150 deletions

View File

@ -23,22 +23,22 @@ import subprocess
import sys
try:
import six # flake8: noqa
import six # NOQA:F401
except ImportError:
if sys.version_info.major == 2:
subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
else:
subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
import six # flake8: noqa
import six # NOQA:F401
try:
import yaml # flake8: noqa
import yaml # NOQA:F401
except ImportError:
if sys.version_info.major == 2:
subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
else:
subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
import yaml # flake8: noqa
import yaml # NOQA:F401
# Holds a list of mapping of mangled function names that have been deprecated

View File

@ -23,8 +23,8 @@
#
import os
import subprocess
from charmhelpers.core import host
from charmhelpers.core.hookenv import (
config as config_get,
relation_get,
@ -83,14 +83,4 @@ def retrieve_ca_cert(cert_file):
def install_ca_cert(ca_cert):
if ca_cert:
cert_file = ('/usr/local/share/ca-certificates/'
'keystone_juju_ca_cert.crt')
old_cert = retrieve_ca_cert(cert_file)
if old_cert and old_cert == ca_cert:
log("CA cert is the same as installed version", level=INFO)
else:
log("Installing new CA cert", level=INFO)
with open(cert_file, 'wb') as crt:
crt.write(ca_cert)
subprocess.check_call(['update-ca-certificates', '--fresh'])
host.install_ca_cert(ca_cert, 'keystone_juju_ca_cert')

View File

@ -168,7 +168,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
'nrpe', 'openvswitch-odl', 'neutron-api-odl',
'odl-controller', 'cinder-backup', 'nexentaedge-data',
'nexentaedge-iscsi-gw', 'nexentaedge-swift-gw',
'cinder-nexentaedge', 'nexentaedge-mgmt']))
'cinder-nexentaedge', 'nexentaedge-mgmt',
'ceilometer-agent']))
if self.openstack:
for svc in services:
@ -292,7 +293,9 @@ class OpenStackAmuletDeployment(AmuletDeployment):
('artful', None): self.artful_pike,
('bionic', None): self.bionic_queens,
('bionic', 'cloud:bionic-rocky'): self.bionic_rocky,
('bionic', 'cloud:bionic-stein'): self.bionic_stein,
('cosmic', None): self.cosmic_rocky,
('disco', None): self.disco_stein,
}
return releases[(self.series, self.openstack)]

View File

@ -57,7 +57,8 @@ OPENSTACK_RELEASES_PAIRS = [
'trusty_mitaka', 'xenial_mitaka', 'xenial_newton',
'yakkety_newton', 'xenial_ocata', 'zesty_ocata',
'xenial_pike', 'artful_pike', 'xenial_queens',
'bionic_queens', 'bionic_rocky', 'cosmic_rocky']
'bionic_queens', 'bionic_rocky', 'cosmic_rocky',
'bionic_stein', 'disco_stein']
class OpenStackAmuletUtils(AmuletUtils):
@ -618,12 +619,12 @@ class OpenStackAmuletUtils(AmuletUtils):
return self.authenticate_keystone(keystone_ip, user, password,
project_name=tenant)
def authenticate_glance_admin(self, keystone):
def authenticate_glance_admin(self, keystone, force_v1_client=False):
"""Authenticates admin user with glance."""
self.log.debug('Authenticating glance admin...')
ep = keystone.service_catalog.url_for(service_type='image',
interface='adminURL')
if keystone.session:
if not force_v1_client and keystone.session:
return glance_clientv2.Client("2", session=keystone.session)
else:
return glance_client.Client(ep, token=keystone.auth_token)
@ -680,18 +681,30 @@ class OpenStackAmuletUtils(AmuletUtils):
nova.flavors.create(name, ram, vcpus, disk, flavorid,
ephemeral, swap, rxtx_factor, is_public)
def create_cirros_image(self, glance, image_name):
"""Download the latest cirros image and upload it to glance,
validate and return a resource pointer.
def glance_create_image(self, glance, image_name, image_url,
download_dir='tests',
hypervisor_type=None,
disk_format='qcow2',
architecture='x86_64',
container_format='bare'):
"""Download an image and upload it to glance, validate its status
and return an image object pointer. KVM defaults, can override for
LXD.
:param glance: pointer to authenticated glance connection
:param glance: pointer to authenticated glance api connection
:param image_name: display name for new image
:param image_url: url to retrieve
:param download_dir: directory to store downloaded image file
:param hypervisor_type: glance image hypervisor property
:param disk_format: glance image disk format
:param architecture: glance image architecture property
:param container_format: glance image container format
:returns: glance image pointer
"""
self.log.debug('Creating glance cirros image '
'({})...'.format(image_name))
self.log.debug('Creating glance image ({}) from '
'{}...'.format(image_name, image_url))
# Download cirros image
# Download image
http_proxy = os.getenv('AMULET_HTTP_PROXY')
self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
if http_proxy:
@ -700,31 +713,34 @@ class OpenStackAmuletUtils(AmuletUtils):
else:
opener = urllib.FancyURLopener()
f = opener.open('http://download.cirros-cloud.net/version/released')
version = f.read().strip()
cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
local_path = os.path.join('tests', cirros_img)
if not os.path.exists(local_path):
cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
version, cirros_img)
opener.retrieve(cirros_url, local_path)
f.close()
abs_file_name = os.path.join(download_dir, image_name)
if not os.path.exists(abs_file_name):
opener.retrieve(image_url, abs_file_name)
# Create glance image
glance_properties = {
'architecture': architecture,
}
if hypervisor_type:
glance_properties['hypervisor_type'] = hypervisor_type
# Create glance image
if float(glance.version) < 2.0:
with open(local_path) as fimage:
image = glance.images.create(name=image_name, is_public=True,
disk_format='qcow2',
container_format='bare',
data=fimage)
with open(abs_file_name) as f:
image = glance.images.create(
name=image_name,
is_public=True,
disk_format=disk_format,
container_format=container_format,
properties=glance_properties,
data=f)
else:
image = glance.images.create(
name=image_name,
disk_format="qcow2",
visibility="public",
container_format="bare")
glance.images.upload(image.id, open(local_path, 'rb'))
disk_format=disk_format,
container_format=container_format)
glance.images.upload(image.id, open(abs_file_name, 'rb'))
glance.images.update(image.id, **glance_properties)
# Wait for image to reach active status
img_id = image.id
@ -753,15 +769,54 @@ class OpenStackAmuletUtils(AmuletUtils):
val_img_stat, val_img_cfmt, val_img_dfmt))
if val_img_name == image_name and val_img_stat == 'active' \
and val_img_pub is True and val_img_cfmt == 'bare' \
and val_img_dfmt == 'qcow2':
and val_img_pub is True and val_img_cfmt == container_format \
and val_img_dfmt == disk_format:
self.log.debug(msg_attr)
else:
msg = ('Volume validation failed, {}'.format(msg_attr))
msg = ('Image validation failed, {}'.format(msg_attr))
amulet.raise_status(amulet.FAIL, msg=msg)
return image
def create_cirros_image(self, glance, image_name, hypervisor_type=None):
"""Download the latest cirros image and upload it to glance,
validate and return a resource pointer.
:param glance: pointer to authenticated glance connection
:param image_name: display name for new image
:param hypervisor_type: glance image hypervisor property
:returns: glance image pointer
"""
# /!\ DEPRECATION WARNING
self.log.warn('/!\\ DEPRECATION WARNING: use '
'glance_create_image instead of '
'create_cirros_image.')
self.log.debug('Creating glance cirros image '
'({})...'.format(image_name))
# Get cirros image URL
http_proxy = os.getenv('AMULET_HTTP_PROXY')
self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
if http_proxy:
proxies = {'http': http_proxy}
opener = urllib.FancyURLopener(proxies)
else:
opener = urllib.FancyURLopener()
f = opener.open('http://download.cirros-cloud.net/version/released')
version = f.read().strip()
cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
version, cirros_img)
f.close()
return self.glance_create_image(
glance,
image_name,
cirros_url,
hypervisor_type=hypervisor_type)
def delete_image(self, glance, image):
"""Delete the specified image."""

View File

@ -25,7 +25,9 @@ from charmhelpers.core.hookenv import (
local_unit,
network_get_primary_address,
config,
related_units,
relation_get,
relation_ids,
unit_get,
NoNetworkBinding,
log,
@ -193,7 +195,7 @@ def install_certs(ssl_dir, certs, chain=None):
if chain:
# Append chain file so that clients that trust the root CA will
# trust certs signed by an intermediate in the chain
cert_data = cert_data + chain
cert_data = cert_data + os.linesep + chain
write_file(
path=os.path.join(ssl_dir, cert_filename),
content=cert_data, perms=0o640)
@ -225,3 +227,49 @@ def process_certificates(service_name, relation_id, unit,
create_ip_cert_links(
ssl_dir,
custom_hostname_link=custom_hostname_link)
def get_requests_for_local_unit(relation_name=None):
"""Extract any certificates data targeted at this unit down relation_name.
:param relation_name: str Name of relation to check for data.
:returns: List of bundles of certificates.
:rtype: List of dicts
"""
local_name = local_unit().replace('/', '_')
raw_certs_key = '{}.processed_requests'.format(local_name)
relation_name = relation_name or 'certificates'
bundles = []
for rid in relation_ids(relation_name):
for unit in related_units(rid):
data = relation_get(rid=rid, unit=unit)
if data.get(raw_certs_key):
bundles.append({
'ca': data['ca'],
'chain': data.get('chain'),
'certs': json.loads(data[raw_certs_key])})
return bundles
def get_bundle_for_cn(cn, relation_name=None):
"""Extract certificates for the given cn.
:param cn: str Canonical Name on certificate.
:param relation_name: str Relation to check for certificates down.
:returns: Dictionary of certificate data,
:rtype: dict.
"""
entries = get_requests_for_local_unit(relation_name)
cert_bundle = {}
for entry in entries:
for _cn, bundle in entry['certs'].items():
if _cn == cn:
cert_bundle = {
'cert': bundle['cert'],
'key': bundle['key'],
'chain': entry['chain'],
'ca': entry['ca']}
break
if cert_bundle:
break
return cert_bundle

View File

@ -98,7 +98,6 @@ from charmhelpers.contrib.network.ip import (
from charmhelpers.contrib.openstack.utils import (
config_flags_parser,
enable_memcache,
snap_install_requested,
CompareOpenStackReleases,
os_release,
)
@ -252,13 +251,8 @@ class SharedDBContext(OSContextGenerator):
'database': self.database,
'database_user': self.user,
'database_password': rdata.get(password_setting),
'database_type': 'mysql'
'database_type': 'mysql+pymysql'
}
# Note(coreycb): We can drop mysql+pymysql if we want when the
# following review lands, though it seems mysql+pymysql would
# be preferred. https://review.openstack.org/#/c/462190/
if snap_install_requested():
ctxt['database_type'] = 'mysql+pymysql'
if self.context_complete(ctxt):
db_ssl(rdata, ctxt, self.ssl_dir)
return ctxt
@ -642,7 +636,7 @@ class HAProxyContext(OSContextGenerator):
return {}
l_unit = local_unit().replace('/', '-')
cluster_hosts = {}
cluster_hosts = collections.OrderedDict()
# NOTE(jamespage): build out map of configured network endpoints
# and associated backends
@ -1523,10 +1517,6 @@ class NeutronAPIContext(OSContextGenerator):
'rel_key': 'enable-nsg-logging',
'default': False,
},
'nsg_log_output_base': {
'rel_key': 'nsg-log-output-base',
'default': None,
},
}
ctxt = self.get_neutron_options({})
for rid in relation_ids('neutron-plugin-api'):
@ -1538,10 +1528,15 @@ class NeutronAPIContext(OSContextGenerator):
if 'l2-population' in rdata:
ctxt.update(self.get_neutron_options(rdata))
extension_drivers = []
if ctxt['enable_qos']:
ctxt['extension_drivers'] = 'qos'
else:
ctxt['extension_drivers'] = ''
extension_drivers.append('qos')
if ctxt['enable_nsg_logging']:
extension_drivers.append('log')
ctxt['extension_drivers'] = ','.join(extension_drivers)
return ctxt
@ -1901,7 +1896,7 @@ class EnsureDirContext(OSContextGenerator):
Some software requires a user to create a target directory to be
scanned for drop-in files with a specific format. This is why this
context is needed to do that before rendering a template.
'''
'''
def __init__(self, dirname, **kwargs):
'''Used merely to ensure that a given directory exists.'''
@ -1911,3 +1906,23 @@ class EnsureDirContext(OSContextGenerator):
def __call__(self):
mkdir(self.dirname, **self.kwargs)
return {}
class VersionsContext(OSContextGenerator):
"""Context to return the openstack and operating system versions.
"""
def __init__(self, pkg='python-keystone'):
"""Initialise context.
:param pkg: Package to extrapolate openstack version from.
:type pkg: str
"""
self.pkg = pkg
def __call__(self):
ostack = os_release(self.pkg, base='icehouse')
osystem = lsb_release()['DISTRIB_CODENAME'].lower()
return {
'openstack_release': ostack,
'operating_system_release': osystem}

View File

@ -23,18 +23,19 @@
Helpers for high availability.
"""
import hashlib
import json
import re
from charmhelpers.core.hookenv import (
expected_related_units,
log,
relation_set,
charm_name,
config,
status_set,
DEBUG,
WARNING,
)
from charmhelpers.core.host import (
@ -62,6 +63,9 @@ JSON_ENCODE_OPTIONS = dict(
separators=(',', ':'),
)
VIP_GROUP_NAME = 'grp_{service}_vips'
DNSHA_GROUP_NAME = 'grp_{service}_hostnames'
class DNSHAException(Exception):
"""Raised when an error occurs setting up DNS HA
@ -110,21 +114,42 @@ def assert_charm_supports_dns_ha():
def expect_ha():
""" Determine if the unit expects to be in HA
Check for VIP or dns-ha settings which indicate the unit should expect to
be related to hacluster.
Check juju goal-state if ha relation is expected, check for VIP or dns-ha
settings which indicate the unit should expect to be related to hacluster.
@returns boolean
"""
return config('vip') or config('dns-ha')
ha_related_units = []
try:
ha_related_units = list(expected_related_units(reltype='ha'))
except (NotImplementedError, KeyError):
pass
return len(ha_related_units) > 0 or config('vip') or config('dns-ha')
def generate_ha_relation_data(service):
def generate_ha_relation_data(service, extra_settings=None):
""" Generate relation data for ha relation
Based on configuration options and unit interfaces, generate a json
encoded dict of relation data items for the hacluster relation,
providing configuration for DNS HA or VIP's + haproxy clone sets.
Example of supplying additional settings::
COLO_CONSOLEAUTH = 'inf: res_nova_consoleauth grp_nova_vips'
AGENT_CONSOLEAUTH = 'ocf:openstack:nova-consoleauth'
AGENT_CA_PARAMS = 'op monitor interval="5s"'
ha_console_settings = {
'colocations': {'vip_consoleauth': COLO_CONSOLEAUTH},
'init_services': {'res_nova_consoleauth': 'nova-consoleauth'},
'resources': {'res_nova_consoleauth': AGENT_CONSOLEAUTH},
'resource_params': {'res_nova_consoleauth': AGENT_CA_PARAMS})
generate_ha_relation_data('nova', extra_settings=ha_console_settings)
@param service: Name of the service being configured
@param extra_settings: Dict of additional resource data
@returns dict: json encoded data for use with relation_set
"""
_haproxy_res = 'res_{}_haproxy'.format(service)
@ -143,6 +168,13 @@ def generate_ha_relation_data(service):
},
}
if extra_settings:
for k, v in extra_settings.items():
if _relation_data.get(k):
_relation_data[k].update(v)
else:
_relation_data[k] = v
if config('dns-ha'):
update_hacluster_dns_ha(service, _relation_data)
else:
@ -210,7 +242,7 @@ def update_hacluster_dns_ha(service, relation_data,
'Informing the ha relation'.format(' '.join(hostname_group)),
DEBUG)
relation_data['groups'] = {
'grp_{}_hostnames'.format(service): ' '.join(hostname_group)
DNSHA_GROUP_NAME.format(service=service): ' '.join(hostname_group)
}
else:
msg = 'DNS HA: Hostname group has no members.'
@ -218,6 +250,27 @@ def update_hacluster_dns_ha(service, relation_data,
raise DNSHAException(msg)
def get_vip_settings(vip):
"""Calculate which nic is on the correct network for the given vip.
If nic or netmask discovery fail then fallback to using charm supplied
config. If fallback is used this is indicated via the fallback variable.
@param vip: VIP to lookup nic and cidr for.
@returns (str, str, bool): eg (iface, netmask, fallback)
"""
iface = get_iface_for_address(vip)
netmask = get_netmask_for_address(vip)
fallback = False
if iface is None:
iface = config('vip_iface')
fallback = True
if netmask is None:
netmask = config('vip_cidr')
fallback = True
return iface, netmask, fallback
def update_hacluster_vip(service, relation_data):
""" Configure VIP resources based on provided configuration
@ -226,40 +279,70 @@ def update_hacluster_vip(service, relation_data):
"""
cluster_config = get_hacluster_config()
vip_group = []
vips_to_delete = []
for vip in cluster_config['vip'].split():
if is_ipv6(vip):
res_neutron_vip = 'ocf:heartbeat:IPv6addr'
res_vip = 'ocf:heartbeat:IPv6addr'
vip_params = 'ipv6addr'
else:
res_neutron_vip = 'ocf:heartbeat:IPaddr2'
res_vip = 'ocf:heartbeat:IPaddr2'
vip_params = 'ip'
iface = (get_iface_for_address(vip) or
config('vip_iface'))
netmask = (get_netmask_for_address(vip) or
config('vip_cidr'))
iface, netmask, fallback = get_vip_settings(vip)
vip_monitoring = 'op monitor depth="0" timeout="20s" interval="10s"'
if iface is not None:
# NOTE(jamespage): Delete old VIP resources
# Old style naming encoding iface in name
# does not work well in environments where
# interface/subnet wiring is not consistent
vip_key = 'res_{}_{}_vip'.format(service, iface)
if vip_key in vip_group:
if vip not in relation_data['resource_params'][vip_key]:
vip_key = '{}_{}'.format(vip_key, vip_params)
else:
log("Resource '%s' (vip='%s') already exists in "
"vip group - skipping" % (vip_key, vip), WARNING)
continue
if vip_key in vips_to_delete:
vip_key = '{}_{}'.format(vip_key, vip_params)
vips_to_delete.append(vip_key)
vip_key = 'res_{}_{}_vip'.format(
service,
hashlib.sha1(vip.encode('UTF-8')).hexdigest()[:7])
relation_data['resources'][vip_key] = res_vip
# NOTE(jamespage):
# Use option provided vip params if these where used
# instead of auto-detected values
if fallback:
relation_data['resource_params'][vip_key] = (
'params {ip}="{vip}" cidr_netmask="{netmask}" '
'nic="{iface}" {vip_monitoring}'.format(
ip=vip_params,
vip=vip,
iface=iface,
netmask=netmask,
vip_monitoring=vip_monitoring))
else:
# NOTE(jamespage):
# let heartbeat figure out which interface and
# netmask to configure, which works nicely
# when network interface naming is not
# consistent across units.
relation_data['resource_params'][vip_key] = (
'params {ip}="{vip}" {vip_monitoring}'.format(
ip=vip_params,
vip=vip,
vip_monitoring=vip_monitoring))
relation_data['resources'][vip_key] = res_neutron_vip
relation_data['resource_params'][vip_key] = (
'params {ip}="{vip}" cidr_netmask="{netmask}" '
'nic="{iface}"'.format(ip=vip_params,
vip=vip,
iface=iface,
netmask=netmask)
)
vip_group.append(vip_key)
if vips_to_delete:
try:
relation_data['delete_resources'].extend(vips_to_delete)
except KeyError:
relation_data['delete_resources'] = vips_to_delete
if len(vip_group) >= 1:
relation_data['groups'] = {
'grp_{}_vips'.format(service): ' '.join(vip_group)
}
key = VIP_GROUP_NAME.format(service=service)
try:
relation_data['groups'][key] = ' '.join(vip_group)
except KeyError:
relation_data['groups'] = {
key: ' '.join(vip_group)
}

View File

@ -1,12 +1,14 @@
{% if auth_host -%}
[keystone_authtoken]
auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
auth_type = password
{% if api_version == "3" -%}
auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/v3
auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v3
project_domain_name = {{ admin_domain_name }}
user_domain_name = {{ admin_domain_name }}
{% else -%}
auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
project_domain_name = default
user_domain_name = default
{% endif -%}

View File

@ -73,6 +73,8 @@ from charmhelpers.core.host import (
service_running,
service_pause,
service_resume,
service_stop,
service_start,
restart_on_change_helper,
)
from charmhelpers.fetch import (
@ -116,6 +118,7 @@ OPENSTACK_RELEASES = (
'pike',
'queens',
'rocky',
'stein',
)
UBUNTU_OPENSTACK_RELEASE = OrderedDict([
@ -134,6 +137,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
('artful', 'pike'),
('bionic', 'queens'),
('cosmic', 'rocky'),
('disco', 'stein'),
])
@ -153,6 +157,7 @@ OPENSTACK_CODENAMES = OrderedDict([
('2017.2', 'pike'),
('2018.1', 'queens'),
('2018.2', 'rocky'),
('2019.1', 'stein'),
])
# The ugly duckling - must list releases oldest to newest
@ -187,6 +192,8 @@ SWIFT_CODENAMES = OrderedDict([
['2.16.0', '2.17.0']),
('rocky',
['2.18.0', '2.19.0']),
('stein',
['2.19.0']),
])
# >= Liberty version->codename mapping
@ -199,6 +206,7 @@ PACKAGE_CODENAMES = {
('16', 'pike'),
('17', 'queens'),
('18', 'rocky'),
('19', 'stein'),
]),
'neutron-common': OrderedDict([
('7', 'liberty'),
@ -208,6 +216,7 @@ PACKAGE_CODENAMES = {
('11', 'pike'),
('12', 'queens'),
('13', 'rocky'),
('14', 'stein'),
]),
'cinder-common': OrderedDict([
('7', 'liberty'),
@ -217,6 +226,7 @@ PACKAGE_CODENAMES = {
('11', 'pike'),
('12', 'queens'),
('13', 'rocky'),
('14', 'stein'),
]),
'keystone': OrderedDict([
('8', 'liberty'),
@ -226,6 +236,7 @@ PACKAGE_CODENAMES = {
('12', 'pike'),
('13', 'queens'),
('14', 'rocky'),
('15', 'stein'),
]),
'horizon-common': OrderedDict([
('8', 'liberty'),
@ -235,6 +246,7 @@ PACKAGE_CODENAMES = {
('12', 'pike'),
('13', 'queens'),
('14', 'rocky'),
('15', 'stein'),
]),
'ceilometer-common': OrderedDict([
('5', 'liberty'),
@ -244,6 +256,7 @@ PACKAGE_CODENAMES = {
('9', 'pike'),
('10', 'queens'),
('11', 'rocky'),
('12', 'stein'),
]),
'heat-common': OrderedDict([
('5', 'liberty'),
@ -253,6 +266,7 @@ PACKAGE_CODENAMES = {
('9', 'pike'),
('10', 'queens'),
('11', 'rocky'),
('12', 'stein'),
]),
'glance-common': OrderedDict([
('11', 'liberty'),
@ -262,6 +276,7 @@ PACKAGE_CODENAMES = {
('15', 'pike'),
('16', 'queens'),
('17', 'rocky'),
('18', 'stein'),
]),
'openstack-dashboard': OrderedDict([
('8', 'liberty'),
@ -271,6 +286,7 @@ PACKAGE_CODENAMES = {
('12', 'pike'),
('13', 'queens'),
('14', 'rocky'),
('15', 'stein'),
]),
}
@ -299,7 +315,7 @@ def get_os_codename_install_source(src):
rel = ''
if src is None:
return rel
if src in ['distro', 'distro-proposed']:
if src in ['distro', 'distro-proposed', 'proposed']:
try:
rel = UBUNTU_OPENSTACK_RELEASE[ubuntu_rel]
except KeyError:
@ -375,7 +391,7 @@ def get_swift_codename(version):
return codenames[0]
# NOTE: fallback - attempt to match with just major.minor version
match = re.match('^(\d+)\.(\d+)', version)
match = re.match(r'^(\d+)\.(\d+)', version)
if match:
major_minor_version = match.group(0)
for codename, versions in six.iteritems(SWIFT_CODENAMES):
@ -395,7 +411,7 @@ def get_os_codename_package(package, fatal=True):
out = subprocess.check_output(cmd)
if six.PY3:
out = out.decode('UTF-8')
except subprocess.CalledProcessError as e:
except subprocess.CalledProcessError:
return None
lines = out.split('\n')
for line in lines:
@ -427,11 +443,11 @@ def get_os_codename_package(package, fatal=True):
vers = apt.upstream_version(pkg.current_ver.ver_str)
if 'swift' in pkg.name:
# Fully x.y.z match for swift versions
match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
match = re.match(r'^(\d+)\.(\d+)\.(\d+)', vers)
else:
# x.y match only for 20XX.X
# and ignore patch level for other packages
match = re.match('^(\d+)\.(\d+)', vers)
match = re.match(r'^(\d+)\.(\d+)', vers)
if match:
vers = match.group(0)
@ -1303,6 +1319,65 @@ def is_unit_paused_set():
return False
def manage_payload_services(action, services=None, charm_func=None):
"""Run an action against all services.
An optional charm_func() can be called. It should raise an Exception to
indicate that the function failed. If it was succesfull it should return
None or an optional message.
The signature for charm_func is:
charm_func() -> message: str
charm_func() is executed after any services are stopped, if supplied.
The services object can either be:
- None : no services were passed (an empty dict is returned)
- a list of strings
- A dictionary (optionally OrderedDict) {service_name: {'service': ..}}
- An array of [{'service': service_name, ...}, ...]
:param action: Action to run: pause, resume, start or stop.
:type action: str
:param services: See above
:type services: See above
:param charm_func: function to run for custom charm pausing.
:type charm_func: f()
:returns: Status boolean and list of messages
:rtype: (bool, [])
:raises: RuntimeError
"""
actions = {
'pause': service_pause,
'resume': service_resume,
'start': service_start,
'stop': service_stop}
action = action.lower()
if action not in actions.keys():
raise RuntimeError(
"action: {} must be one of: {}".format(action,
', '.join(actions.keys())))
services = _extract_services_list_helper(services)
messages = []
success = True
if services:
for service in services.keys():
rc = actions[action](service)
if not rc:
success = False
messages.append("{} didn't {} cleanly.".format(service,
action))
if charm_func:
try:
message = charm_func()
if message:
messages.append(message)
except Exception as e:
success = False
messages.append(str(e))
return success, messages
def pause_unit(assess_status_func, services=None, ports=None,
charm_func=None):
"""Pause a unit by stopping the services and setting 'unit-paused'
@ -1333,20 +1408,10 @@ def pause_unit(assess_status_func, services=None, ports=None,
@returns None
@raises Exception(message) on an error for action_fail().
"""
services = _extract_services_list_helper(services)
messages = []
if services:
for service in services.keys():
stopped = service_pause(service)
if not stopped:
messages.append("{} didn't stop cleanly.".format(service))
if charm_func:
try:
message = charm_func()
if message:
messages.append(message)
except Exception as e:
message.append(str(e))
_, messages = manage_payload_services(
'pause',
services=services,
charm_func=charm_func)
set_unit_paused()
if assess_status_func:
message = assess_status_func()
@ -1385,20 +1450,10 @@ def resume_unit(assess_status_func, services=None, ports=None,
@returns None
@raises Exception(message) on an error for action_fail().
"""
services = _extract_services_list_helper(services)
messages = []
if services:
for service in services.keys():
started = service_resume(service)
if not started:
messages.append("{} didn't start cleanly.".format(service))
if charm_func:
try:
message = charm_func()
if message:
messages.append(message)
except Exception as e:
message.append(str(e))
_, messages = manage_payload_services(
'resume',
services=services,
charm_func=charm_func)
clear_unit_paused()
if assess_status_func:
message = assess_status_func()
@ -1450,20 +1505,33 @@ def pausable_restart_on_change(restart_map, stopstart=False,
see core.utils.restart_on_change() for more details.
Note restart_map can be a callable, in which case, restart_map is only
evaluated at runtime. This means that it is lazy and the underlying
function won't be called if the decorated function is never called. Note,
retains backwards compatibility for passing a non-callable dictionary.
@param f: the function to decorate
@param restart_map: the restart map {conf_file: [services]}
@param restart_map: (optionally callable, which then returns the
restart_map) the restart map {conf_file: [services]}
@param stopstart: DEFAULT false; whether to stop, start or just restart
@returns decorator to use a restart_on_change with pausability
"""
def wrap(f):
# py27 compatible nonlocal variable. When py3 only, replace with
# nonlocal keyword
__restart_map_cache = {'cache': None}
@functools.wraps(f)
def wrapped_f(*args, **kwargs):
if is_unit_paused_set():
return f(*args, **kwargs)
if __restart_map_cache['cache'] is None:
__restart_map_cache['cache'] = restart_map() \
if callable(restart_map) else restart_map
# otherwise, normal restart_on_change functionality
return restart_on_change_helper(
(lambda: f(*args, **kwargs)), restart_map, stopstart,
restart_functions)
(lambda: f(*args, **kwargs)), __restart_map_cache['cache'],
stopstart, restart_functions)
return wrapped_f
return wrap

View File

@ -36,10 +36,12 @@ def loopback_devices():
'''
loopbacks = {}
cmd = ['losetup', '-a']
devs = [d.strip().split(' ') for d in
check_output(cmd).splitlines() if d != '']
output = check_output(cmd)
if six.PY3:
output = output.decode('utf-8')
devs = [d.strip().split(' ') for d in output.splitlines() if d != '']
for dev, _, f in devs:
loopbacks[dev.replace(':', '')] = re.search('\((\S+)\)', f).groups()[0]
loopbacks[dev.replace(':', '')] = re.search(r'\((\S+)\)', f).groups()[0]
return loopbacks

View File

@ -510,6 +510,67 @@ def related_units(relid=None):
subprocess.check_output(units_cmd_line).decode('UTF-8')) or []
def expected_peer_units():
"""Get a generator for units we expect to join peer relation based on
goal-state.
The local unit is excluded from the result to make it easy to gauge
completion of all peers joining the relation with existing hook tools.
Example usage:
log('peer {} of {} joined peer relation'
.format(len(related_units()),
len(list(expected_peer_units()))))
This function will raise NotImplementedError if used with juju versions
without goal-state support.
:returns: iterator
:rtype: types.GeneratorType
:raises: NotImplementedError
"""
if not has_juju_version("2.4.0"):
# goal-state first appeared in 2.4.0.
raise NotImplementedError("goal-state")
_goal_state = goal_state()
return (key for key in _goal_state['units']
if '/' in key and key != local_unit())
def expected_related_units(reltype=None):
"""Get a generator for units we expect to join relation based on
goal-state.
Note that you can not use this function for the peer relation, take a look
at expected_peer_units() for that.
This function will raise KeyError if you request information for a
relation type for which juju goal-state does not have information. It will
raise NotImplementedError if used with juju versions without goal-state
support.
Example usage:
log('participant {} of {} joined relation {}'
.format(len(related_units()),
len(list(expected_related_units())),
relation_type()))
:param reltype: Relation type to list data for, default is to list data for
the realtion type we are currently executing a hook for.
:type reltype: str
:returns: iterator
:rtype: types.GeneratorType
:raises: KeyError, NotImplementedError
"""
if not has_juju_version("2.4.4"):
# goal-state existed in 2.4.0, but did not list individual units to
# join a relation in 2.4.1 through 2.4.3. (LP: #1794739)
raise NotImplementedError("goal-state relation unit count")
reltype = reltype or relation_type()
_goal_state = goal_state()
return (key for key in _goal_state['relations'][reltype] if '/' in key)
@cached
def relation_for_unit(unit=None, rid=None):
"""Get the json represenation of a unit's relation"""
@ -998,6 +1059,7 @@ def application_version_set(version):
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
@cached
def goal_state():
"""Juju goal state values"""
cmd = ['goal-state', '--format=json']

View File

@ -34,13 +34,13 @@ import six
from contextlib import contextmanager
from collections import OrderedDict
from .hookenv import log, DEBUG, local_unit
from .hookenv import log, INFO, DEBUG, local_unit, charm_name
from .fstab import Fstab
from charmhelpers.osplatform import get_platform
__platform__ = get_platform()
if __platform__ == "ubuntu":
from charmhelpers.core.host_factory.ubuntu import (
from charmhelpers.core.host_factory.ubuntu import ( # NOQA:F401
service_available,
add_new_group,
lsb_release,
@ -48,7 +48,7 @@ if __platform__ == "ubuntu":
CompareHostReleases,
) # flake8: noqa -- ignore F401 for this import
elif __platform__ == "centos":
from charmhelpers.core.host_factory.centos import (
from charmhelpers.core.host_factory.centos import ( # NOQA:F401
service_available,
add_new_group,
lsb_release,
@ -58,6 +58,7 @@ elif __platform__ == "centos":
UPDATEDB_PATH = '/etc/updatedb.conf'
def service_start(service_name, **kwargs):
"""Start a system service.
@ -287,8 +288,8 @@ def service_running(service_name, **kwargs):
for key, value in six.iteritems(kwargs):
parameter = '%s=%s' % (key, value)
cmd.append(parameter)
output = subprocess.check_output(cmd,
stderr=subprocess.STDOUT).decode('UTF-8')
output = subprocess.check_output(
cmd, stderr=subprocess.STDOUT).decode('UTF-8')
except subprocess.CalledProcessError:
return False
else:
@ -442,7 +443,7 @@ def add_user_to_group(username, group):
def chage(username, lastday=None, expiredate=None, inactive=None,
mindays=None, maxdays=None, root=None, warndays=None):
mindays=None, maxdays=None, root=None, warndays=None):
"""Change user password expiry information
:param str username: User to update
@ -482,8 +483,10 @@ def chage(username, lastday=None, expiredate=None, inactive=None,
cmd.append(username)
subprocess.check_call(cmd)
remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1')
def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
"""Replicate the contents of a path"""
options = options or ['--delete', '--executability']
@ -535,13 +538,15 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
# lets see if we can grab the file and compare the context, to avoid doing
# a write.
existing_content = None
existing_uid, existing_gid = None, None
existing_uid, existing_gid, existing_perms = None, None, None
try:
with open(path, 'rb') as target:
existing_content = target.read()
stat = os.stat(path)
existing_uid, existing_gid = stat.st_uid, stat.st_gid
except:
existing_uid, existing_gid, existing_perms = (
stat.st_uid, stat.st_gid, stat.st_mode
)
except Exception:
pass
if content != existing_content:
log("Writing file {} {}:{} {:o}".format(path, owner, group, perms),
@ -554,7 +559,7 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
target.write(content)
return
# the contents were the same, but we might still need to change the
# ownership.
# ownership or permissions.
if existing_uid != uid:
log("Changing uid on already existing content: {} -> {}"
.format(existing_uid, uid), level=DEBUG)
@ -563,6 +568,10 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
log("Changing gid on already existing content: {} -> {}"
.format(existing_gid, gid), level=DEBUG)
os.chown(path, -1, gid)
if existing_perms != perms:
log("Changing permissions on existing content: {} -> {}"
.format(existing_perms, perms), level=DEBUG)
os.chmod(path, perms)
def fstab_remove(mp):
@ -827,7 +836,7 @@ def list_nics(nic_type=None):
ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
ip_output = (line.strip() for line in ip_output if line)
key = re.compile('^[0-9]+:\s+(.+):')
key = re.compile(r'^[0-9]+:\s+(.+):')
for line in ip_output:
matched = re.search(key, line)
if matched:
@ -1040,3 +1049,27 @@ def modulo_distribution(modulo=3, wait=30, non_zero_wait=False):
return modulo * wait
else:
return calculated_wait_time
def install_ca_cert(ca_cert, name=None):
"""
Install the given cert as a trusted CA.
The ``name`` is the stem of the filename where the cert is written, and if
not provided, it will default to ``juju-{charm_name}``.
If the cert is empty or None, or is unchanged, nothing is done.
"""
if not ca_cert:
return
if not isinstance(ca_cert, bytes):
ca_cert = ca_cert.encode('utf8')
if not name:
name = 'juju-{}'.format(charm_name())
cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name)
new_hash = hashlib.md5(ca_cert).hexdigest()
if file_hash(cert_file) == new_hash:
return
log("Installing new CA cert at: {}".format(cert_file), level=INFO)
write_file(cert_file, ca_cert)
subprocess.check_call(['update-ca-certificates', '--fresh'])

View File

@ -26,12 +26,12 @@ from charmhelpers.core.hookenv import (
__platform__ = get_platform()
if __platform__ == "ubuntu":
from charmhelpers.core.kernel_factory.ubuntu import (
from charmhelpers.core.kernel_factory.ubuntu import ( # NOQA:F401
persistent_modprobe,
update_initramfs,
) # flake8: noqa -- ignore F401 for this import
elif __platform__ == "centos":
from charmhelpers.core.kernel_factory.centos import (
from charmhelpers.core.kernel_factory.centos import ( # NOQA:F401
persistent_modprobe,
update_initramfs,
) # flake8: noqa -- ignore F401 for this import

View File

@ -166,6 +166,14 @@ CLOUD_ARCHIVE_POCKETS = {
'rocky/proposed': 'bionic-proposed/rocky',
'bionic-rocky/proposed': 'bionic-proposed/rocky',
'bionic-proposed/rocky': 'bionic-proposed/rocky',
# Stein
'stein': 'bionic-updates/stein',
'bionic-stein': 'bionic-updates/stein',
'bionic-stein/updates': 'bionic-updates/stein',
'bionic-updates/stein': 'bionic-updates/stein',
'stein/proposed': 'bionic-proposed/stein',
'bionic-stein/proposed': 'bionic-proposed/stein',
'bionic-proposed/stein': 'bionic-proposed/stein',
}
@ -294,7 +302,7 @@ def apt_unhold(packages, fatal=False):
def import_key(key):
"""Import an ASCII Armor key.
/!\ A Radix64 format keyid is also supported for backwards
A Radix64 format keyid is also supported for backwards
compatibility, but should never be used; the key retrieval
mechanism is insecure and subject to man-in-the-middle attacks
voiding all signature checks using that key.
@ -454,6 +462,9 @@ def _add_apt_repository(spec):
:param spec: the parameter to pass to add_apt_repository
"""
if '{series}' in spec:
series = lsb_release()['DISTRIB_CODENAME']
spec = spec.replace('{series}', series)
_run_with_retries(['add-apt-repository', '--yes', spec])