Sync charm-helpers to ensure Rocky support

Change-Id: I2f0303371e73e48c58de71cb901192460209ccce
This commit is contained in:
Chris MacNaughton 2018-07-13 15:58:15 +02:00
parent 866a352bb6
commit d64d4ae496
26 changed files with 1288 additions and 199 deletions

View File

@ -21,6 +21,9 @@ from collections import OrderedDict
from charmhelpers.contrib.amulet.deployment import (
AmuletDeployment
)
from charmhelpers.contrib.openstack.amulet.utils import (
OPENSTACK_RELEASES_PAIRS
)
DEBUG = logging.DEBUG
ERROR = logging.ERROR
@ -271,11 +274,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
release.
"""
# Must be ordered by OpenStack release (not by Ubuntu release):
(self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty,
self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton,
self.yakkety_newton, self.xenial_ocata, self.zesty_ocata,
self.xenial_pike, self.artful_pike, self.xenial_queens,
self.bionic_queens,) = range(13)
for i, os_pair in enumerate(OPENSTACK_RELEASES_PAIRS):
setattr(self, os_pair, i)
releases = {
('trusty', None): self.trusty_icehouse,
@ -291,6 +291,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
('zesty', None): self.zesty_ocata,
('artful', None): self.artful_pike,
('bionic', None): self.bionic_queens,
('bionic', 'cloud:bionic-rocky'): self.bionic_rocky,
('cosmic', None): self.cosmic_rocky,
}
return releases[(self.series, self.openstack)]
@ -306,6 +308,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
('zesty', 'ocata'),
('artful', 'pike'),
('bionic', 'queens'),
('cosmic', 'rocky'),
])
if self.openstack:
os_origin = self.openstack.split(':')[1]

View File

@ -40,6 +40,7 @@ import novaclient
import pika
import swiftclient
from charmhelpers.core.decorators import retry_on_exception
from charmhelpers.contrib.amulet.utils import (
AmuletUtils
)
@ -50,6 +51,13 @@ ERROR = logging.ERROR
NOVA_CLIENT_VERSION = "2"
OPENSTACK_RELEASES_PAIRS = [
'trusty_icehouse', 'trusty_kilo', 'trusty_liberty',
'trusty_mitaka', 'xenial_mitaka', 'xenial_newton',
'yakkety_newton', 'xenial_ocata', 'zesty_ocata',
'xenial_pike', 'artful_pike', 'xenial_queens',
'bionic_queens', 'bionic_rocky', 'cosmic_rocky']
class OpenStackAmuletUtils(AmuletUtils):
"""OpenStack amulet utilities.
@ -63,7 +71,34 @@ class OpenStackAmuletUtils(AmuletUtils):
super(OpenStackAmuletUtils, self).__init__(log_level)
def validate_endpoint_data(self, endpoints, admin_port, internal_port,
public_port, expected):
public_port, expected, openstack_release=None):
"""Validate endpoint data. Pick the correct validator based on
OpenStack release. Expected data should be in the v2 format:
{
'id': id,
'region': region,
'adminurl': adminurl,
'internalurl': internalurl,
'publicurl': publicurl,
'service_id': service_id}
"""
validation_function = self.validate_v2_endpoint_data
xenial_queens = OPENSTACK_RELEASES_PAIRS.index('xenial_queens')
if openstack_release and openstack_release >= xenial_queens:
validation_function = self.validate_v3_endpoint_data
expected = {
'id': expected['id'],
'region': expected['region'],
'region_id': 'RegionOne',
'url': self.valid_url,
'interface': self.not_null,
'service_id': expected['service_id']}
return validation_function(endpoints, admin_port, internal_port,
public_port, expected)
def validate_v2_endpoint_data(self, endpoints, admin_port, internal_port,
public_port, expected):
"""Validate endpoint data.
Validate actual endpoint data vs expected endpoint data. The ports
@ -141,7 +176,86 @@ class OpenStackAmuletUtils(AmuletUtils):
if len(found) != expected_num_eps:
return 'Unexpected number of endpoints found'
def validate_svc_catalog_endpoint_data(self, expected, actual):
def convert_svc_catalog_endpoint_data_to_v3(self, ep_data):
"""Convert v2 endpoint data into v3.
{
'service_name1': [
{
'adminURL': adminURL,
'id': id,
'region': region.
'publicURL': publicURL,
'internalURL': internalURL
}],
'service_name2': [
{
'adminURL': adminURL,
'id': id,
'region': region.
'publicURL': publicURL,
'internalURL': internalURL
}],
}
"""
self.log.warn("Endpoint ID and Region ID validation is limited to not "
"null checks after v2 to v3 conversion")
for svc in ep_data.keys():
assert len(ep_data[svc]) == 1, "Unknown data format"
svc_ep_data = ep_data[svc][0]
ep_data[svc] = [
{
'url': svc_ep_data['adminURL'],
'interface': 'admin',
'region': svc_ep_data['region'],
'region_id': self.not_null,
'id': self.not_null},
{
'url': svc_ep_data['publicURL'],
'interface': 'public',
'region': svc_ep_data['region'],
'region_id': self.not_null,
'id': self.not_null},
{
'url': svc_ep_data['internalURL'],
'interface': 'internal',
'region': svc_ep_data['region'],
'region_id': self.not_null,
'id': self.not_null}]
return ep_data
def validate_svc_catalog_endpoint_data(self, expected, actual,
openstack_release=None):
"""Validate service catalog endpoint data. Pick the correct validator
for the OpenStack version. Expected data should be in the v2 format:
{
'service_name1': [
{
'adminURL': adminURL,
'id': id,
'region': region.
'publicURL': publicURL,
'internalURL': internalURL
}],
'service_name2': [
{
'adminURL': adminURL,
'id': id,
'region': region.
'publicURL': publicURL,
'internalURL': internalURL
}],
}
"""
validation_function = self.validate_v2_svc_catalog_endpoint_data
xenial_queens = OPENSTACK_RELEASES_PAIRS.index('xenial_queens')
if openstack_release and openstack_release >= xenial_queens:
validation_function = self.validate_v3_svc_catalog_endpoint_data
expected = self.convert_svc_catalog_endpoint_data_to_v3(expected)
return validation_function(expected, actual)
def validate_v2_svc_catalog_endpoint_data(self, expected, actual):
"""Validate service catalog endpoint data.
Validate a list of actual service catalog endpoints vs a list of
@ -310,6 +424,7 @@ class OpenStackAmuletUtils(AmuletUtils):
self.log.debug('Checking if tenant exists ({})...'.format(tenant))
return tenant in [t.name for t in keystone.tenants.list()]
@retry_on_exception(num_retries=5, base_delay=1)
def keystone_wait_for_propagation(self, sentry_relation_pairs,
api_version):
"""Iterate over list of sentry and relation tuples and verify that
@ -328,7 +443,7 @@ class OpenStackAmuletUtils(AmuletUtils):
if rel.get('api_version') != str(api_version):
raise Exception("api_version not propagated through relation"
" data yet ('{}' != '{}')."
"".format(rel['api_version'], api_version))
"".format(rel.get('api_version'), api_version))
def keystone_configure_api_version(self, sentry_relation_pairs, deployment,
api_version):
@ -350,16 +465,13 @@ class OpenStackAmuletUtils(AmuletUtils):
deployment._auto_wait_for_status()
self.keystone_wait_for_propagation(sentry_relation_pairs, api_version)
def authenticate_cinder_admin(self, keystone_sentry, username,
password, tenant, api_version=2):
def authenticate_cinder_admin(self, keystone, api_version=2):
"""Authenticates admin user with cinder."""
# NOTE(beisner): cinder python client doesn't accept tokens.
keystone_ip = keystone_sentry.info['public-address']
ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8'))
self.log.debug('Authenticating cinder admin...')
_clients = {
1: cinder_client.Client,
2: cinder_clientv2.Client}
return _clients[api_version](username, password, tenant, ept)
return _clients[api_version](session=keystone.session)
def authenticate_keystone(self, keystone_ip, username, password,
api_version=False, admin_port=False,
@ -367,13 +479,36 @@ class OpenStackAmuletUtils(AmuletUtils):
project_domain_name=None, project_name=None):
"""Authenticate with Keystone"""
self.log.debug('Authenticating with keystone...')
port = 5000
if admin_port:
port = 35357
base_ep = "http://{}:{}".format(keystone_ip.strip().decode('utf-8'),
port)
if not api_version or api_version == 2:
ep = base_ep + "/v2.0"
if not api_version:
api_version = 2
sess, auth = self.get_keystone_session(
keystone_ip=keystone_ip,
username=username,
password=password,
api_version=api_version,
admin_port=admin_port,
user_domain_name=user_domain_name,
domain_name=domain_name,
project_domain_name=project_domain_name,
project_name=project_name
)
if api_version == 2:
client = keystone_client.Client(session=sess)
else:
client = keystone_client_v3.Client(session=sess)
# This populates the client.service_catalog
client.auth_ref = auth.get_access(sess)
return client
def get_keystone_session(self, keystone_ip, username, password,
api_version=False, admin_port=False,
user_domain_name=None, domain_name=None,
project_domain_name=None, project_name=None):
"""Return a keystone session object"""
ep = self.get_keystone_endpoint(keystone_ip,
api_version=api_version,
admin_port=admin_port)
if api_version == 2:
auth = v2.Password(
username=username,
password=password,
@ -381,12 +516,7 @@ class OpenStackAmuletUtils(AmuletUtils):
auth_url=ep
)
sess = keystone_session.Session(auth=auth)
client = keystone_client.Client(session=sess)
# This populates the client.service_catalog
client.auth_ref = auth.get_access(sess)
return client
else:
ep = base_ep + "/v3"
auth = v3.Password(
user_domain_name=user_domain_name,
username=username,
@ -397,10 +527,57 @@ class OpenStackAmuletUtils(AmuletUtils):
auth_url=ep
)
sess = keystone_session.Session(auth=auth)
client = keystone_client_v3.Client(session=sess)
# This populates the client.service_catalog
client.auth_ref = auth.get_access(sess)
return client
return (sess, auth)
def get_keystone_endpoint(self, keystone_ip, api_version=None,
admin_port=False):
"""Return keystone endpoint"""
port = 5000
if admin_port:
port = 35357
base_ep = "http://{}:{}".format(keystone_ip.strip().decode('utf-8'),
port)
if api_version == 2:
ep = base_ep + "/v2.0"
else:
ep = base_ep + "/v3"
return ep
def get_default_keystone_session(self, keystone_sentry,
openstack_release=None, api_version=2):
"""Return a keystone session object and client object assuming standard
default settings
Example call in amulet tests:
self.keystone_session, self.keystone = u.get_default_keystone_session(
self.keystone_sentry,
openstack_release=self._get_openstack_release())
The session can then be used to auth other clients:
neutronclient.Client(session=session)
aodh_client.Client(session=session)
eyc
"""
self.log.debug('Authenticating keystone admin...')
# 11 => xenial_queens
if api_version == 3 or (openstack_release and openstack_release >= 11):
client_class = keystone_client_v3.Client
api_version = 3
else:
client_class = keystone_client.Client
keystone_ip = keystone_sentry.info['public-address']
session, auth = self.get_keystone_session(
keystone_ip,
api_version=api_version,
username='admin',
password='openstack',
project_name='admin',
user_domain_name='admin_domain',
project_domain_name='admin_domain')
client = client_class(session=session)
# This populates the client.service_catalog
client.auth_ref = auth.get_access(session)
return session, client
def authenticate_keystone_admin(self, keystone_sentry, user, password,
tenant=None, api_version=None,

View File

@ -0,0 +1,227 @@
# Copyright 2014-2018 Canonical Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Common python helper functions used for OpenStack charm certificats.
import os
import json
from charmhelpers.contrib.network.ip import (
get_hostname,
resolve_network_cidr,
)
from charmhelpers.core.hookenv import (
local_unit,
network_get_primary_address,
config,
relation_get,
unit_get,
NoNetworkBinding,
log,
WARNING,
)
from charmhelpers.contrib.openstack.ip import (
ADMIN,
resolve_address,
get_vip_in_network,
INTERNAL,
PUBLIC,
ADDRESS_MAP)
from charmhelpers.core.host import (
mkdir,
write_file,
)
from charmhelpers.contrib.hahelpers.apache import (
install_ca_cert
)
class CertRequest(object):
"""Create a request for certificates to be generated
"""
def __init__(self, json_encode=True):
self.entries = []
self.hostname_entry = None
self.json_encode = json_encode
def add_entry(self, net_type, cn, addresses):
"""Add a request to the batch
:param net_type: str netwrok space name request is for
:param cn: str Canonical Name for certificate
:param addresses: [] List of addresses to be used as SANs
"""
self.entries.append({
'cn': cn,
'addresses': addresses})
def add_hostname_cn(self):
"""Add a request for the hostname of the machine"""
ip = unit_get('private-address')
addresses = [ip]
# If a vip is being used without os-hostname config or
# network spaces then we need to ensure the local units
# cert has the approriate vip in the SAN list
vip = get_vip_in_network(resolve_network_cidr(ip))
if vip:
addresses.append(vip)
self.hostname_entry = {
'cn': get_hostname(ip),
'addresses': addresses}
def add_hostname_cn_ip(self, addresses):
"""Add an address to the SAN list for the hostname request
:param addr: [] List of address to be added
"""
for addr in addresses:
if addr not in self.hostname_entry['addresses']:
self.hostname_entry['addresses'].append(addr)
def get_request(self):
"""Generate request from the batched up entries
"""
if self.hostname_entry:
self.entries.append(self.hostname_entry)
request = {}
for entry in self.entries:
sans = sorted(list(set(entry['addresses'])))
request[entry['cn']] = {'sans': sans}
if self.json_encode:
return {'cert_requests': json.dumps(request, sort_keys=True)}
else:
return {'cert_requests': request}
def get_certificate_request(json_encode=True):
"""Generate a certificatee requests based on the network confioguration
"""
req = CertRequest(json_encode=json_encode)
req.add_hostname_cn()
# Add os-hostname entries
for net_type in [INTERNAL, ADMIN, PUBLIC]:
net_config = config(ADDRESS_MAP[net_type]['override'])
try:
net_addr = resolve_address(endpoint_type=net_type)
ip = network_get_primary_address(
ADDRESS_MAP[net_type]['binding'])
addresses = [net_addr, ip]
vip = get_vip_in_network(resolve_network_cidr(ip))
if vip:
addresses.append(vip)
if net_config:
req.add_entry(
net_type,
net_config,
addresses)
else:
# There is network address with no corresponding hostname.
# Add the ip to the hostname cert to allow for this.
req.add_hostname_cn_ip(addresses)
except NoNetworkBinding:
log("Skipping request for certificate for ip in {} space, no "
"local address found".format(net_type), WARNING)
return req.get_request()
def create_ip_cert_links(ssl_dir, custom_hostname_link=None):
"""Create symlinks for SAN records
:param ssl_dir: str Directory to create symlinks in
:param custom_hostname_link: str Additional link to be created
"""
hostname = get_hostname(unit_get('private-address'))
hostname_cert = os.path.join(
ssl_dir,
'cert_{}'.format(hostname))
hostname_key = os.path.join(
ssl_dir,
'key_{}'.format(hostname))
# Add links to hostname cert, used if os-hostname vars not set
for net_type in [INTERNAL, ADMIN, PUBLIC]:
try:
addr = resolve_address(endpoint_type=net_type)
cert = os.path.join(ssl_dir, 'cert_{}'.format(addr))
key = os.path.join(ssl_dir, 'key_{}'.format(addr))
if os.path.isfile(hostname_cert) and not os.path.isfile(cert):
os.symlink(hostname_cert, cert)
os.symlink(hostname_key, key)
except NoNetworkBinding:
log("Skipping creating cert symlink for ip in {} space, no "
"local address found".format(net_type), WARNING)
if custom_hostname_link:
custom_cert = os.path.join(
ssl_dir,
'cert_{}'.format(custom_hostname_link))
custom_key = os.path.join(
ssl_dir,
'key_{}'.format(custom_hostname_link))
if os.path.isfile(hostname_cert) and not os.path.isfile(custom_cert):
os.symlink(hostname_cert, custom_cert)
os.symlink(hostname_key, custom_key)
def install_certs(ssl_dir, certs, chain=None):
"""Install the certs passed into the ssl dir and append the chain if
provided.
:param ssl_dir: str Directory to create symlinks in
:param certs: {} {'cn': {'cert': 'CERT', 'key': 'KEY'}}
:param chain: str Chain to be appended to certs
"""
for cn, bundle in certs.items():
cert_filename = 'cert_{}'.format(cn)
key_filename = 'key_{}'.format(cn)
cert_data = bundle['cert']
if chain:
# Append chain file so that clients that trust the root CA will
# trust certs signed by an intermediate in the chain
cert_data = cert_data + chain
write_file(
path=os.path.join(ssl_dir, cert_filename),
content=cert_data, perms=0o640)
write_file(
path=os.path.join(ssl_dir, key_filename),
content=bundle['key'], perms=0o640)
def process_certificates(service_name, relation_id, unit,
custom_hostname_link=None):
"""Process the certificates supplied down the relation
:param service_name: str Name of service the certifcates are for.
:param relation_id: str Relation id providing the certs
:param unit: str Unit providing the certs
:param custom_hostname_link: str Name of custom link to create
"""
data = relation_get(rid=relation_id, unit=unit)
ssl_dir = os.path.join('/etc/apache2/ssl/', service_name)
mkdir(path=ssl_dir)
name = local_unit().replace('/', '_')
certs = data.get('{}.processed_requests'.format(name))
chain = data.get('chain')
ca = data.get('ca')
if certs:
certs = json.loads(certs)
install_ca_cert(ca.encode())
install_certs(ssl_dir, certs, chain)
create_ip_cert_links(
ssl_dir,
custom_hostname_link=custom_hostname_link)

View File

@ -190,8 +190,8 @@ class OSContextGenerator(object):
class SharedDBContext(OSContextGenerator):
interfaces = ['shared-db']
def __init__(self,
database=None, user=None, relation_prefix=None, ssl_dir=None):
def __init__(self, database=None, user=None, relation_prefix=None,
ssl_dir=None, relation_id=None):
"""Allows inspecting relation for settings prefixed with
relation_prefix. This is useful for parsing access for multiple
databases returned via the shared-db interface (eg, nova_password,
@ -202,6 +202,7 @@ class SharedDBContext(OSContextGenerator):
self.user = user
self.ssl_dir = ssl_dir
self.rel_name = self.interfaces[0]
self.relation_id = relation_id
def __call__(self):
self.database = self.database or config('database')
@ -235,7 +236,12 @@ class SharedDBContext(OSContextGenerator):
if self.relation_prefix:
password_setting = self.relation_prefix + '_password'
for rid in relation_ids(self.interfaces[0]):
if self.relation_id:
rids = [self.relation_id]
else:
rids = relation_ids(self.interfaces[0])
for rid in rids:
self.related = True
for unit in related_units(rid):
rdata = relation_get(rid=rid, unit=unit)
@ -384,6 +390,7 @@ class IdentityServiceContext(OSContextGenerator):
# so a missing value just indicates keystone needs
# upgrading
ctxt['admin_tenant_id'] = rdata.get('service_tenant_id')
ctxt['admin_domain_id'] = rdata.get('service_domain_id')
return ctxt
return {}
@ -447,11 +454,13 @@ class IdentityCredentialsContext(IdentityServiceContext):
class AMQPContext(OSContextGenerator):
def __init__(self, ssl_dir=None, rel_name='amqp', relation_prefix=None):
def __init__(self, ssl_dir=None, rel_name='amqp', relation_prefix=None,
relation_id=None):
self.ssl_dir = ssl_dir
self.rel_name = rel_name
self.relation_prefix = relation_prefix
self.interfaces = [rel_name]
self.relation_id = relation_id
def __call__(self):
log('Generating template context for amqp', level=DEBUG)
@ -472,7 +481,11 @@ class AMQPContext(OSContextGenerator):
raise OSContextError
ctxt = {}
for rid in relation_ids(self.rel_name):
if self.relation_id:
rids = [self.relation_id]
else:
rids = relation_ids(self.rel_name)
for rid in rids:
ha_vip_only = False
self.related = True
transport_hosts = None
@ -788,17 +801,18 @@ class ApacheSSLContext(OSContextGenerator):
ssl_dir = os.path.join('/etc/apache2/ssl/', self.service_namespace)
mkdir(path=ssl_dir)
cert, key = get_cert(cn)
if cn:
cert_filename = 'cert_{}'.format(cn)
key_filename = 'key_{}'.format(cn)
else:
cert_filename = 'cert'
key_filename = 'key'
if cert and key:
if cn:
cert_filename = 'cert_{}'.format(cn)
key_filename = 'key_{}'.format(cn)
else:
cert_filename = 'cert'
key_filename = 'key'
write_file(path=os.path.join(ssl_dir, cert_filename),
content=b64decode(cert))
write_file(path=os.path.join(ssl_dir, key_filename),
content=b64decode(key))
write_file(path=os.path.join(ssl_dir, cert_filename),
content=b64decode(cert), perms=0o640)
write_file(path=os.path.join(ssl_dir, key_filename),
content=b64decode(key), perms=0o640)
def configure_ca(self):
ca_cert = get_ca_cert()
@ -870,23 +884,31 @@ class ApacheSSLContext(OSContextGenerator):
if not self.external_ports or not https():
return {}
self.configure_ca()
use_keystone_ca = True
for rid in relation_ids('certificates'):
if related_units(rid):
use_keystone_ca = False
if use_keystone_ca:
self.configure_ca()
self.enable_modules()
ctxt = {'namespace': self.service_namespace,
'endpoints': [],
'ext_ports': []}
cns = self.canonical_names()
if cns:
for cn in cns:
self.configure_cert(cn)
else:
# Expect cert/key provided in config (currently assumed that ca
# uses ip for cn)
for net_type in (INTERNAL, ADMIN, PUBLIC):
cn = resolve_address(endpoint_type=net_type)
self.configure_cert(cn)
if use_keystone_ca:
cns = self.canonical_names()
if cns:
for cn in cns:
self.configure_cert(cn)
else:
# Expect cert/key provided in config (currently assumed that ca
# uses ip for cn)
for net_type in (INTERNAL, ADMIN, PUBLIC):
cn = resolve_address(endpoint_type=net_type)
self.configure_cert(cn)
addresses = self.get_network_addresses()
for address, endpoint in addresses:
@ -1872,10 +1894,11 @@ class EnsureDirContext(OSContextGenerator):
context is needed to do that before rendering a template.
'''
def __init__(self, dirname):
def __init__(self, dirname, **kwargs):
'''Used merely to ensure that a given directory exists.'''
self.dirname = dirname
self.kwargs = kwargs
def __call__(self):
mkdir(self.dirname)
mkdir(self.dirname, **self.kwargs)
return {}

View File

@ -184,3 +184,13 @@ def resolve_address(endpoint_type=PUBLIC, override=True):
"clustered=%s)" % (net_type, clustered))
return resolved_address
def get_vip_in_network(network):
matching_vip = None
vips = config('vip')
if vips:
for vip in vips.split():
if is_address_in_network(network, vip):
matching_vip = vip
return matching_vip

View File

@ -0,0 +1,5 @@
[oslo_middleware]
# Bug #1758675
enable_proxy_headers_parsing = true

View File

@ -5,4 +5,7 @@ transport_url = {{ transport_url }}
{% if notification_topics -%}
topics = {{ notification_topics }}
{% endif -%}
{% if notification_format -%}
notification_format = {{ notification_format }}
{% endif -%}
{% endif -%}

View File

@ -133,6 +133,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
('zesty', 'ocata'),
('artful', 'pike'),
('bionic', 'queens'),
('cosmic', 'rocky'),
])
@ -151,6 +152,7 @@ OPENSTACK_CODENAMES = OrderedDict([
('2017.1', 'ocata'),
('2017.2', 'pike'),
('2018.1', 'queens'),
('2018.2', 'rocky'),
])
# The ugly duckling - must list releases oldest to newest
@ -182,7 +184,9 @@ SWIFT_CODENAMES = OrderedDict([
('pike',
['2.13.0', '2.15.0']),
('queens',
['2.16.0']),
['2.16.0', '2.17.0']),
('rocky',
['2.18.0']),
])
# >= Liberty version->codename mapping
@ -306,7 +310,7 @@ def get_os_codename_install_source(src):
if src.startswith('cloud:'):
ca_rel = src.split(':')[1]
ca_rel = ca_rel.split('%s-' % ubuntu_rel)[1].split('/')[0]
ca_rel = ca_rel.split('-')[1].split('/')[0]
return ca_rel
# Best guess match based on deb string provided

View File

@ -0,0 +1,126 @@
# Copyright 2018 Canonical Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import charmhelpers.contrib.openstack.alternatives as alternatives
import charmhelpers.contrib.openstack.context as context
import charmhelpers.core.hookenv as hookenv
import charmhelpers.core.host as host
import charmhelpers.core.templating as templating
import charmhelpers.core.unitdata as unitdata
VAULTLOCKER_BACKEND = 'charm-vaultlocker'
class VaultKVContext(context.OSContextGenerator):
"""Vault KV context for interaction with vault-kv interfaces"""
interfaces = ['secrets-storage']
def __init__(self, secret_backend=None):
super(context.OSContextGenerator, self).__init__()
self.secret_backend = (
secret_backend or 'charm-{}'.format(hookenv.service_name())
)
def __call__(self):
db = unitdata.kv()
last_token = db.get('last-token')
secret_id = db.get('secret-id')
for relation_id in hookenv.relation_ids(self.interfaces[0]):
for unit in hookenv.related_units(relation_id):
data = hookenv.relation_get(unit=unit,
rid=relation_id)
vault_url = data.get('vault_url')
role_id = data.get('{}_role_id'.format(hookenv.local_unit()))
token = data.get('{}_token'.format(hookenv.local_unit()))
if all([vault_url, role_id, token]):
token = json.loads(token)
vault_url = json.loads(vault_url)
# Tokens may change when secret_id's are being
# reissued - if so use token to get new secret_id
if token != last_token:
secret_id = retrieve_secret_id(
url=vault_url,
token=token
)
db.set('secret-id', secret_id)
db.set('last-token', token)
db.flush()
ctxt = {
'vault_url': vault_url,
'role_id': json.loads(role_id),
'secret_id': secret_id,
'secret_backend': self.secret_backend,
}
vault_ca = data.get('vault_ca')
if vault_ca:
ctxt['vault_ca'] = json.loads(vault_ca)
self.complete = True
return ctxt
return {}
def write_vaultlocker_conf(context, priority=100):
"""Write vaultlocker configuration to disk and install alternative
:param context: Dict of data from vault-kv relation
:ptype: context: dict
:param priority: Priority of alternative configuration
:ptype: priority: int"""
charm_vl_path = "/var/lib/charm/{}/vaultlocker.conf".format(
hookenv.service_name()
)
host.mkdir(os.path.dirname(charm_vl_path), perms=0o700)
templating.render(source='vaultlocker.conf.j2',
target=charm_vl_path,
context=context, perms=0o600),
alternatives.install_alternative('vaultlocker.conf',
'/etc/vaultlocker/vaultlocker.conf',
charm_vl_path, priority)
def vault_relation_complete(backend=None):
"""Determine whether vault relation is complete
:param backend: Name of secrets backend requested
:ptype backend: string
:returns: whether the relation to vault is complete
:rtype: bool"""
vault_kv = VaultKVContext(secret_backend=backend or VAULTLOCKER_BACKEND)
vault_kv()
return vault_kv.complete
# TODO: contrib a high level unwrap method to hvac that works
def retrieve_secret_id(url, token):
"""Retrieve a response-wrapped secret_id from Vault
:param url: URL to Vault Server
:ptype url: str
:param token: One shot Token to use
:ptype token: str
:returns: secret_id to use for Vault Access
:rtype: str"""
import hvac
client = hvac.Client(url=url, token=token)
response = client._post('/v1/sys/wrapping/unwrap')
if response.status_code == 200:
data = response.json()
return data['data']['secret_id']

View File

@ -291,7 +291,7 @@ class Pool(object):
class ReplicatedPool(Pool):
def __init__(self, service, name, pg_num=None, replicas=2,
percent_data=10.0):
percent_data=10.0, app_name=None):
super(ReplicatedPool, self).__init__(service=service, name=name)
self.replicas = replicas
if pg_num:
@ -301,6 +301,10 @@ class ReplicatedPool(Pool):
self.pg_num = min(pg_num, max_pgs)
else:
self.pg_num = self.get_pgs(self.replicas, percent_data)
if app_name:
self.app_name = app_name
else:
self.app_name = 'unknown'
def create(self):
if not pool_exists(self.service, self.name):
@ -313,6 +317,12 @@ class ReplicatedPool(Pool):
update_pool(client=self.service,
pool=self.name,
settings={'size': str(self.replicas)})
try:
set_app_name_for_pool(client=self.service,
pool=self.name,
name=self.app_name)
except CalledProcessError:
log('Could not set app name for pool {}'.format(self.name, level=WARNING))
except CalledProcessError:
raise
@ -320,10 +330,14 @@ class ReplicatedPool(Pool):
# Default jerasure erasure coded pool
class ErasurePool(Pool):
def __init__(self, service, name, erasure_code_profile="default",
percent_data=10.0):
percent_data=10.0, app_name=None):
super(ErasurePool, self).__init__(service=service, name=name)
self.erasure_code_profile = erasure_code_profile
self.percent_data = percent_data
if app_name:
self.app_name = app_name
else:
self.app_name = 'unknown'
def create(self):
if not pool_exists(self.service, self.name):
@ -355,6 +369,12 @@ class ErasurePool(Pool):
'erasure', self.erasure_code_profile]
try:
check_call(cmd)
try:
set_app_name_for_pool(client=self.service,
pool=self.name,
name=self.app_name)
except CalledProcessError:
log('Could not set app name for pool {}'.format(self.name, level=WARNING))
except CalledProcessError:
raise
@ -778,6 +798,25 @@ def update_pool(client, pool, settings):
check_call(cmd)
def set_app_name_for_pool(client, pool, name):
"""
Calls `osd pool application enable` for the specified pool name
:param client: Name of the ceph client to use
:type client: str
:param pool: Pool to set app name for
:type pool: str
:param name: app name for the specified pool
:type name: str
:raises: CalledProcessError if ceph call fails
"""
if ceph_version() >= '12.0.0':
cmd = ['ceph', '--id', client, 'osd', 'pool',
'application', 'enable', pool, name]
check_call(cmd)
def create_pool(service, name, replicas=3, pg_num=None):
"""Create a new RADOS pool."""
if pool_exists(service, name):

View File

@ -151,3 +151,32 @@ def extend_logical_volume_by_device(lv_name, block_device):
'''
cmd = ['lvextend', lv_name, block_device]
check_call(cmd)
def create_logical_volume(lv_name, volume_group, size=None):
'''
Create a new logical volume in an existing volume group
:param lv_name: str: name of logical volume to be created.
:param volume_group: str: Name of volume group to use for the new volume.
:param size: str: Size of logical volume to create (100% if not supplied)
:raises subprocess.CalledProcessError: in the event that the lvcreate fails.
'''
if size:
check_call([
'lvcreate',
'--yes',
'-L',
'{}'.format(size),
'-n', lv_name, volume_group
])
# create the lv with all the space available, this is needed because the
# system call is different for LVM
else:
check_call([
'lvcreate',
'--yes',
'-l',
'100%FREE',
'-n', lv_name, volume_group
])

View File

@ -67,3 +67,19 @@ def is_device_mounted(device):
except Exception:
return False
return bool(re.search(r'MOUNTPOINT=".+"', out))
def mkfs_xfs(device, force=False):
"""Format device with XFS filesystem.
By default this should fail if the device already has a filesystem on it.
:param device: Full path to device to format
:ptype device: tr
:param force: Force operation
:ptype: force: boolean"""
cmd = ['mkfs.xfs']
if force:
cmd.append("-f")
cmd += ['-i', 'size=1024', device]
check_call(cmd)

View File

@ -27,6 +27,7 @@ import glob
import os
import json
import yaml
import re
import subprocess
import sys
import errno
@ -67,7 +68,7 @@ def cached(func):
@wraps(func)
def wrapper(*args, **kwargs):
global cache
key = str((func, args, kwargs))
key = json.dumps((func, args, kwargs), sort_keys=True, default=str)
try:
return cache[key]
except KeyError:
@ -289,7 +290,7 @@ class Config(dict):
self.implicit_save = True
self._prev_dict = None
self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
if os.path.exists(self.path):
if os.path.exists(self.path) and os.stat(self.path).st_size:
self.load_previous()
atexit(self._implicit_save)
@ -309,7 +310,11 @@ class Config(dict):
"""
self.path = path or self.path
with open(self.path) as f:
self._prev_dict = json.load(f)
try:
self._prev_dict = json.load(f)
except ValueError as e:
log('Unable to parse previous config data - {}'.format(str(e)),
level=ERROR)
for k, v in copy.deepcopy(self._prev_dict).items():
if k not in self:
self[k] = v
@ -353,22 +358,40 @@ class Config(dict):
self.save()
@cached
_cache_config = None
def config(scope=None):
"""Juju charm configuration"""
config_cmd_line = ['config-get']
if scope is not None:
config_cmd_line.append(scope)
else:
config_cmd_line.append('--all')
config_cmd_line.append('--format=json')
"""
Get the juju charm configuration (scope==None) or individual key,
(scope=str). The returned value is a Python data structure loaded as
JSON from the Juju config command.
:param scope: If set, return the value for the specified key.
:type scope: Optional[str]
:returns: Either the whole config as a Config, or a key from it.
:rtype: Any
"""
global _cache_config
config_cmd_line = ['config-get', '--all', '--format=json']
try:
config_data = json.loads(
subprocess.check_output(config_cmd_line).decode('UTF-8'))
# JSON Decode Exception for Python3.5+
exc_json = json.decoder.JSONDecodeError
except AttributeError:
# JSON Decode Exception for Python2.7 through Python3.4
exc_json = ValueError
try:
if _cache_config is None:
config_data = json.loads(
subprocess.check_output(config_cmd_line).decode('UTF-8'))
_cache_config = Config(config_data)
if scope is not None:
return config_data
return Config(config_data)
except ValueError:
return _cache_config.get(scope)
return _cache_config
except (exc_json, UnicodeDecodeError) as e:
log('Unable to parse output from config-get: config_cmd_line="{}" '
'message="{}"'
.format(config_cmd_line, str(e)), level=ERROR)
return None
@ -949,6 +972,13 @@ def application_version_set(version):
log("Application Version: {}".format(version))
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
def goal_state():
"""Juju goal state values"""
cmd = ['goal-state', '--format=json']
return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
def is_leader():
"""Does the current unit hold the juju leadership
@ -1043,7 +1073,6 @@ def juju_version():
universal_newlines=True).strip()
@cached
def has_juju_version(minimum_version):
"""Return True if the Juju version is at least the provided version"""
return LooseVersion(juju_version()) >= LooseVersion(minimum_version)
@ -1103,6 +1132,8 @@ def _run_atexit():
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
def network_get_primary_address(binding):
'''
Deprecated since Juju 2.3; use network_get()
Retrieve the primary network address for a named binding
:param binding: string. The name of a relation of extra-binding
@ -1123,7 +1154,6 @@ def network_get_primary_address(binding):
return response
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
def network_get(endpoint, relation_id=None):
"""
Retrieve the network details for a relation endpoint
@ -1131,24 +1161,20 @@ def network_get(endpoint, relation_id=None):
:param endpoint: string. The name of a relation endpoint
:param relation_id: int. The ID of the relation for the current context.
:return: dict. The loaded YAML output of the network-get query.
:raise: NotImplementedError if run on Juju < 2.1
:raise: NotImplementedError if request not supported by the Juju version.
"""
if not has_juju_version('2.2'):
raise NotImplementedError(juju_version()) # earlier versions require --primary-address
if relation_id and not has_juju_version('2.3'):
raise NotImplementedError # 2.3 added the -r option
cmd = ['network-get', endpoint, '--format', 'yaml']
if relation_id:
cmd.append('-r')
cmd.append(relation_id)
try:
response = subprocess.check_output(
cmd,
stderr=subprocess.STDOUT).decode('UTF-8').strip()
except CalledProcessError as e:
# Early versions of Juju 2.0.x required the --primary-address argument.
# We catch that condition here and raise NotImplementedError since
# the requested semantics are not available - the caller can then
# use the network_get_primary_address() method instead.
if '--primary-address is currently required' in e.output.decode('UTF-8'):
raise NotImplementedError
raise
response = subprocess.check_output(
cmd,
stderr=subprocess.STDOUT).decode('UTF-8').strip()
return yaml.safe_load(response)
@ -1204,9 +1230,23 @@ def iter_units_for_relation_name(relation_name):
def ingress_address(rid=None, unit=None):
"""
Retrieve the ingress-address from a relation when available. Otherwise,
return the private-address. This function is to be used on the consuming
side of the relation.
Retrieve the ingress-address from a relation when available.
Otherwise, return the private-address.
When used on the consuming side of the relation (unit is a remote
unit), the ingress-address is the IP address that this unit needs
to use to reach the provided service on the remote unit.
When used on the providing side of the relation (unit == local_unit()),
the ingress-address is the IP address that is advertised to remote
units on this relation. Remote units need to use this address to
reach the local provided service on this unit.
Note that charms may document some other method to use in
preference to the ingress_address(), such as an address provided
on a different relation attribute or a service discovery mechanism.
This allows charms to redirect inbound connections to their peers
or different applications such as load balancers.
Usage:
addresses = [ingress_address(rid=u.rid, unit=u.unit)
@ -1220,3 +1260,40 @@ def ingress_address(rid=None, unit=None):
settings = relation_get(rid=rid, unit=unit)
return (settings.get('ingress-address') or
settings.get('private-address'))
def egress_subnets(rid=None, unit=None):
"""
Retrieve the egress-subnets from a relation.
This function is to be used on the providing side of the
relation, and provides the ranges of addresses that client
connections may come from. The result is uninteresting on
the consuming side of a relation (unit == local_unit()).
Returns a stable list of subnets in CIDR format.
eg. ['192.168.1.0/24', '2001::F00F/128']
If egress-subnets is not available, falls back to using the published
ingress-address, or finally private-address.
:param rid: string relation id
:param unit: string unit name
:side effect: calls relation_get
:return: list of subnets in CIDR format. eg. ['192.168.1.0/24', '2001::F00F/128']
"""
def _to_range(addr):
if re.search(r'^(?:\d{1,3}\.){3}\d{1,3}$', addr) is not None:
addr += '/32'
elif ':' in addr and '/' not in addr: # IPv6
addr += '/128'
return addr
settings = relation_get(rid=rid, unit=unit)
if 'egress-subnets' in settings:
return [n.strip() for n in settings['egress-subnets'].split(',') if n.strip()]
if 'ingress-address' in settings:
return [_to_range(settings['ingress-address'])]
if 'private-address' in settings:
return [_to_range(settings['private-address'])]
return [] # Should never happen

View File

@ -972,6 +972,20 @@ def is_container():
def add_to_updatedb_prunepath(path, updatedb_path=UPDATEDB_PATH):
"""Adds the specified path to the mlocate's udpatedb.conf PRUNEPATH list.
This method has no effect if the path specified by updatedb_path does not
exist or is not a file.
@param path: string the path to add to the updatedb.conf PRUNEPATHS value
@param updatedb_path: the path the updatedb.conf file
"""
if not os.path.exists(updatedb_path) or os.path.isdir(updatedb_path):
# If the updatedb.conf file doesn't exist then don't attempt to update
# the file as the package providing mlocate may not be installed on
# the local system
return
with open(updatedb_path, 'r+') as f_id:
updatedb_text = f_id.read()
output = updatedb(updatedb_text, path)
@ -993,7 +1007,7 @@ def updatedb(updatedb_text, new_path):
return output
def modulo_distribution(modulo=3, wait=30):
def modulo_distribution(modulo=3, wait=30, non_zero_wait=False):
""" Modulo distribution
This helper uses the unit number, a modulo value and a constant wait time
@ -1015,7 +1029,14 @@ def modulo_distribution(modulo=3, wait=30):
@param modulo: int The modulo number creates the group distribution
@param wait: int The constant time wait value
@param non_zero_wait: boolean Override unit % modulo == 0,
return modulo * wait. Used to avoid collisions with
leader nodes which are often given priority.
@return: int Calculated time to wait for unit operation
"""
unit_number = int(local_unit().split('/')[1])
return (unit_number % modulo) * wait
calculated_wait_time = (unit_number % modulo) * wait
if non_zero_wait and calculated_wait_time == 0:
return modulo * wait
else:
return calculated_wait_time

View File

@ -307,23 +307,34 @@ class PortManagerCallback(ManagerCallback):
"""
def __call__(self, manager, service_name, event_name):
service = manager.get_service(service_name)
new_ports = service.get('ports', [])
# turn this generator into a list,
# as we'll be going over it multiple times
new_ports = list(service.get('ports', []))
port_file = os.path.join(hookenv.charm_dir(), '.{}.ports'.format(service_name))
if os.path.exists(port_file):
with open(port_file) as fp:
old_ports = fp.read().split(',')
for old_port in old_ports:
if bool(old_port):
old_port = int(old_port)
if old_port not in new_ports:
hookenv.close_port(old_port)
if bool(old_port) and not self.ports_contains(old_port, new_ports):
hookenv.close_port(old_port)
with open(port_file, 'w') as fp:
fp.write(','.join(str(port) for port in new_ports))
for port in new_ports:
# A port is either a number or 'ICMP'
protocol = 'TCP'
if str(port).upper() == 'ICMP':
protocol = 'ICMP'
if event_name == 'start':
hookenv.open_port(port)
hookenv.open_port(port, protocol)
elif event_name == 'stop':
hookenv.close_port(port)
hookenv.close_port(port, protocol)
def ports_contains(self, port, ports):
if not bool(port):
return False
if str(port).upper() != 'ICMP':
port = int(port)
return port in ports
def service_stop(service_name):

View File

@ -31,18 +31,22 @@ __author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
def create(sysctl_dict, sysctl_file):
"""Creates a sysctl.conf file from a YAML associative array
:param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }"
:param sysctl_dict: a dict or YAML-formatted string of sysctl
options eg "{ 'kernel.max_pid': 1337 }"
:type sysctl_dict: str
:param sysctl_file: path to the sysctl file to be saved
:type sysctl_file: str or unicode
:returns: None
"""
try:
sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
except yaml.YAMLError:
log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
level=ERROR)
return
if type(sysctl_dict) is not dict:
try:
sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
except yaml.YAMLError:
log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
level=ERROR)
return
else:
sysctl_dict_parsed = sysctl_dict
with open(sysctl_file, "w") as fd:
for key, value in sysctl_dict_parsed.items():

View File

@ -166,6 +166,10 @@ class Storage(object):
To support dicts, lists, integer, floats, and booleans values
are automatically json encoded/decoded.
Note: to facilitate unit testing, ':memory:' can be passed as the
path parameter which causes sqlite3 to only build the db in memory.
This should only be used for testing purposes.
"""
def __init__(self, path=None):
self.db_path = path
@ -175,8 +179,9 @@ class Storage(object):
else:
self.db_path = os.path.join(
os.environ.get('CHARM_DIR', ''), '.unit-state.db')
with open(self.db_path, 'a') as f:
os.fchmod(f.fileno(), 0o600)
if self.db_path != ':memory:':
with open(self.db_path, 'a') as f:
os.fchmod(f.fileno(), 0o600)
self.conn = sqlite3.connect('%s' % self.db_path)
self.cursor = self.conn.cursor()
self.revision = None

View File

@ -44,6 +44,7 @@ ARCH_TO_PROPOSED_POCKET = {
'x86_64': PROPOSED_POCKET,
'ppc64le': PROPOSED_PORTS_POCKET,
'aarch64': PROPOSED_PORTS_POCKET,
's390x': PROPOSED_PORTS_POCKET,
}
CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu"
CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
@ -157,6 +158,14 @@ CLOUD_ARCHIVE_POCKETS = {
'queens/proposed': 'xenial-proposed/queens',
'xenial-queens/proposed': 'xenial-proposed/queens',
'xenial-proposed/queens': 'xenial-proposed/queens',
# Rocky
'rocky': 'bionic-updates/rocky',
'bionic-rocky': 'bionic-updates/rocky',
'bionic-rocky/updates': 'bionic-updates/rocky',
'bionic-updates/rocky': 'bionic-updates/rocky',
'rocky/proposed': 'bionic-proposed/rocky',
'bionic-rocky/proposed': 'bionic-proposed/rocky',
'bionic-proposed/rocky': 'bionic-proposed/rocky',
}
@ -306,7 +315,7 @@ def import_key(key):
cmd = ['apt-key', 'adv', '--keyserver',
'hkp://keyserver.ubuntu.com:80', '--recv-keys', key]
try:
subprocess.check_call(cmd)
_run_with_retries(cmd)
except subprocess.CalledProcessError:
error = "Error importing PGP key '{}'".format(key)
log(error)

View File

@ -50,7 +50,8 @@ class AmuletDeployment(object):
this_service['units'] = 1
self.d.add(this_service['name'], units=this_service['units'],
constraints=this_service.get('constraints'))
constraints=this_service.get('constraints'),
storage=this_service.get('storage'))
for svc in other_services:
if 'location' in svc:
@ -64,7 +65,8 @@ class AmuletDeployment(object):
svc['units'] = 1
self.d.add(svc['name'], charm=branch_location, units=svc['units'],
constraints=svc.get('constraints'))
constraints=svc.get('constraints'),
storage=svc.get('storage'))
def _add_relations(self, relations):
"""Add all of the relations for the services."""

View File

@ -21,6 +21,9 @@ from collections import OrderedDict
from charmhelpers.contrib.amulet.deployment import (
AmuletDeployment
)
from charmhelpers.contrib.openstack.amulet.utils import (
OPENSTACK_RELEASES_PAIRS
)
DEBUG = logging.DEBUG
ERROR = logging.ERROR
@ -271,11 +274,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
release.
"""
# Must be ordered by OpenStack release (not by Ubuntu release):
(self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty,
self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton,
self.yakkety_newton, self.xenial_ocata, self.zesty_ocata,
self.xenial_pike, self.artful_pike, self.xenial_queens,
self.bionic_queens,) = range(13)
for i, os_pair in enumerate(OPENSTACK_RELEASES_PAIRS):
setattr(self, os_pair, i)
releases = {
('trusty', None): self.trusty_icehouse,
@ -291,6 +291,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
('zesty', None): self.zesty_ocata,
('artful', None): self.artful_pike,
('bionic', None): self.bionic_queens,
('bionic', 'cloud:bionic-rocky'): self.bionic_rocky,
('cosmic', None): self.cosmic_rocky,
}
return releases[(self.series, self.openstack)]
@ -306,6 +308,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
('zesty', 'ocata'),
('artful', 'pike'),
('bionic', 'queens'),
('cosmic', 'rocky'),
])
if self.openstack:
os_origin = self.openstack.split(':')[1]

View File

@ -40,6 +40,7 @@ import novaclient
import pika
import swiftclient
from charmhelpers.core.decorators import retry_on_exception
from charmhelpers.contrib.amulet.utils import (
AmuletUtils
)
@ -50,6 +51,13 @@ ERROR = logging.ERROR
NOVA_CLIENT_VERSION = "2"
OPENSTACK_RELEASES_PAIRS = [
'trusty_icehouse', 'trusty_kilo', 'trusty_liberty',
'trusty_mitaka', 'xenial_mitaka', 'xenial_newton',
'yakkety_newton', 'xenial_ocata', 'zesty_ocata',
'xenial_pike', 'artful_pike', 'xenial_queens',
'bionic_queens', 'bionic_rocky', 'cosmic_rocky']
class OpenStackAmuletUtils(AmuletUtils):
"""OpenStack amulet utilities.
@ -63,7 +71,34 @@ class OpenStackAmuletUtils(AmuletUtils):
super(OpenStackAmuletUtils, self).__init__(log_level)
def validate_endpoint_data(self, endpoints, admin_port, internal_port,
public_port, expected):
public_port, expected, openstack_release=None):
"""Validate endpoint data. Pick the correct validator based on
OpenStack release. Expected data should be in the v2 format:
{
'id': id,
'region': region,
'adminurl': adminurl,
'internalurl': internalurl,
'publicurl': publicurl,
'service_id': service_id}
"""
validation_function = self.validate_v2_endpoint_data
xenial_queens = OPENSTACK_RELEASES_PAIRS.index('xenial_queens')
if openstack_release and openstack_release >= xenial_queens:
validation_function = self.validate_v3_endpoint_data
expected = {
'id': expected['id'],
'region': expected['region'],
'region_id': 'RegionOne',
'url': self.valid_url,
'interface': self.not_null,
'service_id': expected['service_id']}
return validation_function(endpoints, admin_port, internal_port,
public_port, expected)
def validate_v2_endpoint_data(self, endpoints, admin_port, internal_port,
public_port, expected):
"""Validate endpoint data.
Validate actual endpoint data vs expected endpoint data. The ports
@ -141,7 +176,86 @@ class OpenStackAmuletUtils(AmuletUtils):
if len(found) != expected_num_eps:
return 'Unexpected number of endpoints found'
def validate_svc_catalog_endpoint_data(self, expected, actual):
def convert_svc_catalog_endpoint_data_to_v3(self, ep_data):
"""Convert v2 endpoint data into v3.
{
'service_name1': [
{
'adminURL': adminURL,
'id': id,
'region': region.
'publicURL': publicURL,
'internalURL': internalURL
}],
'service_name2': [
{
'adminURL': adminURL,
'id': id,
'region': region.
'publicURL': publicURL,
'internalURL': internalURL
}],
}
"""
self.log.warn("Endpoint ID and Region ID validation is limited to not "
"null checks after v2 to v3 conversion")
for svc in ep_data.keys():
assert len(ep_data[svc]) == 1, "Unknown data format"
svc_ep_data = ep_data[svc][0]
ep_data[svc] = [
{
'url': svc_ep_data['adminURL'],
'interface': 'admin',
'region': svc_ep_data['region'],
'region_id': self.not_null,
'id': self.not_null},
{
'url': svc_ep_data['publicURL'],
'interface': 'public',
'region': svc_ep_data['region'],
'region_id': self.not_null,
'id': self.not_null},
{
'url': svc_ep_data['internalURL'],
'interface': 'internal',
'region': svc_ep_data['region'],
'region_id': self.not_null,
'id': self.not_null}]
return ep_data
def validate_svc_catalog_endpoint_data(self, expected, actual,
openstack_release=None):
"""Validate service catalog endpoint data. Pick the correct validator
for the OpenStack version. Expected data should be in the v2 format:
{
'service_name1': [
{
'adminURL': adminURL,
'id': id,
'region': region.
'publicURL': publicURL,
'internalURL': internalURL
}],
'service_name2': [
{
'adminURL': adminURL,
'id': id,
'region': region.
'publicURL': publicURL,
'internalURL': internalURL
}],
}
"""
validation_function = self.validate_v2_svc_catalog_endpoint_data
xenial_queens = OPENSTACK_RELEASES_PAIRS.index('xenial_queens')
if openstack_release and openstack_release >= xenial_queens:
validation_function = self.validate_v3_svc_catalog_endpoint_data
expected = self.convert_svc_catalog_endpoint_data_to_v3(expected)
return validation_function(expected, actual)
def validate_v2_svc_catalog_endpoint_data(self, expected, actual):
"""Validate service catalog endpoint data.
Validate a list of actual service catalog endpoints vs a list of
@ -310,6 +424,7 @@ class OpenStackAmuletUtils(AmuletUtils):
self.log.debug('Checking if tenant exists ({})...'.format(tenant))
return tenant in [t.name for t in keystone.tenants.list()]
@retry_on_exception(num_retries=5, base_delay=1)
def keystone_wait_for_propagation(self, sentry_relation_pairs,
api_version):
"""Iterate over list of sentry and relation tuples and verify that
@ -328,7 +443,7 @@ class OpenStackAmuletUtils(AmuletUtils):
if rel.get('api_version') != str(api_version):
raise Exception("api_version not propagated through relation"
" data yet ('{}' != '{}')."
"".format(rel['api_version'], api_version))
"".format(rel.get('api_version'), api_version))
def keystone_configure_api_version(self, sentry_relation_pairs, deployment,
api_version):
@ -350,16 +465,13 @@ class OpenStackAmuletUtils(AmuletUtils):
deployment._auto_wait_for_status()
self.keystone_wait_for_propagation(sentry_relation_pairs, api_version)
def authenticate_cinder_admin(self, keystone_sentry, username,
password, tenant, api_version=2):
def authenticate_cinder_admin(self, keystone, api_version=2):
"""Authenticates admin user with cinder."""
# NOTE(beisner): cinder python client doesn't accept tokens.
keystone_ip = keystone_sentry.info['public-address']
ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8'))
self.log.debug('Authenticating cinder admin...')
_clients = {
1: cinder_client.Client,
2: cinder_clientv2.Client}
return _clients[api_version](username, password, tenant, ept)
return _clients[api_version](session=keystone.session)
def authenticate_keystone(self, keystone_ip, username, password,
api_version=False, admin_port=False,
@ -367,13 +479,36 @@ class OpenStackAmuletUtils(AmuletUtils):
project_domain_name=None, project_name=None):
"""Authenticate with Keystone"""
self.log.debug('Authenticating with keystone...')
port = 5000
if admin_port:
port = 35357
base_ep = "http://{}:{}".format(keystone_ip.strip().decode('utf-8'),
port)
if not api_version or api_version == 2:
ep = base_ep + "/v2.0"
if not api_version:
api_version = 2
sess, auth = self.get_keystone_session(
keystone_ip=keystone_ip,
username=username,
password=password,
api_version=api_version,
admin_port=admin_port,
user_domain_name=user_domain_name,
domain_name=domain_name,
project_domain_name=project_domain_name,
project_name=project_name
)
if api_version == 2:
client = keystone_client.Client(session=sess)
else:
client = keystone_client_v3.Client(session=sess)
# This populates the client.service_catalog
client.auth_ref = auth.get_access(sess)
return client
def get_keystone_session(self, keystone_ip, username, password,
api_version=False, admin_port=False,
user_domain_name=None, domain_name=None,
project_domain_name=None, project_name=None):
"""Return a keystone session object"""
ep = self.get_keystone_endpoint(keystone_ip,
api_version=api_version,
admin_port=admin_port)
if api_version == 2:
auth = v2.Password(
username=username,
password=password,
@ -381,12 +516,7 @@ class OpenStackAmuletUtils(AmuletUtils):
auth_url=ep
)
sess = keystone_session.Session(auth=auth)
client = keystone_client.Client(session=sess)
# This populates the client.service_catalog
client.auth_ref = auth.get_access(sess)
return client
else:
ep = base_ep + "/v3"
auth = v3.Password(
user_domain_name=user_domain_name,
username=username,
@ -397,10 +527,57 @@ class OpenStackAmuletUtils(AmuletUtils):
auth_url=ep
)
sess = keystone_session.Session(auth=auth)
client = keystone_client_v3.Client(session=sess)
# This populates the client.service_catalog
client.auth_ref = auth.get_access(sess)
return client
return (sess, auth)
def get_keystone_endpoint(self, keystone_ip, api_version=None,
admin_port=False):
"""Return keystone endpoint"""
port = 5000
if admin_port:
port = 35357
base_ep = "http://{}:{}".format(keystone_ip.strip().decode('utf-8'),
port)
if api_version == 2:
ep = base_ep + "/v2.0"
else:
ep = base_ep + "/v3"
return ep
def get_default_keystone_session(self, keystone_sentry,
openstack_release=None, api_version=2):
"""Return a keystone session object and client object assuming standard
default settings
Example call in amulet tests:
self.keystone_session, self.keystone = u.get_default_keystone_session(
self.keystone_sentry,
openstack_release=self._get_openstack_release())
The session can then be used to auth other clients:
neutronclient.Client(session=session)
aodh_client.Client(session=session)
eyc
"""
self.log.debug('Authenticating keystone admin...')
# 11 => xenial_queens
if api_version == 3 or (openstack_release and openstack_release >= 11):
client_class = keystone_client_v3.Client
api_version = 3
else:
client_class = keystone_client.Client
keystone_ip = keystone_sentry.info['public-address']
session, auth = self.get_keystone_session(
keystone_ip,
api_version=api_version,
username='admin',
password='openstack',
project_name='admin',
user_domain_name='admin_domain',
project_domain_name='admin_domain')
client = client_class(session=session)
# This populates the client.service_catalog
client.auth_ref = auth.get_access(session)
return session, client
def authenticate_keystone_admin(self, keystone_sentry, user, password,
tenant=None, api_version=None,

View File

@ -27,6 +27,7 @@ import glob
import os
import json
import yaml
import re
import subprocess
import sys
import errno
@ -67,7 +68,7 @@ def cached(func):
@wraps(func)
def wrapper(*args, **kwargs):
global cache
key = str((func, args, kwargs))
key = json.dumps((func, args, kwargs), sort_keys=True, default=str)
try:
return cache[key]
except KeyError:
@ -289,7 +290,7 @@ class Config(dict):
self.implicit_save = True
self._prev_dict = None
self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
if os.path.exists(self.path):
if os.path.exists(self.path) and os.stat(self.path).st_size:
self.load_previous()
atexit(self._implicit_save)
@ -309,7 +310,11 @@ class Config(dict):
"""
self.path = path or self.path
with open(self.path) as f:
self._prev_dict = json.load(f)
try:
self._prev_dict = json.load(f)
except ValueError as e:
log('Unable to parse previous config data - {}'.format(str(e)),
level=ERROR)
for k, v in copy.deepcopy(self._prev_dict).items():
if k not in self:
self[k] = v
@ -353,22 +358,40 @@ class Config(dict):
self.save()
@cached
_cache_config = None
def config(scope=None):
"""Juju charm configuration"""
config_cmd_line = ['config-get']
if scope is not None:
config_cmd_line.append(scope)
else:
config_cmd_line.append('--all')
config_cmd_line.append('--format=json')
"""
Get the juju charm configuration (scope==None) or individual key,
(scope=str). The returned value is a Python data structure loaded as
JSON from the Juju config command.
:param scope: If set, return the value for the specified key.
:type scope: Optional[str]
:returns: Either the whole config as a Config, or a key from it.
:rtype: Any
"""
global _cache_config
config_cmd_line = ['config-get', '--all', '--format=json']
try:
config_data = json.loads(
subprocess.check_output(config_cmd_line).decode('UTF-8'))
# JSON Decode Exception for Python3.5+
exc_json = json.decoder.JSONDecodeError
except AttributeError:
# JSON Decode Exception for Python2.7 through Python3.4
exc_json = ValueError
try:
if _cache_config is None:
config_data = json.loads(
subprocess.check_output(config_cmd_line).decode('UTF-8'))
_cache_config = Config(config_data)
if scope is not None:
return config_data
return Config(config_data)
except ValueError:
return _cache_config.get(scope)
return _cache_config
except (exc_json, UnicodeDecodeError) as e:
log('Unable to parse output from config-get: config_cmd_line="{}" '
'message="{}"'
.format(config_cmd_line, str(e)), level=ERROR)
return None
@ -949,6 +972,13 @@ def application_version_set(version):
log("Application Version: {}".format(version))
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
def goal_state():
"""Juju goal state values"""
cmd = ['goal-state', '--format=json']
return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
def is_leader():
"""Does the current unit hold the juju leadership
@ -1043,7 +1073,6 @@ def juju_version():
universal_newlines=True).strip()
@cached
def has_juju_version(minimum_version):
"""Return True if the Juju version is at least the provided version"""
return LooseVersion(juju_version()) >= LooseVersion(minimum_version)
@ -1103,6 +1132,8 @@ def _run_atexit():
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
def network_get_primary_address(binding):
'''
Deprecated since Juju 2.3; use network_get()
Retrieve the primary network address for a named binding
:param binding: string. The name of a relation of extra-binding
@ -1123,7 +1154,6 @@ def network_get_primary_address(binding):
return response
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
def network_get(endpoint, relation_id=None):
"""
Retrieve the network details for a relation endpoint
@ -1131,24 +1161,20 @@ def network_get(endpoint, relation_id=None):
:param endpoint: string. The name of a relation endpoint
:param relation_id: int. The ID of the relation for the current context.
:return: dict. The loaded YAML output of the network-get query.
:raise: NotImplementedError if run on Juju < 2.1
:raise: NotImplementedError if request not supported by the Juju version.
"""
if not has_juju_version('2.2'):
raise NotImplementedError(juju_version()) # earlier versions require --primary-address
if relation_id and not has_juju_version('2.3'):
raise NotImplementedError # 2.3 added the -r option
cmd = ['network-get', endpoint, '--format', 'yaml']
if relation_id:
cmd.append('-r')
cmd.append(relation_id)
try:
response = subprocess.check_output(
cmd,
stderr=subprocess.STDOUT).decode('UTF-8').strip()
except CalledProcessError as e:
# Early versions of Juju 2.0.x required the --primary-address argument.
# We catch that condition here and raise NotImplementedError since
# the requested semantics are not available - the caller can then
# use the network_get_primary_address() method instead.
if '--primary-address is currently required' in e.output.decode('UTF-8'):
raise NotImplementedError
raise
response = subprocess.check_output(
cmd,
stderr=subprocess.STDOUT).decode('UTF-8').strip()
return yaml.safe_load(response)
@ -1204,9 +1230,23 @@ def iter_units_for_relation_name(relation_name):
def ingress_address(rid=None, unit=None):
"""
Retrieve the ingress-address from a relation when available. Otherwise,
return the private-address. This function is to be used on the consuming
side of the relation.
Retrieve the ingress-address from a relation when available.
Otherwise, return the private-address.
When used on the consuming side of the relation (unit is a remote
unit), the ingress-address is the IP address that this unit needs
to use to reach the provided service on the remote unit.
When used on the providing side of the relation (unit == local_unit()),
the ingress-address is the IP address that is advertised to remote
units on this relation. Remote units need to use this address to
reach the local provided service on this unit.
Note that charms may document some other method to use in
preference to the ingress_address(), such as an address provided
on a different relation attribute or a service discovery mechanism.
This allows charms to redirect inbound connections to their peers
or different applications such as load balancers.
Usage:
addresses = [ingress_address(rid=u.rid, unit=u.unit)
@ -1220,3 +1260,40 @@ def ingress_address(rid=None, unit=None):
settings = relation_get(rid=rid, unit=unit)
return (settings.get('ingress-address') or
settings.get('private-address'))
def egress_subnets(rid=None, unit=None):
"""
Retrieve the egress-subnets from a relation.
This function is to be used on the providing side of the
relation, and provides the ranges of addresses that client
connections may come from. The result is uninteresting on
the consuming side of a relation (unit == local_unit()).
Returns a stable list of subnets in CIDR format.
eg. ['192.168.1.0/24', '2001::F00F/128']
If egress-subnets is not available, falls back to using the published
ingress-address, or finally private-address.
:param rid: string relation id
:param unit: string unit name
:side effect: calls relation_get
:return: list of subnets in CIDR format. eg. ['192.168.1.0/24', '2001::F00F/128']
"""
def _to_range(addr):
if re.search(r'^(?:\d{1,3}\.){3}\d{1,3}$', addr) is not None:
addr += '/32'
elif ':' in addr and '/' not in addr: # IPv6
addr += '/128'
return addr
settings = relation_get(rid=rid, unit=unit)
if 'egress-subnets' in settings:
return [n.strip() for n in settings['egress-subnets'].split(',') if n.strip()]
if 'ingress-address' in settings:
return [_to_range(settings['ingress-address'])]
if 'private-address' in settings:
return [_to_range(settings['private-address'])]
return [] # Should never happen

View File

@ -972,6 +972,20 @@ def is_container():
def add_to_updatedb_prunepath(path, updatedb_path=UPDATEDB_PATH):
"""Adds the specified path to the mlocate's udpatedb.conf PRUNEPATH list.
This method has no effect if the path specified by updatedb_path does not
exist or is not a file.
@param path: string the path to add to the updatedb.conf PRUNEPATHS value
@param updatedb_path: the path the updatedb.conf file
"""
if not os.path.exists(updatedb_path) or os.path.isdir(updatedb_path):
# If the updatedb.conf file doesn't exist then don't attempt to update
# the file as the package providing mlocate may not be installed on
# the local system
return
with open(updatedb_path, 'r+') as f_id:
updatedb_text = f_id.read()
output = updatedb(updatedb_text, path)
@ -993,7 +1007,7 @@ def updatedb(updatedb_text, new_path):
return output
def modulo_distribution(modulo=3, wait=30):
def modulo_distribution(modulo=3, wait=30, non_zero_wait=False):
""" Modulo distribution
This helper uses the unit number, a modulo value and a constant wait time
@ -1015,7 +1029,14 @@ def modulo_distribution(modulo=3, wait=30):
@param modulo: int The modulo number creates the group distribution
@param wait: int The constant time wait value
@param non_zero_wait: boolean Override unit % modulo == 0,
return modulo * wait. Used to avoid collisions with
leader nodes which are often given priority.
@return: int Calculated time to wait for unit operation
"""
unit_number = int(local_unit().split('/')[1])
return (unit_number % modulo) * wait
calculated_wait_time = (unit_number % modulo) * wait
if non_zero_wait and calculated_wait_time == 0:
return modulo * wait
else:
return calculated_wait_time

View File

@ -307,23 +307,34 @@ class PortManagerCallback(ManagerCallback):
"""
def __call__(self, manager, service_name, event_name):
service = manager.get_service(service_name)
new_ports = service.get('ports', [])
# turn this generator into a list,
# as we'll be going over it multiple times
new_ports = list(service.get('ports', []))
port_file = os.path.join(hookenv.charm_dir(), '.{}.ports'.format(service_name))
if os.path.exists(port_file):
with open(port_file) as fp:
old_ports = fp.read().split(',')
for old_port in old_ports:
if bool(old_port):
old_port = int(old_port)
if old_port not in new_ports:
hookenv.close_port(old_port)
if bool(old_port) and not self.ports_contains(old_port, new_ports):
hookenv.close_port(old_port)
with open(port_file, 'w') as fp:
fp.write(','.join(str(port) for port in new_ports))
for port in new_ports:
# A port is either a number or 'ICMP'
protocol = 'TCP'
if str(port).upper() == 'ICMP':
protocol = 'ICMP'
if event_name == 'start':
hookenv.open_port(port)
hookenv.open_port(port, protocol)
elif event_name == 'stop':
hookenv.close_port(port)
hookenv.close_port(port, protocol)
def ports_contains(self, port, ports):
if not bool(port):
return False
if str(port).upper() != 'ICMP':
port = int(port)
return port in ports
def service_stop(service_name):

View File

@ -31,18 +31,22 @@ __author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
def create(sysctl_dict, sysctl_file):
"""Creates a sysctl.conf file from a YAML associative array
:param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }"
:param sysctl_dict: a dict or YAML-formatted string of sysctl
options eg "{ 'kernel.max_pid': 1337 }"
:type sysctl_dict: str
:param sysctl_file: path to the sysctl file to be saved
:type sysctl_file: str or unicode
:returns: None
"""
try:
sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
except yaml.YAMLError:
log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
level=ERROR)
return
if type(sysctl_dict) is not dict:
try:
sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
except yaml.YAMLError:
log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
level=ERROR)
return
else:
sysctl_dict_parsed = sysctl_dict
with open(sysctl_file, "w") as fd:
for key, value in sysctl_dict_parsed.items():

View File

@ -166,6 +166,10 @@ class Storage(object):
To support dicts, lists, integer, floats, and booleans values
are automatically json encoded/decoded.
Note: to facilitate unit testing, ':memory:' can be passed as the
path parameter which causes sqlite3 to only build the db in memory.
This should only be used for testing purposes.
"""
def __init__(self, path=None):
self.db_path = path
@ -175,8 +179,9 @@ class Storage(object):
else:
self.db_path = os.path.join(
os.environ.get('CHARM_DIR', ''), '.unit-state.db')
with open(self.db_path, 'a') as f:
os.fchmod(f.fileno(), 0o600)
if self.db_path != ':memory:':
with open(self.db_path, 'a') as f:
os.fchmod(f.fileno(), 0o600)
self.conn = sqlite3.connect('%s' % self.db_path)
self.cursor = self.conn.cursor()
self.revision = None