openstack
/
charm-ceph-osd
Code Issues Proposed changes

Sync charm-helpers 

Change-Id: I12b0ba1b814cbba2dbb3474de5c0b180df03628a
This commit is contained in:
Ryan Beisner 2017-08-24 16:47:27 -05:00
parent ea59c761a4
commit 4189b4f8ee
12 changed files with 469 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hooks/charmhelpers/contrib/charmsupport/nrpe.py
View File

@ -125,7 +125,7 @@ class CheckException(Exception):
class Check(object):
shortname_re = '[A-Za-z0-9-_]+$'
shortname_re = '[A-Za-z0-9-_.]+$'
service_template = ("""
#---------------------------------------------------
# This file is Juju managed

5
hooks/charmhelpers/contrib/hardening/apache/checks/config.py
View File

@ -46,8 +46,9 @@ def get_audits():
context = ApacheConfContext()
settings = utils.get_settings('apache')
audits = [
FilePermissionAudit(paths='/etc/apache2/apache2.conf', user='root',
group='root', mode=0o0640),
FilePermissionAudit(paths=os.path.join(
settings['common']['apache_dir'], 'apache2.conf'),
user='root', group='root', mode=0o0640),
TemplatedFile(os.path.join(settings['common']['apache_dir'],
'mods-available/alias.conf'),

96
hooks/charmhelpers/contrib/openstack/context.py
View File

@ -41,9 +41,9 @@ from charmhelpers.core.hookenv import (
charm_name,
DEBUG,
INFO,
WARNING,
ERROR,
status_set,
network_get_primary_address
)
from charmhelpers.core.sysctl import create as sysctl_create
@ -80,6 +80,9 @@ from charmhelpers.contrib.openstack.neutron import (
from charmhelpers.contrib.openstack.ip import (
resolve_address,
INTERNAL,
ADMIN,
PUBLIC,
ADDRESS_MAP,
)
from charmhelpers.contrib.network.ip import (
get_address_in_network,
@ -87,7 +90,6 @@ from charmhelpers.contrib.network.ip import (
get_ipv6_addr,
get_netmask_for_address,
format_ipv6_addr,
is_address_in_network,
is_bridge_member,
is_ipv6_disabled,
)
@ -97,6 +99,7 @@ from charmhelpers.contrib.openstack.utils import (
git_determine_usr_bin,
git_determine_python_path,
enable_memcache,
snap_install_requested,
)
from charmhelpers.core.unitdata import kv
@ -244,6 +247,11 @@ class SharedDBContext(OSContextGenerator):
'database_password': rdata.get(password_setting),
'database_type': 'mysql'
}
# Note(coreycb): We can drop mysql+pymysql if we want when the
# following review lands, though it seems mysql+pymysql would
# be preferred. https://review.openstack.org/#/c/462190/
if snap_install_requested():
ctxt['database_type'] = 'mysql+pymysql'
if self.context_complete(ctxt):
db_ssl(rdata, ctxt, self.ssl_dir)
return ctxt
@ -510,6 +518,10 @@ class CephContext(OSContextGenerator):
ctxt['auth'] = relation_get('auth', rid=rid, unit=unit)
if not ctxt.get('key'):
ctxt['key'] = relation_get('key', rid=rid, unit=unit)
if not ctxt.get('rbd_features'):
default_features = relation_get('rbd-features', rid=rid, unit=unit)
if default_features is not None:
ctxt['rbd_features'] = default_features
ceph_addrs = relation_get('ceph-public-address', rid=rid,
unit=unit)
@ -610,7 +622,6 @@ class HAProxyContext(OSContextGenerator):
ctxt['haproxy_connect_timeout'] = config('haproxy-connect-timeout')
if config('prefer-ipv6'):
ctxt['ipv6'] = True
ctxt['local_host'] = 'ip6-localhost'
ctxt['haproxy_host'] = '::'
else:
@ -726,11 +737,17 @@ class ApacheSSLContext(OSContextGenerator):
return sorted(list(set(cns)))
def get_network_addresses(self):
"""For each network configured, return corresponding address and vip
(if available).
"""For each network configured, return corresponding address and
hostnamr or vip (if available).
Returns a list of tuples of the form:
[(address_in_net_a, hostname_in_net_a),
(address_in_net_b, hostname_in_net_b),
...]
or, if no hostnames(s) available:
[(address_in_net_a, vip_in_net_a),
(address_in_net_b, vip_in_net_b),
...]
@ -742,32 +759,27 @@ class ApacheSSLContext(OSContextGenerator):
...]
"""
addresses = []
if config('vip'):
vips = config('vip').split()
else:
vips = []
for net_type in ['os-internal-network', 'os-admin-network',
'os-public-network']:
addr = get_address_in_network(config(net_type),
unit_get('private-address'))
if len(vips) > 1 and is_clustered():
if not config(net_type):
log("Multiple networks configured but net_type "
"is None (%s)." % net_type, level=WARNING)
continue
for vip in vips:
if is_address_in_network(config(net_type), vip):
addresses.append((addr, vip))
break
elif is_clustered() and config('vip'):
addresses.append((addr, config('vip')))
for net_type in [INTERNAL, ADMIN, PUBLIC]:
net_config = config(ADDRESS_MAP[net_type]['config'])
# NOTE(jamespage): Fallback must always be private address
# as this is used to bind services on the
# local unit.
fallback = unit_get("private-address")
if net_config:
addr = get_address_in_network(net_config,
fallback)
else:
addresses.append((addr, addr))
try:
addr = network_get_primary_address(
ADDRESS_MAP[net_type]['binding']
)
except NotImplementedError:
addr = fallback
return sorted(addresses)
endpoint = resolve_address(net_type)
addresses.append((addr, endpoint))
return sorted(set(addresses))
def __call__(self):
if isinstance(self.external_ports, six.string_types):
@ -794,7 +806,7 @@ class ApacheSSLContext(OSContextGenerator):
self.configure_cert(cn)
addresses = self.get_network_addresses()
for address, endpoint in sorted(set(addresses)):
for address, endpoint in addresses:
for api_port in self.external_ports:
ext_port = determine_apache_port(api_port,
singlenode_mode=True)
@ -1397,14 +1409,38 @@ class NeutronAPIContext(OSContextGenerator):
'rel_key': 'dns-domain',
'default': None,
},
'polling_interval': {
'rel_key': 'polling-interval',
'default': 2,
},
'rpc_response_timeout': {
'rel_key': 'rpc-response-timeout',
'default': 60,
},
'report_interval': {
'rel_key': 'report-interval',
'default': 30,
},
'enable_qos': {
'rel_key': 'enable-qos',
'default': False,
},
}
ctxt = self.get_neutron_options({})
for rid in relation_ids('neutron-plugin-api'):
for unit in related_units(rid):
rdata = relation_get(rid=rid, unit=unit)
# The l2-population key is used by the context as a way of
# checking if the api service on the other end is sending data
# in a recent format.
if 'l2-population' in rdata:
ctxt.update(self.get_neutron_options(rdata))
if ctxt['enable_qos']:
ctxt['extension_drivers'] = 'qos'
else:
ctxt['extension_drivers'] = ''
return ctxt
def get_neutron_options(self, rdata):

135
hooks/charmhelpers/contrib/openstack/utils.py
View File

@ -51,6 +51,7 @@ from charmhelpers.core.hookenv import (
status_set,
hook_name,
application_version_set,
cached,
)
from charmhelpers.core.strutils import BasicStringComparator
@ -90,6 +91,13 @@ from charmhelpers.fetch import (
GPGKeyError,
get_upstream_version
)
from charmhelpers.fetch.snap import (
snap_install,
snap_refresh,
SNAP_CHANNELS,
)
from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
from charmhelpers.contrib.openstack.exceptions import OSContextError
@ -178,7 +186,7 @@ SWIFT_CODENAMES = OrderedDict([
('ocata',
['2.11.0', '2.12.0', '2.13.0']),
('pike',
['2.13.0']),
['2.13.0', '2.15.0']),
])
# >= Liberty version->codename mapping
@ -327,8 +335,10 @@ def get_os_codename_install_source(src):
return ca_rel
# Best guess match based on deb string provided
if src.startswith('deb') or src.startswith('ppa'):
for k, v in six.iteritems(OPENSTACK_CODENAMES):
if (src.startswith('deb') or
src.startswith('ppa') or
src.startswith('snap')):
for v in OPENSTACK_CODENAMES.values():
if v in src:
return v
@ -397,6 +407,19 @@ def get_swift_codename(version):
def get_os_codename_package(package, fatal=True):
'''Derive OpenStack release codename from an installed package.'''
if snap_install_requested():
cmd = ['snap', 'list', package]
try:
out = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
return None
lines = out.split('\n')
for line in lines:
if package in line:
# Second item in list is Version
return line.split()[1]
import apt_pkg as apt
cache = apt_cache()
@ -613,6 +636,9 @@ def openstack_upgrade_available(package):
import apt_pkg as apt
src = config('openstack-origin')
cur_vers = get_os_version_package(package)
if not cur_vers:
# The package has not been installed yet do not attempt upgrade
return False
if "swift" in package:
codename = get_os_codename_install_source(src)
avail_vers = get_os_version_codename_swift(codename)
@ -1863,6 +1889,30 @@ def pausable_restart_on_change(restart_map, stopstart=False,
return wrap
def ordered(orderme):
"""Converts the provided dictionary into a collections.OrderedDict.
The items in the returned OrderedDict will be inserted based on the
natural sort order of the keys. Nested dictionaries will also be sorted
in order to ensure fully predictable ordering.
:param orderme: the dict to order
:return: collections.OrderedDict
:raises: ValueError: if `orderme` isn't a dict instance.
"""
if not isinstance(orderme, dict):
raise ValueError('argument must be a dict type')
result = OrderedDict()
for k, v in sorted(six.iteritems(orderme), key=lambda x: x[0]):
if isinstance(v, dict):
result[k] = ordered(v)
else:
result[k] = v
return result
def config_flags_parser(config_flags):
"""Parses config flags string into dict.
@ -1874,15 +1924,13 @@ def config_flags_parser(config_flags):
example, a string in the format of 'key1=value1, key2=value2' will
return a dict of:
{'key1': 'value1',
'key2': 'value2'}.
{'key1': 'value1', 'key2': 'value2'}.
2. A string in the above format, but supporting a comma-delimited list
of values for the same key. For example, a string in the format of
'key1=value1, key2=value3,value4,value5' will return a dict of:
{'key1', 'value1',
'key2', 'value2,value3,value4'}
{'key1': 'value1', 'key2': 'value2,value3,value4'}
3. A string containing a colon character (:) prior to an equal
character (=) will be treated as yaml and parsed as such. This can be
@ -1902,7 +1950,7 @@ def config_flags_parser(config_flags):
equals = config_flags.find('=')
if colon > 0:
if colon < equals or equals < 0:
return yaml.safe_load(config_flags)
return ordered(yaml.safe_load(config_flags))
if config_flags.find('==') >= 0:
juju_log("config_flags is not in expected format (key=value)",
@ -1915,7 +1963,7 @@ def config_flags_parser(config_flags):
# split on '='.
split = config_flags.strip(' =').split('=')
limit = len(split)
flags = {}
flags = OrderedDict()
for i in range(0, limit - 1):
current = split[i]
next = split[i + 1]
@ -1994,3 +2042,72 @@ def update_json_file(filename, items):
policy.update(items)
with open(filename, "w") as fd:
fd.write(json.dumps(policy, indent=4))
@cached
def snap_install_requested():
""" Determine if installing from snaps
If openstack-origin is of the form snap:channel-series-release
and channel is in SNAPS_CHANNELS return True.
"""
origin = config('openstack-origin') or ""
if not origin.startswith('snap:'):
return False
_src = origin[5:]
channel, series, release = _src.split('-')
if channel.lower() in SNAP_CHANNELS:
return True
return False
def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
"""Generate a dictionary of snap install information from origin
@param snaps: List of snaps
@param src: String of openstack-origin or source of the form
snap:channel-series-track
@param mode: String classic, devmode or jailmode
@returns: Dictionary of snaps with channels and modes
"""
if not src.startswith('snap:'):
juju_log("Snap source is not a snap origin", 'WARN')
return {}
_src = src[5:]
_channel, _series, _release = _src.split('-')
channel = '--channel={}/{}'.format(_release, _channel)
return {snap: {'channel': channel, 'mode': mode}
for snap in snaps}
def install_os_snaps(snaps, refresh=False):
"""Install OpenStack snaps from channel and with mode
@param snaps: Dictionary of snaps with channels and modes of the form:
{'snap_name': {'channel': 'snap_channel',
'mode': 'snap_mode'}}
Where channel a snapstore channel and mode is --classic, --devmode or
--jailmode.
@param post_snap_install: Callback function to run after snaps have been
installed
"""
def _ensure_flag(flag):
if flag.startswith('--'):
return flag
return '--{}'.format(flag)
if refresh:
for snap in snaps.keys():
snap_refresh(snap,
_ensure_flag(snaps[snap]['channel']),
_ensure_flag(snaps[snap]['mode']))
else:
for snap in snaps.keys():
snap_install(snap,
_ensure_flag(snaps[snap]['channel']),
_ensure_flag(snaps[snap]['mode']))

74
hooks/charmhelpers/contrib/storage/linux/bcache.py Normal file
View File

@ -0,0 +1,74 @@
# Copyright 2017 Canonical Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import json
from charmhelpers.core.hookenv import log
stats_intervals = ['stats_day', 'stats_five_minute',
'stats_hour', 'stats_total']
SYSFS = '/sys'
class Bcache(object):
"""Bcache behaviour
"""
def __init__(self, cachepath):
self.cachepath = cachepath
@classmethod
def fromdevice(cls, devname):
return cls('{}/block/{}/bcache'.format(SYSFS, devname))
def __str__(self):
return self.cachepath
def get_stats(self, interval):
"""Get cache stats
"""
intervaldir = 'stats_{}'.format(interval)
path = "{}/{}".format(self.cachepath, intervaldir)
out = dict()
for elem in os.listdir(path):
out[elem] = open('{}/{}'.format(path, elem)).read().strip()
return out
def get_bcache_fs():
"""Return all cache sets
"""
cachesetroot = "{}/fs/bcache".format(SYSFS)
try:
dirs = os.listdir(cachesetroot)
except OSError:
log("No bcache fs found")
return []
cacheset = set([Bcache('{}/{}'.format(cachesetroot, d)) for d in dirs if not d.startswith('register')])
return cacheset
def get_stats_action(cachespec, interval):
"""Action for getting bcache statistics for a given cachespec.
Cachespec can either be a device name, eg. 'sdb', which will retrieve
cache stats for the given device, or 'global', which will retrieve stats
for all cachesets
"""
if cachespec == 'global':
caches = get_bcache_fs()
else:
caches = [Bcache.fromdevice(cachespec)]
res = dict((c.cachepath, c.get_stats(interval)) for c in caches)
return json.dumps(res, indent=4, separators=(',', ': '))

2
hooks/charmhelpers/contrib/storage/linux/ceph.py
View File

@ -1372,7 +1372,7 @@ class CephConfContext(object):
return {}
conf = config_flags_parser(conf)
if type(conf) != dict:
if not isinstance(conf, dict):
log("Provided config-flags is not a dictionary - ignoring",
level=WARNING)
return {}

40
hooks/charmhelpers/core/hookenv.py
View File

@ -43,6 +43,7 @@ ERROR = "ERROR"
WARNING = "WARNING"
INFO = "INFO"
DEBUG = "DEBUG"
TRACE = "TRACE"
MARKER = object()
cache = {}
@ -202,6 +203,27 @@ def service_name():
return local_unit().split('/')[0]
def principal_unit():
"""Returns the principal unit of this unit, otherwise None"""
# Juju 2.2 and above provides JUJU_PRINCIPAL_UNIT
principal_unit = os.environ.get('JUJU_PRINCIPAL_UNIT', None)
# If it's empty, then this unit is the principal
if principal_unit == '':
return os.environ['JUJU_UNIT_NAME']
elif principal_unit is not None:
return principal_unit
# For Juju 2.1 and below, let's try work out the principle unit by
# the various charms' metadata.yaml.
for reltype in relation_types():
for rid in relation_ids(reltype):
for unit in related_units(rid):
md = _metadata_unit(unit)
subordinate = md.pop('subordinate', None)
if not subordinate:
return unit
return None
@cached
def remote_service_name(relid=None):
"""The remote service name for a given relation-id (or the current relation)"""
@ -478,6 +500,21 @@ def metadata():
return yaml.safe_load(md)
def _metadata_unit(unit):
"""Given the name of a unit (e.g. apache2/0), get the unit charm's
metadata.yaml. Very similar to metadata() but allows us to inspect
other units. Unit needs to be co-located, such as a subordinate or
principal/primary.
:returns: metadata.yaml as a python object.
"""
basedir = os.sep.join(charm_dir().split(os.sep)[:-2])
unitdir = 'unit-{}'.format(unit.replace(os.sep, '-'))
with open(os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')) as md:
return yaml.safe_load(md)
@cached
def relation_types():
"""Get a list of relation types supported by this charm"""
@ -753,6 +790,9 @@ class Hooks(object):
def charm_dir():
"""Return the root directory of the current charm"""
d = os.environ.get('JUJU_CHARM_DIR')
if d is not None:
return d
return os.environ.get('CHARM_DIR')

36
hooks/charmhelpers/core/host.py
View File

@ -34,7 +34,7 @@ import six
from contextlib import contextmanager
from collections import OrderedDict
from .hookenv import log
from .hookenv import log, DEBUG
from .fstab import Fstab
from charmhelpers.osplatform import get_platform
@ -487,13 +487,37 @@ def mkdir(path, owner='root', group='root', perms=0o555, force=False):
def write_file(path, content, owner='root', group='root', perms=0o444):
"""Create or overwrite a file with the contents of a byte string."""
log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
uid = pwd.getpwnam(owner).pw_uid
gid = grp.getgrnam(group).gr_gid
with open(path, 'wb') as target:
os.fchown(target.fileno(), uid, gid)
os.fchmod(target.fileno(), perms)
target.write(content)
# lets see if we can grab the file and compare the context, to avoid doing
# a write.
existing_content = None
existing_uid, existing_gid = None, None
try:
with open(path, 'rb') as target:
existing_content = target.read()
stat = os.stat(path)
existing_uid, existing_gid = stat.st_uid, stat.st_gid
except:
pass
if content != existing_content:
log("Writing file {} {}:{} {:o}".format(path, owner, group, perms),
level=DEBUG)
with open(path, 'wb') as target:
os.fchown(target.fileno(), uid, gid)
os.fchmod(target.fileno(), perms)
target.write(content)
return
# the contents were the same, but we might still need to change the
# ownership.
if existing_uid != uid:
log("Changing uid on already existing content: {} -> {}"
.format(existing_uid, uid), level=DEBUG)
os.chown(path, uid, -1)
if existing_gid != gid:
log("Changing gid on already existing content: {} -> {}"
.format(existing_gid, gid), level=DEBUG)
os.chown(path, -1, gid)
def fstab_remove(mp):

22
hooks/charmhelpers/fetch/snap.py
View File

@ -18,15 +18,23 @@ If writing reactive charms, use the snap layer:
https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html
"""
import subprocess
from os import environ
import os
from time import sleep
from charmhelpers.core.hookenv import log
__author__ = 'Joseph Borg <joseph.borg@canonical.com>'
SNAP_NO_LOCK = 1 # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved).
# The return code for "couldn't acquire lock" in Snap
# (hopefully this will be improved).
SNAP_NO_LOCK = 1
SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks.
SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
SNAP_CHANNELS = [
'edge',
'beta',
'candidate',
'stable',
]
class CouldNotAcquireLockException(Exception):
@ -47,13 +55,17 @@ def _snap_exec(commands):
while return_code is None or return_code == SNAP_NO_LOCK:
try:
return_code = subprocess.check_call(['snap'] + commands, env=environ)
return_code = subprocess.check_call(['snap'] + commands,
env=os.environ)
except subprocess.CalledProcessError as e:
retry_count += + 1
if retry_count > SNAP_NO_LOCK_RETRY_COUNT:
raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT)
raise CouldNotAcquireLockException(
'Could not aquire lock after {} attempts'
.format(SNAP_NO_LOCK_RETRY_COUNT))
return_code = e.returncode
log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN')
log('Snap failed to acquire lock, trying again in {} seconds.'
.format(SNAP_NO_LOCK_RETRY_DELAY, level='WARN'))
sleep(SNAP_NO_LOCK_RETRY_DELAY)
return return_code

67
hooks/charmhelpers/fetch/ubuntu.py
View File

@ -27,6 +27,7 @@ from charmhelpers.core.host import (
from charmhelpers.core.hookenv import (
log,
DEBUG,
WARNING,
)
from charmhelpers.fetch import SourceConfigError, GPGKeyError
@ -139,7 +140,7 @@ CLOUD_ARCHIVE_POCKETS = {
'xenial-updates/ocata': 'xenial-updates/ocata',
'ocata/proposed': 'xenial-proposed/ocata',
'xenial-ocata/proposed': 'xenial-proposed/ocata',
'xenial-ocata/newton': 'xenial-proposed/ocata',
'xenial-proposed/ocata': 'xenial-proposed/ocata',
# Pike
'pike': 'xenial-updates/pike',
'xenial-pike': 'xenial-updates/pike',
@ -147,7 +148,7 @@ CLOUD_ARCHIVE_POCKETS = {
'xenial-updates/pike': 'xenial-updates/pike',
'pike/proposed': 'xenial-proposed/pike',
'xenial-pike/proposed': 'xenial-proposed/pike',
'xenial-pike/newton': 'xenial-proposed/pike',
'xenial-proposed/pike': 'xenial-proposed/pike',
# Queens
'queens': 'xenial-updates/queens',
'xenial-queens': 'xenial-updates/queens',
@ -155,13 +156,13 @@ CLOUD_ARCHIVE_POCKETS = {
'xenial-updates/queens': 'xenial-updates/queens',
'queens/proposed': 'xenial-proposed/queens',
'xenial-queens/proposed': 'xenial-proposed/queens',
'xenial-queens/newton': 'xenial-proposed/queens',
'xenial-proposed/queens': 'xenial-proposed/queens',
}
APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries.
CMD_RETRY_COUNT = 30 # Retry a failing fatal command X times.
CMD_RETRY_COUNT = 3 # Retry a failing fatal command X times.
def filter_installed_packages(packages):
@ -261,34 +262,47 @@ def apt_unhold(packages, fatal=False):
return apt_mark(packages, 'unhold', fatal=fatal)
def import_key(keyid):
"""Import a key in either ASCII Armor or Radix64 format.
def import_key(key):
"""Import an ASCII Armor key.
`keyid` is either the keyid to fetch from a PGP server, or
the key in ASCII armor foramt.
/!\ A Radix64 format keyid is also supported for backwards
compatibility, but should never be used; the key retrieval
mechanism is insecure and subject to man-in-the-middle attacks
voiding all signature checks using that key.
:param keyid: String of key (or key id).
:param keyid: The key in ASCII armor format,
including BEGIN and END markers.
:raises: GPGKeyError if the key could not be imported
"""
key = keyid.strip()
if (key.startswith('-----BEGIN PGP PUBLIC KEY BLOCK-----') and
key.endswith('-----END PGP PUBLIC KEY BLOCK-----')):
key = key.strip()
if '-' in key or '\n' in key:
# Send everything not obviously a keyid to GPG to import, as
# we trust its validation better than our own. eg. handling
# comments before the key.
log("PGP key found (looks like ASCII Armor format)", level=DEBUG)
log("Importing ASCII Armor PGP key", level=DEBUG)
with NamedTemporaryFile() as keyfile:
with open(keyfile.name, 'w') as fd:
fd.write(key)
fd.write("\n")
cmd = ['apt-key', 'add', keyfile.name]
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
error = "Error importing PGP key '{}'".format(key)
log(error)
raise GPGKeyError(error)
if ('-----BEGIN PGP PUBLIC KEY BLOCK-----' in key and
'-----END PGP PUBLIC KEY BLOCK-----' in key):
log("Importing ASCII Armor PGP key", level=DEBUG)
with NamedTemporaryFile() as keyfile:
with open(keyfile.name, 'w') as fd:
fd.write(key)
fd.write("\n")
cmd = ['apt-key', 'add', keyfile.name]
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
error = "Error importing PGP key '{}'".format(key)
log(error)
raise GPGKeyError(error)
else:
raise GPGKeyError("ASCII armor markers missing from GPG key")
else:
log("PGP key found (looks like Radix64 format)", level=DEBUG)
log("Importing PGP key from keyserver", level=DEBUG)
# We should only send things obviously not a keyid offsite
# via this unsecured protocol, as it may be a secret or part
# of one.
log("PGP key found (looks like Radix64 format)", level=WARNING)
log("INSECURLY importing PGP key from keyserver; "
"full key not provided.", level=WARNING)
cmd = ['apt-key', 'adv', '--keyserver',
'hkp://keyserver.ubuntu.com:80', '--recv-keys', key]
try:
@ -364,6 +378,7 @@ def add_source(source, key=None, fail_invalid=False):
(r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging),
(r"^cloud:(.*)-(.*)$", _add_cloud_distro_check),
(r"^cloud:(.*)$", _add_cloud_pocket),
(r"^snap:.*-(.*)-(.*)$", _add_cloud_distro_check),
])
if source is None:
source = ''

40
tests/charmhelpers/core/hookenv.py
View File

@ -43,6 +43,7 @@ ERROR = "ERROR"
WARNING = "WARNING"
INFO = "INFO"
DEBUG = "DEBUG"
TRACE = "TRACE"
MARKER = object()
cache = {}
@ -202,6 +203,27 @@ def service_name():
return local_unit().split('/')[0]
def principal_unit():
"""Returns the principal unit of this unit, otherwise None"""
# Juju 2.2 and above provides JUJU_PRINCIPAL_UNIT
principal_unit = os.environ.get('JUJU_PRINCIPAL_UNIT', None)
# If it's empty, then this unit is the principal
if principal_unit == '':
return os.environ['JUJU_UNIT_NAME']
elif principal_unit is not None:
return principal_unit
# For Juju 2.1 and below, let's try work out the principle unit by
# the various charms' metadata.yaml.
for reltype in relation_types():
for rid in relation_ids(reltype):
for unit in related_units(rid):
md = _metadata_unit(unit)
subordinate = md.pop('subordinate', None)
if not subordinate:
return unit
return None
@cached
def remote_service_name(relid=None):
"""The remote service name for a given relation-id (or the current relation)"""
@ -478,6 +500,21 @@ def metadata():
return yaml.safe_load(md)
def _metadata_unit(unit):
"""Given the name of a unit (e.g. apache2/0), get the unit charm's
metadata.yaml. Very similar to metadata() but allows us to inspect
other units. Unit needs to be co-located, such as a subordinate or
principal/primary.
:returns: metadata.yaml as a python object.
"""
basedir = os.sep.join(charm_dir().split(os.sep)[:-2])
unitdir = 'unit-{}'.format(unit.replace(os.sep, '-'))
with open(os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')) as md:
return yaml.safe_load(md)
@cached
def relation_types():
"""Get a list of relation types supported by this charm"""
@ -753,6 +790,9 @@ class Hooks(object):
def charm_dir():
"""Return the root directory of the current charm"""
d = os.environ.get('JUJU_CHARM_DIR')
if d is not None:
return d
return os.environ.get('CHARM_DIR')

36
tests/charmhelpers/core/host.py
View File

@ -34,7 +34,7 @@ import six
from contextlib import contextmanager
from collections import OrderedDict
from .hookenv import log
from .hookenv import log, DEBUG
from .fstab import Fstab
from charmhelpers.osplatform import get_platform
@ -487,13 +487,37 @@ def mkdir(path, owner='root', group='root', perms=0o555, force=False):
def write_file(path, content, owner='root', group='root', perms=0o444):
"""Create or overwrite a file with the contents of a byte string."""
log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
uid = pwd.getpwnam(owner).pw_uid
gid = grp.getgrnam(group).gr_gid
with open(path, 'wb') as target:
os.fchown(target.fileno(), uid, gid)
os.fchmod(target.fileno(), perms)
target.write(content)
# lets see if we can grab the file and compare the context, to avoid doing
# a write.
existing_content = None
existing_uid, existing_gid = None, None
try:
with open(path, 'rb') as target:
existing_content = target.read()
stat = os.stat(path)
existing_uid, existing_gid = stat.st_uid, stat.st_gid
except:
pass
if content != existing_content:
log("Writing file {} {}:{} {:o}".format(path, owner, group, perms),
level=DEBUG)
with open(path, 'wb') as target:
os.fchown(target.fileno(), uid, gid)
os.fchmod(target.fileno(), perms)
target.write(content)
return
# the contents were the same, but we might still need to change the
# ownership.
if existing_uid != uid:
log("Changing uid on already existing content: {} -> {}"
.format(existing_uid, uid), level=DEBUG)
os.chown(path, uid, -1)
if existing_gid != gid:
log("Changing gid on already existing content: {} -> {}"
.format(existing_gid, gid), level=DEBUG)
os.chown(path, -1, gid)
def fstab_remove(mp):