* added ch syncer tests as well as hooks
* synced ch * removed old tests * added new tests
This commit is contained in:
parent
323d18cf48
commit
74a83aaf88
5
Makefile
5
Makefile
|
@ -15,7 +15,7 @@ test:
|
|||
# raise_status() messages to stderr:
|
||||
# https://bugs.launchpad.net/amulet/+bug/1320357
|
||||
@juju test -v -p AMULET_HTTP_PROXY --timeout 900 \
|
||||
00-setup 10-bundles-test
|
||||
00-setup 15-basic-trusty-icehouse
|
||||
|
||||
bin/charm_helpers_sync.py:
|
||||
@mkdir -p bin
|
||||
|
@ -23,7 +23,8 @@ bin/charm_helpers_sync.py:
|
|||
> bin/charm_helpers_sync.py
|
||||
|
||||
sync: bin/charm_helpers_sync.py
|
||||
@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers.yaml
|
||||
@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-hooks.yaml
|
||||
@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-tests.yaml
|
||||
|
||||
publish: lint unit_test
|
||||
bzr push lp:charms/hacluster
|
||||
|
|
|
@ -0,0 +1,253 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# Authors:
|
||||
# Adam Gandelman <adamg@ubuntu.com>
|
||||
|
||||
import logging
|
||||
import optparse
|
||||
import os
|
||||
import subprocess
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import yaml
|
||||
from fnmatch import fnmatch
|
||||
|
||||
import six
|
||||
|
||||
CHARM_HELPERS_BRANCH = 'lp:charm-helpers'
|
||||
|
||||
|
||||
def parse_config(conf_file):
|
||||
if not os.path.isfile(conf_file):
|
||||
logging.error('Invalid config file: %s.' % conf_file)
|
||||
return False
|
||||
return yaml.load(open(conf_file).read())
|
||||
|
||||
|
||||
def clone_helpers(work_dir, branch):
|
||||
dest = os.path.join(work_dir, 'charm-helpers')
|
||||
logging.info('Checking out %s to %s.' % (branch, dest))
|
||||
cmd = ['bzr', 'checkout', '--lightweight', branch, dest]
|
||||
subprocess.check_call(cmd)
|
||||
return dest
|
||||
|
||||
|
||||
def _module_path(module):
|
||||
return os.path.join(*module.split('.'))
|
||||
|
||||
|
||||
def _src_path(src, module):
|
||||
return os.path.join(src, 'charmhelpers', _module_path(module))
|
||||
|
||||
|
||||
def _dest_path(dest, module):
|
||||
return os.path.join(dest, _module_path(module))
|
||||
|
||||
|
||||
def _is_pyfile(path):
|
||||
return os.path.isfile(path + '.py')
|
||||
|
||||
|
||||
def ensure_init(path):
|
||||
'''
|
||||
ensure directories leading up to path are importable, omitting
|
||||
parent directory, eg path='/hooks/helpers/foo'/:
|
||||
hooks/
|
||||
hooks/helpers/__init__.py
|
||||
hooks/helpers/foo/__init__.py
|
||||
'''
|
||||
for d, dirs, files in os.walk(os.path.join(*path.split('/')[:2])):
|
||||
_i = os.path.join(d, '__init__.py')
|
||||
if not os.path.exists(_i):
|
||||
logging.info('Adding missing __init__.py: %s' % _i)
|
||||
open(_i, 'wb').close()
|
||||
|
||||
|
||||
def sync_pyfile(src, dest):
|
||||
src = src + '.py'
|
||||
src_dir = os.path.dirname(src)
|
||||
logging.info('Syncing pyfile: %s -> %s.' % (src, dest))
|
||||
if not os.path.exists(dest):
|
||||
os.makedirs(dest)
|
||||
shutil.copy(src, dest)
|
||||
if os.path.isfile(os.path.join(src_dir, '__init__.py')):
|
||||
shutil.copy(os.path.join(src_dir, '__init__.py'),
|
||||
dest)
|
||||
ensure_init(dest)
|
||||
|
||||
|
||||
def get_filter(opts=None):
|
||||
opts = opts or []
|
||||
if 'inc=*' in opts:
|
||||
# do not filter any files, include everything
|
||||
return None
|
||||
|
||||
def _filter(dir, ls):
|
||||
incs = [opt.split('=').pop() for opt in opts if 'inc=' in opt]
|
||||
_filter = []
|
||||
for f in ls:
|
||||
_f = os.path.join(dir, f)
|
||||
|
||||
if not os.path.isdir(_f) and not _f.endswith('.py') and incs:
|
||||
if True not in [fnmatch(_f, inc) for inc in incs]:
|
||||
logging.debug('Not syncing %s, does not match include '
|
||||
'filters (%s)' % (_f, incs))
|
||||
_filter.append(f)
|
||||
else:
|
||||
logging.debug('Including file, which matches include '
|
||||
'filters (%s): %s' % (incs, _f))
|
||||
elif (os.path.isfile(_f) and not _f.endswith('.py')):
|
||||
logging.debug('Not syncing file: %s' % f)
|
||||
_filter.append(f)
|
||||
elif (os.path.isdir(_f) and not
|
||||
os.path.isfile(os.path.join(_f, '__init__.py'))):
|
||||
logging.debug('Not syncing directory: %s' % f)
|
||||
_filter.append(f)
|
||||
return _filter
|
||||
return _filter
|
||||
|
||||
|
||||
def sync_directory(src, dest, opts=None):
|
||||
if os.path.exists(dest):
|
||||
logging.debug('Removing existing directory: %s' % dest)
|
||||
shutil.rmtree(dest)
|
||||
logging.info('Syncing directory: %s -> %s.' % (src, dest))
|
||||
|
||||
shutil.copytree(src, dest, ignore=get_filter(opts))
|
||||
ensure_init(dest)
|
||||
|
||||
|
||||
def sync(src, dest, module, opts=None):
|
||||
|
||||
# Sync charmhelpers/__init__.py for bootstrap code.
|
||||
sync_pyfile(_src_path(src, '__init__'), dest)
|
||||
|
||||
# Sync other __init__.py files in the path leading to module.
|
||||
m = []
|
||||
steps = module.split('.')[:-1]
|
||||
while steps:
|
||||
m.append(steps.pop(0))
|
||||
init = '.'.join(m + ['__init__'])
|
||||
sync_pyfile(_src_path(src, init),
|
||||
os.path.dirname(_dest_path(dest, init)))
|
||||
|
||||
# Sync the module, or maybe a .py file.
|
||||
if os.path.isdir(_src_path(src, module)):
|
||||
sync_directory(_src_path(src, module), _dest_path(dest, module), opts)
|
||||
elif _is_pyfile(_src_path(src, module)):
|
||||
sync_pyfile(_src_path(src, module),
|
||||
os.path.dirname(_dest_path(dest, module)))
|
||||
else:
|
||||
logging.warn('Could not sync: %s. Neither a pyfile or directory, '
|
||||
'does it even exist?' % module)
|
||||
|
||||
|
||||
def parse_sync_options(options):
|
||||
if not options:
|
||||
return []
|
||||
return options.split(',')
|
||||
|
||||
|
||||
def extract_options(inc, global_options=None):
|
||||
global_options = global_options or []
|
||||
if global_options and isinstance(global_options, six.string_types):
|
||||
global_options = [global_options]
|
||||
if '|' not in inc:
|
||||
return (inc, global_options)
|
||||
inc, opts = inc.split('|')
|
||||
return (inc, parse_sync_options(opts) + global_options)
|
||||
|
||||
|
||||
def sync_helpers(include, src, dest, options=None):
|
||||
if not os.path.isdir(dest):
|
||||
os.makedirs(dest)
|
||||
|
||||
global_options = parse_sync_options(options)
|
||||
|
||||
for inc in include:
|
||||
if isinstance(inc, str):
|
||||
inc, opts = extract_options(inc, global_options)
|
||||
sync(src, dest, inc, opts)
|
||||
elif isinstance(inc, dict):
|
||||
# could also do nested dicts here.
|
||||
for k, v in six.iteritems(inc):
|
||||
if isinstance(v, list):
|
||||
for m in v:
|
||||
inc, opts = extract_options(m, global_options)
|
||||
sync(src, dest, '%s.%s' % (k, inc), opts)
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = optparse.OptionParser()
|
||||
parser.add_option('-c', '--config', action='store', dest='config',
|
||||
default=None, help='helper config file')
|
||||
parser.add_option('-D', '--debug', action='store_true', dest='debug',
|
||||
default=False, help='debug')
|
||||
parser.add_option('-b', '--branch', action='store', dest='branch',
|
||||
help='charm-helpers bzr branch (overrides config)')
|
||||
parser.add_option('-d', '--destination', action='store', dest='dest_dir',
|
||||
help='sync destination dir (overrides config)')
|
||||
(opts, args) = parser.parse_args()
|
||||
|
||||
if opts.debug:
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
else:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
if opts.config:
|
||||
logging.info('Loading charm helper config from %s.' % opts.config)
|
||||
config = parse_config(opts.config)
|
||||
if not config:
|
||||
logging.error('Could not parse config from %s.' % opts.config)
|
||||
sys.exit(1)
|
||||
else:
|
||||
config = {}
|
||||
|
||||
if 'branch' not in config:
|
||||
config['branch'] = CHARM_HELPERS_BRANCH
|
||||
if opts.branch:
|
||||
config['branch'] = opts.branch
|
||||
if opts.dest_dir:
|
||||
config['destination'] = opts.dest_dir
|
||||
|
||||
if 'destination' not in config:
|
||||
logging.error('No destination dir. specified as option or config.')
|
||||
sys.exit(1)
|
||||
|
||||
if 'include' not in config:
|
||||
if not args:
|
||||
logging.error('No modules to sync specified as option or config.')
|
||||
sys.exit(1)
|
||||
config['include'] = []
|
||||
[config['include'].append(a) for a in args]
|
||||
|
||||
sync_options = None
|
||||
if 'options' in config:
|
||||
sync_options = config['options']
|
||||
tmpd = tempfile.mkdtemp()
|
||||
try:
|
||||
checkout = clone_helpers(tmpd, config['branch'])
|
||||
sync_helpers(config['include'], checkout, config['destination'],
|
||||
options=sync_options)
|
||||
except Exception as e:
|
||||
logging.error("Could not sync: %s" % e)
|
||||
raise e
|
||||
finally:
|
||||
logging.debug('Cleaning up %s' % tmpd)
|
||||
shutil.rmtree(tmpd)
|
|
@ -0,0 +1,5 @@
|
|||
branch: lp:charm-helpers
|
||||
destination: tests/charmhelpers
|
||||
include:
|
||||
- contrib.amulet
|
||||
- contrib.openstack.amulet
|
|
@ -48,6 +48,9 @@ from charmhelpers.core.hookenv import (
|
|||
from charmhelpers.core.decorators import (
|
||||
retry_on_exception,
|
||||
)
|
||||
from charmhelpers.core.strutils import (
|
||||
bool_from_string,
|
||||
)
|
||||
|
||||
|
||||
class HAIncompleteConfig(Exception):
|
||||
|
@ -164,7 +167,8 @@ def https():
|
|||
.
|
||||
returns: boolean
|
||||
'''
|
||||
if config_get('use-https') == "yes":
|
||||
use_https = config_get('use-https')
|
||||
if use_https and bool_from_string(use_https):
|
||||
return True
|
||||
if config_get('ssl_cert') and config_get('ssl_key'):
|
||||
return True
|
||||
|
|
|
@ -17,13 +17,16 @@
|
|||
import glob
|
||||
import re
|
||||
import subprocess
|
||||
import six
|
||||
import socket
|
||||
|
||||
from functools import partial
|
||||
|
||||
from charmhelpers.core.hookenv import unit_get
|
||||
from charmhelpers.fetch import apt_install
|
||||
from charmhelpers.core.hookenv import (
|
||||
log
|
||||
log,
|
||||
WARNING,
|
||||
)
|
||||
|
||||
try:
|
||||
|
@ -365,3 +368,83 @@ def is_bridge_member(nic):
|
|||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def is_ip(address):
|
||||
"""
|
||||
Returns True if address is a valid IP address.
|
||||
"""
|
||||
try:
|
||||
# Test to see if already an IPv4 address
|
||||
socket.inet_aton(address)
|
||||
return True
|
||||
except socket.error:
|
||||
return False
|
||||
|
||||
|
||||
def ns_query(address):
|
||||
try:
|
||||
import dns.resolver
|
||||
except ImportError:
|
||||
apt_install('python-dnspython')
|
||||
import dns.resolver
|
||||
|
||||
if isinstance(address, dns.name.Name):
|
||||
rtype = 'PTR'
|
||||
elif isinstance(address, six.string_types):
|
||||
rtype = 'A'
|
||||
else:
|
||||
return None
|
||||
|
||||
answers = dns.resolver.query(address, rtype)
|
||||
if answers:
|
||||
return str(answers[0])
|
||||
return None
|
||||
|
||||
|
||||
def get_host_ip(hostname, fallback=None):
|
||||
"""
|
||||
Resolves the IP for a given hostname, or returns
|
||||
the input if it is already an IP.
|
||||
"""
|
||||
if is_ip(hostname):
|
||||
return hostname
|
||||
|
||||
ip_addr = ns_query(hostname)
|
||||
if not ip_addr:
|
||||
try:
|
||||
ip_addr = socket.gethostbyname(hostname)
|
||||
except:
|
||||
log("Failed to resolve hostname '%s'" % (hostname),
|
||||
level=WARNING)
|
||||
return fallback
|
||||
return ip_addr
|
||||
|
||||
|
||||
def get_hostname(address, fqdn=True):
|
||||
"""
|
||||
Resolves hostname for given IP, or returns the input
|
||||
if it is already a hostname.
|
||||
"""
|
||||
if is_ip(address):
|
||||
try:
|
||||
import dns.reversename
|
||||
except ImportError:
|
||||
apt_install("python-dnspython")
|
||||
import dns.reversename
|
||||
|
||||
rev = dns.reversename.from_address(address)
|
||||
result = ns_query(rev)
|
||||
if not result:
|
||||
return None
|
||||
else:
|
||||
result = address
|
||||
|
||||
if fqdn:
|
||||
# strip trailing .
|
||||
if result.endswith('.'):
|
||||
return result[:-1]
|
||||
else:
|
||||
return result
|
||||
else:
|
||||
return result.split('.')[0]
|
||||
|
|
|
@ -23,12 +23,13 @@ from functools import wraps
|
|||
import subprocess
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
|
||||
import six
|
||||
import yaml
|
||||
|
||||
from charmhelpers.contrib.network import ip
|
||||
|
||||
from charmhelpers.core.hookenv import (
|
||||
config,
|
||||
log as juju_log,
|
||||
|
@ -103,6 +104,7 @@ SWIFT_CODENAMES = OrderedDict([
|
|||
('2.1.0', 'juno'),
|
||||
('2.2.0', 'juno'),
|
||||
('2.2.1', 'kilo'),
|
||||
('2.2.2', 'kilo'),
|
||||
])
|
||||
|
||||
DEFAULT_LOOPBACK_SIZE = '5G'
|
||||
|
@ -420,77 +422,10 @@ def clean_storage(block_device):
|
|||
else:
|
||||
zap_disk(block_device)
|
||||
|
||||
|
||||
def is_ip(address):
|
||||
"""
|
||||
Returns True if address is a valid IP address.
|
||||
"""
|
||||
try:
|
||||
# Test to see if already an IPv4 address
|
||||
socket.inet_aton(address)
|
||||
return True
|
||||
except socket.error:
|
||||
return False
|
||||
|
||||
|
||||
def ns_query(address):
|
||||
try:
|
||||
import dns.resolver
|
||||
except ImportError:
|
||||
apt_install('python-dnspython')
|
||||
import dns.resolver
|
||||
|
||||
if isinstance(address, dns.name.Name):
|
||||
rtype = 'PTR'
|
||||
elif isinstance(address, six.string_types):
|
||||
rtype = 'A'
|
||||
else:
|
||||
return None
|
||||
|
||||
answers = dns.resolver.query(address, rtype)
|
||||
if answers:
|
||||
return str(answers[0])
|
||||
return None
|
||||
|
||||
|
||||
def get_host_ip(hostname):
|
||||
"""
|
||||
Resolves the IP for a given hostname, or returns
|
||||
the input if it is already an IP.
|
||||
"""
|
||||
if is_ip(hostname):
|
||||
return hostname
|
||||
|
||||
return ns_query(hostname)
|
||||
|
||||
|
||||
def get_hostname(address, fqdn=True):
|
||||
"""
|
||||
Resolves hostname for given IP, or returns the input
|
||||
if it is already a hostname.
|
||||
"""
|
||||
if is_ip(address):
|
||||
try:
|
||||
import dns.reversename
|
||||
except ImportError:
|
||||
apt_install('python-dnspython')
|
||||
import dns.reversename
|
||||
|
||||
rev = dns.reversename.from_address(address)
|
||||
result = ns_query(rev)
|
||||
if not result:
|
||||
return None
|
||||
else:
|
||||
result = address
|
||||
|
||||
if fqdn:
|
||||
# strip trailing .
|
||||
if result.endswith('.'):
|
||||
return result[:-1]
|
||||
else:
|
||||
return result
|
||||
else:
|
||||
return result.split('.')[0]
|
||||
is_ip = ip.is_ip
|
||||
ns_query = ip.ns_query
|
||||
get_host_ip = ip.get_host_ip
|
||||
get_hostname = ip.get_hostname
|
||||
|
||||
|
||||
def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'):
|
||||
|
|
|
@ -17,8 +17,6 @@
|
|||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
|
||||
|
||||
from charmhelpers.fetch import apt_install, apt_update
|
||||
from charmhelpers.core.hookenv import log
|
||||
|
||||
|
@ -29,6 +27,8 @@ except ImportError:
|
|||
apt_install('python-pip')
|
||||
from pip import main as pip_execute
|
||||
|
||||
__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
|
||||
|
||||
|
||||
def parse_options(given, available):
|
||||
"""Given a set of options, check if available"""
|
||||
|
|
|
@ -17,11 +17,11 @@
|
|||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
|
||||
|
||||
import io
|
||||
import os
|
||||
|
||||
__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
|
||||
|
||||
|
||||
class Fstab(io.FileIO):
|
||||
"""This class extends file in order to implement a file reader/writer
|
||||
|
@ -77,7 +77,7 @@ class Fstab(io.FileIO):
|
|||
for line in self.readlines():
|
||||
line = line.decode('us-ascii')
|
||||
try:
|
||||
if line.strip() and not line.startswith("#"):
|
||||
if line.strip() and not line.strip().startswith("#"):
|
||||
yield self._hydrate_entry(line)
|
||||
except ValueError:
|
||||
pass
|
||||
|
@ -104,7 +104,7 @@ class Fstab(io.FileIO):
|
|||
|
||||
found = False
|
||||
for index, line in enumerate(lines):
|
||||
if not line.startswith("#"):
|
||||
if line.strip() and not line.strip().startswith("#"):
|
||||
if self._hydrate_entry(line) == entry:
|
||||
found = True
|
||||
break
|
||||
|
|
|
@ -566,3 +566,29 @@ class Hooks(object):
|
|||
def charm_dir():
|
||||
"""Return the root directory of the current charm"""
|
||||
return os.environ.get('CHARM_DIR')
|
||||
|
||||
|
||||
@cached
|
||||
def action_get(key=None):
|
||||
"""Gets the value of an action parameter, or all key/value param pairs"""
|
||||
cmd = ['action-get']
|
||||
if key is not None:
|
||||
cmd.append(key)
|
||||
cmd.append('--format=json')
|
||||
action_data = json.loads(subprocess.check_output(cmd).decode('UTF-8'))
|
||||
return action_data
|
||||
|
||||
|
||||
def action_set(values):
|
||||
"""Sets the values to be returned after the action finishes"""
|
||||
cmd = ['action-set']
|
||||
for k, v in list(values.items()):
|
||||
cmd.append('{}={}'.format(k, v))
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
|
||||
def action_fail(message):
|
||||
"""Sets the action status to failed and sets the error message.
|
||||
|
||||
The results set by action_set are preserved."""
|
||||
subprocess.check_call(['action-fail', message])
|
||||
|
|
|
@ -191,11 +191,11 @@ def mkdir(path, owner='root', group='root', perms=0o555, force=False):
|
|||
|
||||
|
||||
def write_file(path, content, owner='root', group='root', perms=0o444):
|
||||
"""Create or overwrite a file with the contents of a string"""
|
||||
"""Create or overwrite a file with the contents of a byte string."""
|
||||
log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
|
||||
uid = pwd.getpwnam(owner).pw_uid
|
||||
gid = grp.getgrnam(group).gr_gid
|
||||
with open(path, 'w') as target:
|
||||
with open(path, 'wb') as target:
|
||||
os.fchown(target.fileno(), uid, gid)
|
||||
os.fchmod(target.fileno(), perms)
|
||||
target.write(content)
|
||||
|
@ -305,11 +305,11 @@ def restart_on_change(restart_map, stopstart=False):
|
|||
ceph_client_changed function.
|
||||
"""
|
||||
def wrap(f):
|
||||
def wrapped_f(*args):
|
||||
def wrapped_f(*args, **kwargs):
|
||||
checksums = {}
|
||||
for path in restart_map:
|
||||
checksums[path] = file_hash(path)
|
||||
f(*args)
|
||||
f(*args, **kwargs)
|
||||
restarts = []
|
||||
for path in restart_map:
|
||||
if checksums[path] != file_hash(path):
|
||||
|
@ -339,12 +339,16 @@ def lsb_release():
|
|||
def pwgen(length=None):
|
||||
"""Generate a random pasword."""
|
||||
if length is None:
|
||||
# A random length is ok to use a weak PRNG
|
||||
length = random.choice(range(35, 45))
|
||||
alphanumeric_chars = [
|
||||
l for l in (string.ascii_letters + string.digits)
|
||||
if l not in 'l0QD1vAEIOUaeiou']
|
||||
# Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the
|
||||
# actual password
|
||||
random_generator = random.SystemRandom()
|
||||
random_chars = [
|
||||
random.choice(alphanumeric_chars) for _ in range(length)]
|
||||
random_generator.choice(alphanumeric_chars) for _ in range(length)]
|
||||
return(''.join(random_chars))
|
||||
|
||||
|
||||
|
@ -361,7 +365,7 @@ def list_nics(nic_type):
|
|||
ip_output = (line for line in ip_output if line)
|
||||
for line in ip_output:
|
||||
if line.split()[1].startswith(int_type):
|
||||
matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line)
|
||||
matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line)
|
||||
if matched:
|
||||
interface = matched.groups()[0]
|
||||
else:
|
||||
|
|
|
@ -45,12 +45,14 @@ class RelationContext(dict):
|
|||
"""
|
||||
name = None
|
||||
interface = None
|
||||
required_keys = []
|
||||
|
||||
def __init__(self, name=None, additional_required_keys=None):
|
||||
if not hasattr(self, 'required_keys'):
|
||||
self.required_keys = []
|
||||
|
||||
if name is not None:
|
||||
self.name = name
|
||||
if additional_required_keys is not None:
|
||||
if additional_required_keys:
|
||||
self.required_keys.extend(additional_required_keys)
|
||||
self.get_data()
|
||||
|
||||
|
@ -134,7 +136,10 @@ class MysqlRelation(RelationContext):
|
|||
"""
|
||||
name = 'db'
|
||||
interface = 'mysql'
|
||||
required_keys = ['host', 'user', 'password', 'database']
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.required_keys = ['host', 'user', 'password', 'database']
|
||||
RelationContext.__init__(self, *args, **kwargs)
|
||||
|
||||
|
||||
class HttpRelation(RelationContext):
|
||||
|
@ -146,7 +151,10 @@ class HttpRelation(RelationContext):
|
|||
"""
|
||||
name = 'website'
|
||||
interface = 'http'
|
||||
required_keys = ['host', 'port']
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.required_keys = ['host', 'port']
|
||||
RelationContext.__init__(self, *args, **kwargs)
|
||||
|
||||
def provide_data(self):
|
||||
return {
|
||||
|
|
|
@ -0,0 +1,42 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import six
|
||||
|
||||
|
||||
def bool_from_string(value):
|
||||
"""Interpret string value as boolean.
|
||||
|
||||
Returns True if value translates to True otherwise False.
|
||||
"""
|
||||
if isinstance(value, six.string_types):
|
||||
value = six.text_type(value)
|
||||
else:
|
||||
msg = "Unable to interpret non-string value '%s' as boolean" % (value)
|
||||
raise ValueError(msg)
|
||||
|
||||
value = value.strip().lower()
|
||||
|
||||
if value in ['y', 'yes', 'true', 't']:
|
||||
return True
|
||||
elif value in ['n', 'no', 'false', 'f']:
|
||||
return False
|
||||
|
||||
msg = "Unable to interpret string value '%s' as boolean" % (value)
|
||||
raise ValueError(msg)
|
|
@ -17,8 +17,6 @@
|
|||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
|
||||
|
||||
import yaml
|
||||
|
||||
from subprocess import check_call
|
||||
|
@ -26,25 +24,33 @@ from subprocess import check_call
|
|||
from charmhelpers.core.hookenv import (
|
||||
log,
|
||||
DEBUG,
|
||||
ERROR,
|
||||
)
|
||||
|
||||
__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
|
||||
|
||||
|
||||
def create(sysctl_dict, sysctl_file):
|
||||
"""Creates a sysctl.conf file from a YAML associative array
|
||||
|
||||
:param sysctl_dict: a dict of sysctl options eg { 'kernel.max_pid': 1337 }
|
||||
:type sysctl_dict: dict
|
||||
:param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }"
|
||||
:type sysctl_dict: str
|
||||
:param sysctl_file: path to the sysctl file to be saved
|
||||
:type sysctl_file: str or unicode
|
||||
:returns: None
|
||||
"""
|
||||
sysctl_dict = yaml.load(sysctl_dict)
|
||||
try:
|
||||
sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
|
||||
except yaml.YAMLError:
|
||||
log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
|
||||
level=ERROR)
|
||||
return
|
||||
|
||||
with open(sysctl_file, "w") as fd:
|
||||
for key, value in sysctl_dict.items():
|
||||
for key, value in sysctl_dict_parsed.items():
|
||||
fd.write("{}={}\n".format(key, value))
|
||||
|
||||
log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict),
|
||||
log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed),
|
||||
level=DEBUG)
|
||||
|
||||
check_call(["sysctl", "-p", sysctl_file])
|
||||
|
|
|
@ -21,7 +21,7 @@ from charmhelpers.core import hookenv
|
|||
|
||||
|
||||
def render(source, target, context, owner='root', group='root',
|
||||
perms=0o444, templates_dir=None):
|
||||
perms=0o444, templates_dir=None, encoding='UTF-8'):
|
||||
"""
|
||||
Render a template.
|
||||
|
||||
|
@ -64,5 +64,5 @@ def render(source, target, context, owner='root', group='root',
|
|||
level=hookenv.ERROR)
|
||||
raise e
|
||||
content = template.render(context)
|
||||
host.mkdir(os.path.dirname(target), owner, group)
|
||||
host.write_file(target, content, owner, group, perms)
|
||||
host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
|
||||
host.write_file(target, content.encode(encoding), owner, group, perms)
|
||||
|
|
|
@ -0,0 +1,477 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
#
|
||||
# Authors:
|
||||
# Kapil Thangavelu <kapil.foss@gmail.com>
|
||||
#
|
||||
"""
|
||||
Intro
|
||||
-----
|
||||
|
||||
A simple way to store state in units. This provides a key value
|
||||
storage with support for versioned, transactional operation,
|
||||
and can calculate deltas from previous values to simplify unit logic
|
||||
when processing changes.
|
||||
|
||||
|
||||
Hook Integration
|
||||
----------------
|
||||
|
||||
There are several extant frameworks for hook execution, including
|
||||
|
||||
- charmhelpers.core.hookenv.Hooks
|
||||
- charmhelpers.core.services.ServiceManager
|
||||
|
||||
The storage classes are framework agnostic, one simple integration is
|
||||
via the HookData contextmanager. It will record the current hook
|
||||
execution environment (including relation data, config data, etc.),
|
||||
setup a transaction and allow easy access to the changes from
|
||||
previously seen values. One consequence of the integration is the
|
||||
reservation of particular keys ('rels', 'unit', 'env', 'config',
|
||||
'charm_revisions') for their respective values.
|
||||
|
||||
Here's a fully worked integration example using hookenv.Hooks::
|
||||
|
||||
from charmhelper.core import hookenv, unitdata
|
||||
|
||||
hook_data = unitdata.HookData()
|
||||
db = unitdata.kv()
|
||||
hooks = hookenv.Hooks()
|
||||
|
||||
@hooks.hook
|
||||
def config_changed():
|
||||
# Print all changes to configuration from previously seen
|
||||
# values.
|
||||
for changed, (prev, cur) in hook_data.conf.items():
|
||||
print('config changed', changed,
|
||||
'previous value', prev,
|
||||
'current value', cur)
|
||||
|
||||
# Get some unit specific bookeeping
|
||||
if not db.get('pkg_key'):
|
||||
key = urllib.urlopen('https://example.com/pkg_key').read()
|
||||
db.set('pkg_key', key)
|
||||
|
||||
# Directly access all charm config as a mapping.
|
||||
conf = db.getrange('config', True)
|
||||
|
||||
# Directly access all relation data as a mapping
|
||||
rels = db.getrange('rels', True)
|
||||
|
||||
if __name__ == '__main__':
|
||||
with hook_data():
|
||||
hook.execute()
|
||||
|
||||
|
||||
A more basic integration is via the hook_scope context manager which simply
|
||||
manages transaction scope (and records hook name, and timestamp)::
|
||||
|
||||
>>> from unitdata import kv
|
||||
>>> db = kv()
|
||||
>>> with db.hook_scope('install'):
|
||||
... # do work, in transactional scope.
|
||||
... db.set('x', 1)
|
||||
>>> db.get('x')
|
||||
1
|
||||
|
||||
|
||||
Usage
|
||||
-----
|
||||
|
||||
Values are automatically json de/serialized to preserve basic typing
|
||||
and complex data struct capabilities (dicts, lists, ints, booleans, etc).
|
||||
|
||||
Individual values can be manipulated via get/set::
|
||||
|
||||
>>> kv.set('y', True)
|
||||
>>> kv.get('y')
|
||||
True
|
||||
|
||||
# We can set complex values (dicts, lists) as a single key.
|
||||
>>> kv.set('config', {'a': 1, 'b': True'})
|
||||
|
||||
# Also supports returning dictionaries as a record which
|
||||
# provides attribute access.
|
||||
>>> config = kv.get('config', record=True)
|
||||
>>> config.b
|
||||
True
|
||||
|
||||
|
||||
Groups of keys can be manipulated with update/getrange::
|
||||
|
||||
>>> kv.update({'z': 1, 'y': 2}, prefix="gui.")
|
||||
>>> kv.getrange('gui.', strip=True)
|
||||
{'z': 1, 'y': 2}
|
||||
|
||||
When updating values, its very helpful to understand which values
|
||||
have actually changed and how have they changed. The storage
|
||||
provides a delta method to provide for this::
|
||||
|
||||
>>> data = {'debug': True, 'option': 2}
|
||||
>>> delta = kv.delta(data, 'config.')
|
||||
>>> delta.debug.previous
|
||||
None
|
||||
>>> delta.debug.current
|
||||
True
|
||||
>>> delta
|
||||
{'debug': (None, True), 'option': (None, 2)}
|
||||
|
||||
Note the delta method does not persist the actual change, it needs to
|
||||
be explicitly saved via 'update' method::
|
||||
|
||||
>>> kv.update(data, 'config.')
|
||||
|
||||
Values modified in the context of a hook scope retain historical values
|
||||
associated to the hookname.
|
||||
|
||||
>>> with db.hook_scope('config-changed'):
|
||||
... db.set('x', 42)
|
||||
>>> db.gethistory('x')
|
||||
[(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'),
|
||||
(2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')]
|
||||
|
||||
"""
|
||||
|
||||
import collections
|
||||
import contextlib
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import pprint
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
__author__ = 'Kapil Thangavelu <kapil.foss@gmail.com>'
|
||||
|
||||
|
||||
class Storage(object):
|
||||
"""Simple key value database for local unit state within charms.
|
||||
|
||||
Modifications are automatically committed at hook exit. That's
|
||||
currently regardless of exit code.
|
||||
|
||||
To support dicts, lists, integer, floats, and booleans values
|
||||
are automatically json encoded/decoded.
|
||||
"""
|
||||
def __init__(self, path=None):
|
||||
self.db_path = path
|
||||
if path is None:
|
||||
self.db_path = os.path.join(
|
||||
os.environ.get('CHARM_DIR', ''), '.unit-state.db')
|
||||
self.conn = sqlite3.connect('%s' % self.db_path)
|
||||
self.cursor = self.conn.cursor()
|
||||
self.revision = None
|
||||
self._closed = False
|
||||
self._init()
|
||||
|
||||
def close(self):
|
||||
if self._closed:
|
||||
return
|
||||
self.flush(False)
|
||||
self.cursor.close()
|
||||
self.conn.close()
|
||||
self._closed = True
|
||||
|
||||
def _scoped_query(self, stmt, params=None):
|
||||
if params is None:
|
||||
params = []
|
||||
return stmt, params
|
||||
|
||||
def get(self, key, default=None, record=False):
|
||||
self.cursor.execute(
|
||||
*self._scoped_query(
|
||||
'select data from kv where key=?', [key]))
|
||||
result = self.cursor.fetchone()
|
||||
if not result:
|
||||
return default
|
||||
if record:
|
||||
return Record(json.loads(result[0]))
|
||||
return json.loads(result[0])
|
||||
|
||||
def getrange(self, key_prefix, strip=False):
|
||||
stmt = "select key, data from kv where key like '%s%%'" % key_prefix
|
||||
self.cursor.execute(*self._scoped_query(stmt))
|
||||
result = self.cursor.fetchall()
|
||||
|
||||
if not result:
|
||||
return None
|
||||
if not strip:
|
||||
key_prefix = ''
|
||||
return dict([
|
||||
(k[len(key_prefix):], json.loads(v)) for k, v in result])
|
||||
|
||||
def update(self, mapping, prefix=""):
|
||||
for k, v in mapping.items():
|
||||
self.set("%s%s" % (prefix, k), v)
|
||||
|
||||
def unset(self, key):
|
||||
self.cursor.execute('delete from kv where key=?', [key])
|
||||
if self.revision and self.cursor.rowcount:
|
||||
self.cursor.execute(
|
||||
'insert into kv_revisions values (?, ?, ?)',
|
||||
[key, self.revision, json.dumps('DELETED')])
|
||||
|
||||
def set(self, key, value):
|
||||
serialized = json.dumps(value)
|
||||
|
||||
self.cursor.execute(
|
||||
'select data from kv where key=?', [key])
|
||||
exists = self.cursor.fetchone()
|
||||
|
||||
# Skip mutations to the same value
|
||||
if exists:
|
||||
if exists[0] == serialized:
|
||||
return value
|
||||
|
||||
if not exists:
|
||||
self.cursor.execute(
|
||||
'insert into kv (key, data) values (?, ?)',
|
||||
(key, serialized))
|
||||
else:
|
||||
self.cursor.execute('''
|
||||
update kv
|
||||
set data = ?
|
||||
where key = ?''', [serialized, key])
|
||||
|
||||
# Save
|
||||
if not self.revision:
|
||||
return value
|
||||
|
||||
self.cursor.execute(
|
||||
'select 1 from kv_revisions where key=? and revision=?',
|
||||
[key, self.revision])
|
||||
exists = self.cursor.fetchone()
|
||||
|
||||
if not exists:
|
||||
self.cursor.execute(
|
||||
'''insert into kv_revisions (
|
||||
revision, key, data) values (?, ?, ?)''',
|
||||
(self.revision, key, serialized))
|
||||
else:
|
||||
self.cursor.execute(
|
||||
'''
|
||||
update kv_revisions
|
||||
set data = ?
|
||||
where key = ?
|
||||
and revision = ?''',
|
||||
[serialized, key, self.revision])
|
||||
|
||||
return value
|
||||
|
||||
def delta(self, mapping, prefix):
|
||||
"""
|
||||
return a delta containing values that have changed.
|
||||
"""
|
||||
previous = self.getrange(prefix, strip=True)
|
||||
if not previous:
|
||||
pk = set()
|
||||
else:
|
||||
pk = set(previous.keys())
|
||||
ck = set(mapping.keys())
|
||||
delta = DeltaSet()
|
||||
|
||||
# added
|
||||
for k in ck.difference(pk):
|
||||
delta[k] = Delta(None, mapping[k])
|
||||
|
||||
# removed
|
||||
for k in pk.difference(ck):
|
||||
delta[k] = Delta(previous[k], None)
|
||||
|
||||
# changed
|
||||
for k in pk.intersection(ck):
|
||||
c = mapping[k]
|
||||
p = previous[k]
|
||||
if c != p:
|
||||
delta[k] = Delta(p, c)
|
||||
|
||||
return delta
|
||||
|
||||
@contextlib.contextmanager
|
||||
def hook_scope(self, name=""):
|
||||
"""Scope all future interactions to the current hook execution
|
||||
revision."""
|
||||
assert not self.revision
|
||||
self.cursor.execute(
|
||||
'insert into hooks (hook, date) values (?, ?)',
|
||||
(name or sys.argv[0],
|
||||
datetime.datetime.utcnow().isoformat()))
|
||||
self.revision = self.cursor.lastrowid
|
||||
try:
|
||||
yield self.revision
|
||||
self.revision = None
|
||||
except:
|
||||
self.flush(False)
|
||||
self.revision = None
|
||||
raise
|
||||
else:
|
||||
self.flush()
|
||||
|
||||
def flush(self, save=True):
|
||||
if save:
|
||||
self.conn.commit()
|
||||
elif self._closed:
|
||||
return
|
||||
else:
|
||||
self.conn.rollback()
|
||||
|
||||
def _init(self):
|
||||
self.cursor.execute('''
|
||||
create table if not exists kv (
|
||||
key text,
|
||||
data text,
|
||||
primary key (key)
|
||||
)''')
|
||||
self.cursor.execute('''
|
||||
create table if not exists kv_revisions (
|
||||
key text,
|
||||
revision integer,
|
||||
data text,
|
||||
primary key (key, revision)
|
||||
)''')
|
||||
self.cursor.execute('''
|
||||
create table if not exists hooks (
|
||||
version integer primary key autoincrement,
|
||||
hook text,
|
||||
date text
|
||||
)''')
|
||||
self.conn.commit()
|
||||
|
||||
def gethistory(self, key, deserialize=False):
|
||||
self.cursor.execute(
|
||||
'''
|
||||
select kv.revision, kv.key, kv.data, h.hook, h.date
|
||||
from kv_revisions kv,
|
||||
hooks h
|
||||
where kv.key=?
|
||||
and kv.revision = h.version
|
||||
''', [key])
|
||||
if deserialize is False:
|
||||
return self.cursor.fetchall()
|
||||
return map(_parse_history, self.cursor.fetchall())
|
||||
|
||||
def debug(self, fh=sys.stderr):
|
||||
self.cursor.execute('select * from kv')
|
||||
pprint.pprint(self.cursor.fetchall(), stream=fh)
|
||||
self.cursor.execute('select * from kv_revisions')
|
||||
pprint.pprint(self.cursor.fetchall(), stream=fh)
|
||||
|
||||
|
||||
def _parse_history(d):
|
||||
return (d[0], d[1], json.loads(d[2]), d[3],
|
||||
datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f"))
|
||||
|
||||
|
||||
class HookData(object):
|
||||
"""Simple integration for existing hook exec frameworks.
|
||||
|
||||
Records all unit information, and stores deltas for processing
|
||||
by the hook.
|
||||
|
||||
Sample::
|
||||
|
||||
from charmhelper.core import hookenv, unitdata
|
||||
|
||||
changes = unitdata.HookData()
|
||||
db = unitdata.kv()
|
||||
hooks = hookenv.Hooks()
|
||||
|
||||
@hooks.hook
|
||||
def config_changed():
|
||||
# View all changes to configuration
|
||||
for changed, (prev, cur) in changes.conf.items():
|
||||
print('config changed', changed,
|
||||
'previous value', prev,
|
||||
'current value', cur)
|
||||
|
||||
# Get some unit specific bookeeping
|
||||
if not db.get('pkg_key'):
|
||||
key = urllib.urlopen('https://example.com/pkg_key').read()
|
||||
db.set('pkg_key', key)
|
||||
|
||||
if __name__ == '__main__':
|
||||
with changes():
|
||||
hook.execute()
|
||||
|
||||
"""
|
||||
def __init__(self):
|
||||
self.kv = kv()
|
||||
self.conf = None
|
||||
self.rels = None
|
||||
|
||||
@contextlib.contextmanager
|
||||
def __call__(self):
|
||||
from charmhelpers.core import hookenv
|
||||
hook_name = hookenv.hook_name()
|
||||
|
||||
with self.kv.hook_scope(hook_name):
|
||||
self._record_charm_version(hookenv.charm_dir())
|
||||
delta_config, delta_relation = self._record_hook(hookenv)
|
||||
yield self.kv, delta_config, delta_relation
|
||||
|
||||
def _record_charm_version(self, charm_dir):
|
||||
# Record revisions.. charm revisions are meaningless
|
||||
# to charm authors as they don't control the revision.
|
||||
# so logic dependnent on revision is not particularly
|
||||
# useful, however it is useful for debugging analysis.
|
||||
charm_rev = open(
|
||||
os.path.join(charm_dir, 'revision')).read().strip()
|
||||
charm_rev = charm_rev or '0'
|
||||
revs = self.kv.get('charm_revisions', [])
|
||||
if charm_rev not in revs:
|
||||
revs.append(charm_rev.strip() or '0')
|
||||
self.kv.set('charm_revisions', revs)
|
||||
|
||||
def _record_hook(self, hookenv):
|
||||
data = hookenv.execution_environment()
|
||||
self.conf = conf_delta = self.kv.delta(data['conf'], 'config')
|
||||
self.rels = rels_delta = self.kv.delta(data['rels'], 'rels')
|
||||
self.kv.set('env', data['env'])
|
||||
self.kv.set('unit', data['unit'])
|
||||
self.kv.set('relid', data.get('relid'))
|
||||
return conf_delta, rels_delta
|
||||
|
||||
|
||||
class Record(dict):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def __getattr__(self, k):
|
||||
if k in self:
|
||||
return self[k]
|
||||
raise AttributeError(k)
|
||||
|
||||
|
||||
class DeltaSet(Record):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
Delta = collections.namedtuple('Delta', ['previous', 'current'])
|
||||
|
||||
|
||||
_KV = None
|
||||
|
||||
|
||||
def kv():
|
||||
global _KV
|
||||
if _KV is None:
|
||||
_KV = Storage()
|
||||
return _KV
|
|
@ -18,6 +18,16 @@ import os
|
|||
import hashlib
|
||||
import re
|
||||
|
||||
from charmhelpers.fetch import (
|
||||
BaseFetchHandler,
|
||||
UnhandledSource
|
||||
)
|
||||
from charmhelpers.payload.archive import (
|
||||
get_archive_handler,
|
||||
extract,
|
||||
)
|
||||
from charmhelpers.core.host import mkdir, check_hash
|
||||
|
||||
import six
|
||||
if six.PY3:
|
||||
from urllib.request import (
|
||||
|
@ -35,16 +45,6 @@ else:
|
|||
)
|
||||
from urlparse import urlparse, urlunparse, parse_qs
|
||||
|
||||
from charmhelpers.fetch import (
|
||||
BaseFetchHandler,
|
||||
UnhandledSource
|
||||
)
|
||||
from charmhelpers.payload.archive import (
|
||||
get_archive_handler,
|
||||
extract,
|
||||
)
|
||||
from charmhelpers.core.host import mkdir, check_hash
|
||||
|
||||
|
||||
def splituser(host):
|
||||
'''urllib.splituser(), but six's support of this seems broken'''
|
||||
|
|
|
@ -32,7 +32,7 @@ except ImportError:
|
|||
apt_install("python-git")
|
||||
from git import Repo
|
||||
|
||||
from git.exc import GitCommandError
|
||||
from git.exc import GitCommandError # noqa E402
|
||||
|
||||
|
||||
class GitUrlFetchHandler(BaseFetchHandler):
|
||||
|
|
|
@ -290,7 +290,7 @@ def configure_cluster_global():
|
|||
cmd = "crm configure property no-quorum-policy=ignore"
|
||||
|
||||
pcmk.commit(cmd)
|
||||
cmd = ('crm configure rsc_defaults $id="rsc-options '
|
||||
cmd = ('crm configure rsc_defaults $id="rsc-options" '
|
||||
'resource-stickiness="100"')
|
||||
pcmk.commit(cmd)
|
||||
|
||||
|
|
|
@ -1,13 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
# Check if amulet is installed before adding repository and updating apt-get.
|
||||
dpkg -s amulet
|
||||
if [ $? -ne 0 ]; then
|
||||
sudo add-apt-repository -y ppa:juju/stable
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y amulet
|
||||
fi
|
||||
#!/bin/bash -eux
|
||||
# Install amulet packages
|
||||
sudo apt-get update --yes
|
||||
|
||||
# Install any additional python packages or software here.
|
||||
sudo apt-get install --yes python-amulet python-keystoneclient || true
|
||||
|
|
|
@ -1,33 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# This amulet test deploys the bundles.yaml file in this directory.
|
||||
|
||||
import os
|
||||
import unittest
|
||||
import yaml
|
||||
import amulet
|
||||
|
||||
seconds_to_wait = 600
|
||||
|
||||
|
||||
class BundleTest(unittest.TestCase):
|
||||
""" Create a class for testing the charm in the unit test framework. """
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
""" Set up an amulet deployment using the bundle. """
|
||||
d = amulet.Deployment()
|
||||
bundle_path = os.path.join(os.path.dirname(__file__), 'bundles.yaml')
|
||||
with open(bundle_path, 'r') as bundle_file:
|
||||
contents = yaml.safe_load(bundle_file)
|
||||
d.load(contents)
|
||||
d.setup(seconds_to_wait)
|
||||
d.sentry.wait(seconds_to_wait)
|
||||
cls.d = d
|
||||
|
||||
def test_deployed(self):
|
||||
""" Test to see if the bundle deployed successfully. """
|
||||
self.assertTrue(self.d.deployed)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
|
@ -0,0 +1,9 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
"""Amulet tests on a basic hacluster deployment on trusty-icehouse."""
|
||||
|
||||
from basic_deployment import HAClusterBasicDeployment
|
||||
|
||||
if __name__ == '__main__':
|
||||
deployment = HAClusterBasicDeployment(series='trusty')
|
||||
deployment.run_tests()
|
|
@ -0,0 +1,106 @@
|
|||
#!/usr/bin/env python
|
||||
import os
|
||||
import amulet
|
||||
|
||||
import keystoneclient.v2_0 as keystone_client
|
||||
|
||||
from charmhelpers.contrib.openstack.amulet.deployment import (
|
||||
OpenStackAmuletDeployment
|
||||
)
|
||||
from charmhelpers.contrib.openstack.amulet.utils import (
|
||||
OpenStackAmuletUtils,
|
||||
DEBUG, # flake8: noqa
|
||||
ERROR
|
||||
)
|
||||
|
||||
# Use DEBUG to turn on debug logging
|
||||
u = OpenStackAmuletUtils(DEBUG)
|
||||
KEYSTONE_SVC_VIP = '10.5.100.1'
|
||||
|
||||
seconds_to_wait = 600
|
||||
|
||||
|
||||
class HAClusterBasicDeployment(OpenStackAmuletDeployment):
|
||||
|
||||
def __init__(self, series=None, openstack=None, source=None, stable=False):
|
||||
"""Deploy the entire test environment."""
|
||||
super(HAClusterBasicDeployment, self).__init__(series, openstack,
|
||||
source, stable)
|
||||
self._add_services()
|
||||
self._add_relations()
|
||||
self._configure_services()
|
||||
self._deploy()
|
||||
self._initialize_tests()
|
||||
|
||||
def _add_services(self):
|
||||
this_service = {'name': 'hacluster'}
|
||||
other_services = [{'name': 'mysql'}, {'name': 'keystone', 'units': 3}]
|
||||
super(HAClusterBasicDeployment, self)._add_services(this_service,
|
||||
other_services)
|
||||
|
||||
def _add_relations(self):
|
||||
relations = {'keystone:shared-db': 'mysql:shared-db',
|
||||
'hacluster:ha': 'keystone:ha'}
|
||||
super(HAClusterBasicDeployment, self)._add_relations(relations)
|
||||
|
||||
def _configure_services(self):
|
||||
keystone_config = {'admin-password': 'openstack',
|
||||
'admin-token': 'ubuntutesting',
|
||||
'vip': KEYSTONE_SVC_VIP}
|
||||
mysql_config = {'dataset-size': '50%'}
|
||||
configs = {'keystone': keystone_config,
|
||||
'mysql': mysql_config}
|
||||
super(HAClusterBasicDeployment, self)._configure_services(configs)
|
||||
|
||||
def _authenticate_keystone_admin(self, keystone_sentry, user, password,
|
||||
tenant, service_ip=None):
|
||||
"""Authenticates admin user with the keystone admin endpoint.
|
||||
|
||||
This should be factored into:L
|
||||
|
||||
charmhelpers.contrib.openstack.amulet.utils.OpenStackAmuletUtils
|
||||
"""
|
||||
if not service_ip:
|
||||
unit = keystone_sentry
|
||||
service_ip = unit.relation('shared-db',
|
||||
'mysql:shared-db')['private-address']
|
||||
|
||||
ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8'))
|
||||
return keystone_client.Client(username=user, password=password,
|
||||
tenant_name=tenant, auth_url=ep)
|
||||
|
||||
def _initialize_tests(self):
|
||||
"""Perform final initialization before tests get run."""
|
||||
# Access the sentries for inspecting service units
|
||||
self.mysql_sentry = self.d.sentry.unit['mysql/0']
|
||||
self.keystone_sentry = self.d.sentry.unit['keystone/0']
|
||||
# NOTE: the hacluster unit id may not correspond with its parent unit
|
||||
# id.
|
||||
self.hacluster_sentry = self.d.sentry.unit['hacluster/0']
|
||||
|
||||
# Authenticate keystone admin
|
||||
vip = KEYSTONE_SVC_VIP
|
||||
self.keystone = self._authenticate_keystone_admin(self.keystone_sentry,
|
||||
user='admin',
|
||||
password='openstack',
|
||||
tenant='admin',
|
||||
service_ip=vip)
|
||||
|
||||
# Create a demo tenant/role/user
|
||||
self.demo_tenant = 'demoTenant'
|
||||
self.demo_role = 'demoRole'
|
||||
self.demo_user = 'demoUser'
|
||||
if not u.tenant_exists(self.keystone, self.demo_tenant):
|
||||
tenant = self.keystone.tenants.create(tenant_name=self.demo_tenant,
|
||||
description='demo tenant',
|
||||
enabled=True)
|
||||
self.keystone.roles.create(name=self.demo_role)
|
||||
self.keystone.users.create(name=self.demo_user, password='password',
|
||||
tenant_id=tenant.id,
|
||||
email='demo@demo.com')
|
||||
|
||||
# Authenticate keystone demo
|
||||
self.keystone_demo = u.authenticate_keystone_user(self.keystone,
|
||||
user=self.demo_user,
|
||||
password='password',
|
||||
tenant=self.demo_tenant)
|
|
@ -1,15 +0,0 @@
|
|||
hacluster-mysql:
|
||||
series: trusty
|
||||
services:
|
||||
hacluster:
|
||||
charm: hacluster
|
||||
num_units: 0
|
||||
mysql:
|
||||
charm: cs:trusty/mysql
|
||||
num_units: 2
|
||||
options:
|
||||
"dataset-size": 128M
|
||||
vip: 192.168.21.1
|
||||
relations:
|
||||
- - "mysql:ha"
|
||||
- "hacluster:ha"
|
|
@ -0,0 +1,38 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# Bootstrap charm-helpers, installing its dependencies if necessary using
|
||||
# only standard libraries.
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
try:
|
||||
import six # flake8: noqa
|
||||
except ImportError:
|
||||
if sys.version_info.major == 2:
|
||||
subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
|
||||
else:
|
||||
subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
|
||||
import six # flake8: noqa
|
||||
|
||||
try:
|
||||
import yaml # flake8: noqa
|
||||
except ImportError:
|
||||
if sys.version_info.major == 2:
|
||||
subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
|
||||
else:
|
||||
subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
|
||||
import yaml # flake8: noqa
|
|
@ -0,0 +1,15 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
|
@ -0,0 +1,15 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
|
@ -0,0 +1,93 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import amulet
|
||||
import os
|
||||
import six
|
||||
|
||||
|
||||
class AmuletDeployment(object):
|
||||
"""Amulet deployment.
|
||||
|
||||
This class provides generic Amulet deployment and test runner
|
||||
methods.
|
||||
"""
|
||||
|
||||
def __init__(self, series=None):
|
||||
"""Initialize the deployment environment."""
|
||||
self.series = None
|
||||
|
||||
if series:
|
||||
self.series = series
|
||||
self.d = amulet.Deployment(series=self.series)
|
||||
else:
|
||||
self.d = amulet.Deployment()
|
||||
|
||||
def _add_services(self, this_service, other_services):
|
||||
"""Add services.
|
||||
|
||||
Add services to the deployment where this_service is the local charm
|
||||
that we're testing and other_services are the other services that
|
||||
are being used in the local amulet tests.
|
||||
"""
|
||||
if this_service['name'] != os.path.basename(os.getcwd()):
|
||||
s = this_service['name']
|
||||
msg = "The charm's root directory name needs to be {}".format(s)
|
||||
amulet.raise_status(amulet.FAIL, msg=msg)
|
||||
|
||||
if 'units' not in this_service:
|
||||
this_service['units'] = 1
|
||||
|
||||
self.d.add(this_service['name'], units=this_service['units'])
|
||||
|
||||
for svc in other_services:
|
||||
if 'location' in svc:
|
||||
branch_location = svc['location']
|
||||
elif self.series:
|
||||
branch_location = 'cs:{}/{}'.format(self.series, svc['name']),
|
||||
else:
|
||||
branch_location = None
|
||||
|
||||
if 'units' not in svc:
|
||||
svc['units'] = 1
|
||||
|
||||
self.d.add(svc['name'], charm=branch_location, units=svc['units'])
|
||||
|
||||
def _add_relations(self, relations):
|
||||
"""Add all of the relations for the services."""
|
||||
for k, v in six.iteritems(relations):
|
||||
self.d.relate(k, v)
|
||||
|
||||
def _configure_services(self, configs):
|
||||
"""Configure all of the services."""
|
||||
for service, config in six.iteritems(configs):
|
||||
self.d.configure(service, config)
|
||||
|
||||
def _deploy(self):
|
||||
"""Deploy environment and wait for all hooks to finish executing."""
|
||||
try:
|
||||
self.d.setup(timeout=900)
|
||||
self.d.sentry.wait(timeout=900)
|
||||
except amulet.helpers.TimeoutError:
|
||||
amulet.raise_status(amulet.FAIL, msg="Deployment timed out")
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
def run_tests(self):
|
||||
"""Run all of the methods that are prefixed with 'test_'."""
|
||||
for test in dir(self):
|
||||
if test.startswith('test_'):
|
||||
getattr(self, test)()
|
|
@ -0,0 +1,314 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import ConfigParser
|
||||
import io
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
||||
import six
|
||||
|
||||
|
||||
class AmuletUtils(object):
|
||||
"""Amulet utilities.
|
||||
|
||||
This class provides common utility functions that are used by Amulet
|
||||
tests.
|
||||
"""
|
||||
|
||||
def __init__(self, log_level=logging.ERROR):
|
||||
self.log = self.get_logger(level=log_level)
|
||||
|
||||
def get_logger(self, name="amulet-logger", level=logging.DEBUG):
|
||||
"""Get a logger object that will log to stdout."""
|
||||
log = logging
|
||||
logger = log.getLogger(name)
|
||||
fmt = log.Formatter("%(asctime)s %(funcName)s "
|
||||
"%(levelname)s: %(message)s")
|
||||
|
||||
handler = log.StreamHandler(stream=sys.stdout)
|
||||
handler.setLevel(level)
|
||||
handler.setFormatter(fmt)
|
||||
|
||||
logger.addHandler(handler)
|
||||
logger.setLevel(level)
|
||||
|
||||
return logger
|
||||
|
||||
def valid_ip(self, ip):
|
||||
if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def valid_url(self, url):
|
||||
p = re.compile(
|
||||
r'^(?:http|ftp)s?://'
|
||||
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # noqa
|
||||
r'localhost|'
|
||||
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'
|
||||
r'(?::\d+)?'
|
||||
r'(?:/?|[/?]\S+)$',
|
||||
re.IGNORECASE)
|
||||
if p.match(url):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def validate_services(self, commands):
|
||||
"""Validate services.
|
||||
|
||||
Verify the specified services are running on the corresponding
|
||||
service units.
|
||||
"""
|
||||
for k, v in six.iteritems(commands):
|
||||
for cmd in v:
|
||||
output, code = k.run(cmd)
|
||||
if code != 0:
|
||||
return "command `{}` returned {}".format(cmd, str(code))
|
||||
return None
|
||||
|
||||
def _get_config(self, unit, filename):
|
||||
"""Get a ConfigParser object for parsing a unit's config file."""
|
||||
file_contents = unit.file_contents(filename)
|
||||
config = ConfigParser.ConfigParser()
|
||||
config.readfp(io.StringIO(file_contents))
|
||||
return config
|
||||
|
||||
def validate_config_data(self, sentry_unit, config_file, section,
|
||||
expected):
|
||||
"""Validate config file data.
|
||||
|
||||
Verify that the specified section of the config file contains
|
||||
the expected option key:value pairs.
|
||||
"""
|
||||
config = self._get_config(sentry_unit, config_file)
|
||||
|
||||
if section != 'DEFAULT' and not config.has_section(section):
|
||||
return "section [{}] does not exist".format(section)
|
||||
|
||||
for k in expected.keys():
|
||||
if not config.has_option(section, k):
|
||||
return "section [{}] is missing option {}".format(section, k)
|
||||
if config.get(section, k) != expected[k]:
|
||||
return "section [{}] {}:{} != expected {}:{}".format(
|
||||
section, k, config.get(section, k), k, expected[k])
|
||||
return None
|
||||
|
||||
def _validate_dict_data(self, expected, actual):
|
||||
"""Validate dictionary data.
|
||||
|
||||
Compare expected dictionary data vs actual dictionary data.
|
||||
The values in the 'expected' dictionary can be strings, bools, ints,
|
||||
longs, or can be a function that evaluate a variable and returns a
|
||||
bool.
|
||||
"""
|
||||
for k, v in six.iteritems(expected):
|
||||
if k in actual:
|
||||
if (isinstance(v, six.string_types) or
|
||||
isinstance(v, bool) or
|
||||
isinstance(v, six.integer_types)):
|
||||
if v != actual[k]:
|
||||
return "{}:{}".format(k, actual[k])
|
||||
elif not v(actual[k]):
|
||||
return "{}:{}".format(k, actual[k])
|
||||
else:
|
||||
return "key '{}' does not exist".format(k)
|
||||
return None
|
||||
|
||||
def validate_relation_data(self, sentry_unit, relation, expected):
|
||||
"""Validate actual relation data based on expected relation data."""
|
||||
actual = sentry_unit.relation(relation[0], relation[1])
|
||||
self.log.debug('actual: {}'.format(repr(actual)))
|
||||
return self._validate_dict_data(expected, actual)
|
||||
|
||||
def _validate_list_data(self, expected, actual):
|
||||
"""Compare expected list vs actual list data."""
|
||||
for e in expected:
|
||||
if e not in actual:
|
||||
return "expected item {} not found in actual list".format(e)
|
||||
return None
|
||||
|
||||
def not_null(self, string):
|
||||
if string is not None:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def _get_file_mtime(self, sentry_unit, filename):
|
||||
"""Get last modification time of file."""
|
||||
return sentry_unit.file_stat(filename)['mtime']
|
||||
|
||||
def _get_dir_mtime(self, sentry_unit, directory):
|
||||
"""Get last modification time of directory."""
|
||||
return sentry_unit.directory_stat(directory)['mtime']
|
||||
|
||||
def _get_proc_start_time(self, sentry_unit, service, pgrep_full=False):
|
||||
"""Get process' start time.
|
||||
|
||||
Determine start time of the process based on the last modification
|
||||
time of the /proc/pid directory. If pgrep_full is True, the process
|
||||
name is matched against the full command line.
|
||||
"""
|
||||
if pgrep_full:
|
||||
cmd = 'pgrep -o -f {}'.format(service)
|
||||
else:
|
||||
cmd = 'pgrep -o {}'.format(service)
|
||||
cmd = cmd + ' | grep -v pgrep || exit 0'
|
||||
cmd_out = sentry_unit.run(cmd)
|
||||
self.log.debug('CMDout: ' + str(cmd_out))
|
||||
if cmd_out[0]:
|
||||
self.log.debug('Pid for %s %s' % (service, str(cmd_out[0])))
|
||||
proc_dir = '/proc/{}'.format(cmd_out[0].strip())
|
||||
return self._get_dir_mtime(sentry_unit, proc_dir)
|
||||
|
||||
def service_restarted(self, sentry_unit, service, filename,
|
||||
pgrep_full=False, sleep_time=20):
|
||||
"""Check if service was restarted.
|
||||
|
||||
Compare a service's start time vs a file's last modification time
|
||||
(such as a config file for that service) to determine if the service
|
||||
has been restarted.
|
||||
"""
|
||||
time.sleep(sleep_time)
|
||||
if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >=
|
||||
self._get_file_mtime(sentry_unit, filename)):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def service_restarted_since(self, sentry_unit, mtime, service,
|
||||
pgrep_full=False, sleep_time=20,
|
||||
retry_count=2):
|
||||
"""Check if service was been started after a given time.
|
||||
|
||||
Args:
|
||||
sentry_unit (sentry): The sentry unit to check for the service on
|
||||
mtime (float): The epoch time to check against
|
||||
service (string): service name to look for in process table
|
||||
pgrep_full (boolean): Use full command line search mode with pgrep
|
||||
sleep_time (int): Seconds to sleep before looking for process
|
||||
retry_count (int): If service is not found, how many times to retry
|
||||
|
||||
Returns:
|
||||
bool: True if service found and its start time it newer than mtime,
|
||||
False if service is older than mtime or if service was
|
||||
not found.
|
||||
"""
|
||||
self.log.debug('Checking %s restarted since %s' % (service, mtime))
|
||||
time.sleep(sleep_time)
|
||||
proc_start_time = self._get_proc_start_time(sentry_unit, service,
|
||||
pgrep_full)
|
||||
while retry_count > 0 and not proc_start_time:
|
||||
self.log.debug('No pid file found for service %s, will retry %i '
|
||||
'more times' % (service, retry_count))
|
||||
time.sleep(30)
|
||||
proc_start_time = self._get_proc_start_time(sentry_unit, service,
|
||||
pgrep_full)
|
||||
retry_count = retry_count - 1
|
||||
|
||||
if not proc_start_time:
|
||||
self.log.warn('No proc start time found, assuming service did '
|
||||
'not start')
|
||||
return False
|
||||
if proc_start_time >= mtime:
|
||||
self.log.debug('proc start time is newer than provided mtime'
|
||||
'(%s >= %s)' % (proc_start_time, mtime))
|
||||
return True
|
||||
else:
|
||||
self.log.warn('proc start time (%s) is older than provided mtime '
|
||||
'(%s), service did not restart' % (proc_start_time,
|
||||
mtime))
|
||||
return False
|
||||
|
||||
def config_updated_since(self, sentry_unit, filename, mtime,
|
||||
sleep_time=20):
|
||||
"""Check if file was modified after a given time.
|
||||
|
||||
Args:
|
||||
sentry_unit (sentry): The sentry unit to check the file mtime on
|
||||
filename (string): The file to check mtime of
|
||||
mtime (float): The epoch time to check against
|
||||
sleep_time (int): Seconds to sleep before looking for process
|
||||
|
||||
Returns:
|
||||
bool: True if file was modified more recently than mtime, False if
|
||||
file was modified before mtime,
|
||||
"""
|
||||
self.log.debug('Checking %s updated since %s' % (filename, mtime))
|
||||
time.sleep(sleep_time)
|
||||
file_mtime = self._get_file_mtime(sentry_unit, filename)
|
||||
if file_mtime >= mtime:
|
||||
self.log.debug('File mtime is newer than provided mtime '
|
||||
'(%s >= %s)' % (file_mtime, mtime))
|
||||
return True
|
||||
else:
|
||||
self.log.warn('File mtime %s is older than provided mtime %s'
|
||||
% (file_mtime, mtime))
|
||||
return False
|
||||
|
||||
def validate_service_config_changed(self, sentry_unit, mtime, service,
|
||||
filename, pgrep_full=False,
|
||||
sleep_time=20, retry_count=2):
|
||||
"""Check service and file were updated after mtime
|
||||
|
||||
Args:
|
||||
sentry_unit (sentry): The sentry unit to check for the service on
|
||||
mtime (float): The epoch time to check against
|
||||
service (string): service name to look for in process table
|
||||
filename (string): The file to check mtime of
|
||||
pgrep_full (boolean): Use full command line search mode with pgrep
|
||||
sleep_time (int): Seconds to sleep before looking for process
|
||||
retry_count (int): If service is not found, how many times to retry
|
||||
|
||||
Typical Usage:
|
||||
u = OpenStackAmuletUtils(ERROR)
|
||||
...
|
||||
mtime = u.get_sentry_time(self.cinder_sentry)
|
||||
self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'})
|
||||
if not u.validate_service_config_changed(self.cinder_sentry,
|
||||
mtime,
|
||||
'cinder-api',
|
||||
'/etc/cinder/cinder.conf')
|
||||
amulet.raise_status(amulet.FAIL, msg='update failed')
|
||||
Returns:
|
||||
bool: True if both service and file where updated/restarted after
|
||||
mtime, False if service is older than mtime or if service was
|
||||
not found or if filename was modified before mtime.
|
||||
"""
|
||||
self.log.debug('Checking %s restarted since %s' % (service, mtime))
|
||||
time.sleep(sleep_time)
|
||||
service_restart = self.service_restarted_since(sentry_unit, mtime,
|
||||
service,
|
||||
pgrep_full=pgrep_full,
|
||||
sleep_time=0,
|
||||
retry_count=retry_count)
|
||||
config_update = self.config_updated_since(sentry_unit, filename, mtime,
|
||||
sleep_time=0)
|
||||
return service_restart and config_update
|
||||
|
||||
def get_sentry_time(self, sentry_unit):
|
||||
"""Return current epoch time on a sentry"""
|
||||
cmd = "date +'%s'"
|
||||
return float(sentry_unit.run(cmd)[0])
|
||||
|
||||
def relation_error(self, name, data):
|
||||
return 'unexpected relation data in {} - {}'.format(name, data)
|
||||
|
||||
def endpoint_error(self, name, data):
|
||||
return 'unexpected endpoint data in {} - {}'.format(name, data)
|
|
@ -0,0 +1,15 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
|
@ -0,0 +1,15 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
|
@ -0,0 +1,111 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import six
|
||||
from charmhelpers.contrib.amulet.deployment import (
|
||||
AmuletDeployment
|
||||
)
|
||||
|
||||
|
||||
class OpenStackAmuletDeployment(AmuletDeployment):
|
||||
"""OpenStack amulet deployment.
|
||||
|
||||
This class inherits from AmuletDeployment and has additional support
|
||||
that is specifically for use by OpenStack charms.
|
||||
"""
|
||||
|
||||
def __init__(self, series=None, openstack=None, source=None, stable=True):
|
||||
"""Initialize the deployment environment."""
|
||||
super(OpenStackAmuletDeployment, self).__init__(series)
|
||||
self.openstack = openstack
|
||||
self.source = source
|
||||
self.stable = stable
|
||||
# Note(coreycb): this needs to be changed when new next branches come
|
||||
# out.
|
||||
self.current_next = "trusty"
|
||||
|
||||
def _determine_branch_locations(self, other_services):
|
||||
"""Determine the branch locations for the other services.
|
||||
|
||||
Determine if the local branch being tested is derived from its
|
||||
stable or next (dev) branch, and based on this, use the corresonding
|
||||
stable or next branches for the other_services."""
|
||||
base_charms = ['mysql', 'mongodb', 'rabbitmq-server']
|
||||
|
||||
if self.stable:
|
||||
for svc in other_services:
|
||||
temp = 'lp:charms/{}'
|
||||
svc['location'] = temp.format(svc['name'])
|
||||
else:
|
||||
for svc in other_services:
|
||||
if svc['name'] in base_charms:
|
||||
temp = 'lp:charms/{}'
|
||||
svc['location'] = temp.format(svc['name'])
|
||||
else:
|
||||
temp = 'lp:~openstack-charmers/charms/{}/{}/next'
|
||||
svc['location'] = temp.format(self.current_next,
|
||||
svc['name'])
|
||||
return other_services
|
||||
|
||||
def _add_services(self, this_service, other_services):
|
||||
"""Add services to the deployment and set openstack-origin/source."""
|
||||
other_services = self._determine_branch_locations(other_services)
|
||||
|
||||
super(OpenStackAmuletDeployment, self)._add_services(this_service,
|
||||
other_services)
|
||||
|
||||
services = other_services
|
||||
services.append(this_service)
|
||||
use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
|
||||
'ceph-osd', 'ceph-radosgw']
|
||||
# Openstack subordinate charms do not expose an origin option as that
|
||||
# is controlled by the principle
|
||||
ignore = ['neutron-openvswitch']
|
||||
|
||||
if self.openstack:
|
||||
for svc in services:
|
||||
if svc['name'] not in use_source + ignore:
|
||||
config = {'openstack-origin': self.openstack}
|
||||
self.d.configure(svc['name'], config)
|
||||
|
||||
if self.source:
|
||||
for svc in services:
|
||||
if svc['name'] in use_source and svc['name'] not in ignore:
|
||||
config = {'source': self.source}
|
||||
self.d.configure(svc['name'], config)
|
||||
|
||||
def _configure_services(self, configs):
|
||||
"""Configure all of the services."""
|
||||
for service, config in six.iteritems(configs):
|
||||
self.d.configure(service, config)
|
||||
|
||||
def _get_openstack_release(self):
|
||||
"""Get openstack release.
|
||||
|
||||
Return an integer representing the enum value of the openstack
|
||||
release.
|
||||
"""
|
||||
(self.precise_essex, self.precise_folsom, self.precise_grizzly,
|
||||
self.precise_havana, self.precise_icehouse,
|
||||
self.trusty_icehouse) = range(6)
|
||||
releases = {
|
||||
('precise', None): self.precise_essex,
|
||||
('precise', 'cloud:precise-folsom'): self.precise_folsom,
|
||||
('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
|
||||
('precise', 'cloud:precise-havana'): self.precise_havana,
|
||||
('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
|
||||
('trusty', None): self.trusty_icehouse}
|
||||
return releases[(self.series, self.openstack)]
|
|
@ -0,0 +1,294 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import urllib
|
||||
|
||||
import glanceclient.v1.client as glance_client
|
||||
import keystoneclient.v2_0 as keystone_client
|
||||
import novaclient.v1_1.client as nova_client
|
||||
|
||||
import six
|
||||
|
||||
from charmhelpers.contrib.amulet.utils import (
|
||||
AmuletUtils
|
||||
)
|
||||
|
||||
DEBUG = logging.DEBUG
|
||||
ERROR = logging.ERROR
|
||||
|
||||
|
||||
class OpenStackAmuletUtils(AmuletUtils):
|
||||
"""OpenStack amulet utilities.
|
||||
|
||||
This class inherits from AmuletUtils and has additional support
|
||||
that is specifically for use by OpenStack charms.
|
||||
"""
|
||||
|
||||
def __init__(self, log_level=ERROR):
|
||||
"""Initialize the deployment environment."""
|
||||
super(OpenStackAmuletUtils, self).__init__(log_level)
|
||||
|
||||
def validate_endpoint_data(self, endpoints, admin_port, internal_port,
|
||||
public_port, expected):
|
||||
"""Validate endpoint data.
|
||||
|
||||
Validate actual endpoint data vs expected endpoint data. The ports
|
||||
are used to find the matching endpoint.
|
||||
"""
|
||||
found = False
|
||||
for ep in endpoints:
|
||||
self.log.debug('endpoint: {}'.format(repr(ep)))
|
||||
if (admin_port in ep.adminurl and
|
||||
internal_port in ep.internalurl and
|
||||
public_port in ep.publicurl):
|
||||
found = True
|
||||
actual = {'id': ep.id,
|
||||
'region': ep.region,
|
||||
'adminurl': ep.adminurl,
|
||||
'internalurl': ep.internalurl,
|
||||
'publicurl': ep.publicurl,
|
||||
'service_id': ep.service_id}
|
||||
ret = self._validate_dict_data(expected, actual)
|
||||
if ret:
|
||||
return 'unexpected endpoint data - {}'.format(ret)
|
||||
|
||||
if not found:
|
||||
return 'endpoint not found'
|
||||
|
||||
def validate_svc_catalog_endpoint_data(self, expected, actual):
|
||||
"""Validate service catalog endpoint data.
|
||||
|
||||
Validate a list of actual service catalog endpoints vs a list of
|
||||
expected service catalog endpoints.
|
||||
"""
|
||||
self.log.debug('actual: {}'.format(repr(actual)))
|
||||
for k, v in six.iteritems(expected):
|
||||
if k in actual:
|
||||
ret = self._validate_dict_data(expected[k][0], actual[k][0])
|
||||
if ret:
|
||||
return self.endpoint_error(k, ret)
|
||||
else:
|
||||
return "endpoint {} does not exist".format(k)
|
||||
return ret
|
||||
|
||||
def validate_tenant_data(self, expected, actual):
|
||||
"""Validate tenant data.
|
||||
|
||||
Validate a list of actual tenant data vs list of expected tenant
|
||||
data.
|
||||
"""
|
||||
self.log.debug('actual: {}'.format(repr(actual)))
|
||||
for e in expected:
|
||||
found = False
|
||||
for act in actual:
|
||||
a = {'enabled': act.enabled, 'description': act.description,
|
||||
'name': act.name, 'id': act.id}
|
||||
if e['name'] == a['name']:
|
||||
found = True
|
||||
ret = self._validate_dict_data(e, a)
|
||||
if ret:
|
||||
return "unexpected tenant data - {}".format(ret)
|
||||
if not found:
|
||||
return "tenant {} does not exist".format(e['name'])
|
||||
return ret
|
||||
|
||||
def validate_role_data(self, expected, actual):
|
||||
"""Validate role data.
|
||||
|
||||
Validate a list of actual role data vs a list of expected role
|
||||
data.
|
||||
"""
|
||||
self.log.debug('actual: {}'.format(repr(actual)))
|
||||
for e in expected:
|
||||
found = False
|
||||
for act in actual:
|
||||
a = {'name': act.name, 'id': act.id}
|
||||
if e['name'] == a['name']:
|
||||
found = True
|
||||
ret = self._validate_dict_data(e, a)
|
||||
if ret:
|
||||
return "unexpected role data - {}".format(ret)
|
||||
if not found:
|
||||
return "role {} does not exist".format(e['name'])
|
||||
return ret
|
||||
|
||||
def validate_user_data(self, expected, actual):
|
||||
"""Validate user data.
|
||||
|
||||
Validate a list of actual user data vs a list of expected user
|
||||
data.
|
||||
"""
|
||||
self.log.debug('actual: {}'.format(repr(actual)))
|
||||
for e in expected:
|
||||
found = False
|
||||
for act in actual:
|
||||
a = {'enabled': act.enabled, 'name': act.name,
|
||||
'email': act.email, 'tenantId': act.tenantId,
|
||||
'id': act.id}
|
||||
if e['name'] == a['name']:
|
||||
found = True
|
||||
ret = self._validate_dict_data(e, a)
|
||||
if ret:
|
||||
return "unexpected user data - {}".format(ret)
|
||||
if not found:
|
||||
return "user {} does not exist".format(e['name'])
|
||||
return ret
|
||||
|
||||
def validate_flavor_data(self, expected, actual):
|
||||
"""Validate flavor data.
|
||||
|
||||
Validate a list of actual flavors vs a list of expected flavors.
|
||||
"""
|
||||
self.log.debug('actual: {}'.format(repr(actual)))
|
||||
act = [a.name for a in actual]
|
||||
return self._validate_list_data(expected, act)
|
||||
|
||||
def tenant_exists(self, keystone, tenant):
|
||||
"""Return True if tenant exists."""
|
||||
return tenant in [t.name for t in keystone.tenants.list()]
|
||||
|
||||
def authenticate_keystone_admin(self, keystone_sentry, user, password,
|
||||
tenant):
|
||||
"""Authenticates admin user with the keystone admin endpoint."""
|
||||
unit = keystone_sentry
|
||||
service_ip = unit.relation('shared-db',
|
||||
'mysql:shared-db')['private-address']
|
||||
ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8'))
|
||||
return keystone_client.Client(username=user, password=password,
|
||||
tenant_name=tenant, auth_url=ep)
|
||||
|
||||
def authenticate_keystone_user(self, keystone, user, password, tenant):
|
||||
"""Authenticates a regular user with the keystone public endpoint."""
|
||||
ep = keystone.service_catalog.url_for(service_type='identity',
|
||||
endpoint_type='publicURL')
|
||||
return keystone_client.Client(username=user, password=password,
|
||||
tenant_name=tenant, auth_url=ep)
|
||||
|
||||
def authenticate_glance_admin(self, keystone):
|
||||
"""Authenticates admin user with glance."""
|
||||
ep = keystone.service_catalog.url_for(service_type='image',
|
||||
endpoint_type='adminURL')
|
||||
return glance_client.Client(ep, token=keystone.auth_token)
|
||||
|
||||
def authenticate_nova_user(self, keystone, user, password, tenant):
|
||||
"""Authenticates a regular user with nova-api."""
|
||||
ep = keystone.service_catalog.url_for(service_type='identity',
|
||||
endpoint_type='publicURL')
|
||||
return nova_client.Client(username=user, api_key=password,
|
||||
project_id=tenant, auth_url=ep)
|
||||
|
||||
def create_cirros_image(self, glance, image_name):
|
||||
"""Download the latest cirros image and upload it to glance."""
|
||||
http_proxy = os.getenv('AMULET_HTTP_PROXY')
|
||||
self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
|
||||
if http_proxy:
|
||||
proxies = {'http': http_proxy}
|
||||
opener = urllib.FancyURLopener(proxies)
|
||||
else:
|
||||
opener = urllib.FancyURLopener()
|
||||
|
||||
f = opener.open("http://download.cirros-cloud.net/version/released")
|
||||
version = f.read().strip()
|
||||
cirros_img = "cirros-{}-x86_64-disk.img".format(version)
|
||||
local_path = os.path.join('tests', cirros_img)
|
||||
|
||||
if not os.path.exists(local_path):
|
||||
cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net",
|
||||
version, cirros_img)
|
||||
opener.retrieve(cirros_url, local_path)
|
||||
f.close()
|
||||
|
||||
with open(local_path) as f:
|
||||
image = glance.images.create(name=image_name, is_public=True,
|
||||
disk_format='qcow2',
|
||||
container_format='bare', data=f)
|
||||
count = 1
|
||||
status = image.status
|
||||
while status != 'active' and count < 10:
|
||||
time.sleep(3)
|
||||
image = glance.images.get(image.id)
|
||||
status = image.status
|
||||
self.log.debug('image status: {}'.format(status))
|
||||
count += 1
|
||||
|
||||
if status != 'active':
|
||||
self.log.error('image creation timed out')
|
||||
return None
|
||||
|
||||
return image
|
||||
|
||||
def delete_image(self, glance, image):
|
||||
"""Delete the specified image."""
|
||||
num_before = len(list(glance.images.list()))
|
||||
glance.images.delete(image)
|
||||
|
||||
count = 1
|
||||
num_after = len(list(glance.images.list()))
|
||||
while num_after != (num_before - 1) and count < 10:
|
||||
time.sleep(3)
|
||||
num_after = len(list(glance.images.list()))
|
||||
self.log.debug('number of images: {}'.format(num_after))
|
||||
count += 1
|
||||
|
||||
if num_after != (num_before - 1):
|
||||
self.log.error('image deletion timed out')
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def create_instance(self, nova, image_name, instance_name, flavor):
|
||||
"""Create the specified instance."""
|
||||
image = nova.images.find(name=image_name)
|
||||
flavor = nova.flavors.find(name=flavor)
|
||||
instance = nova.servers.create(name=instance_name, image=image,
|
||||
flavor=flavor)
|
||||
|
||||
count = 1
|
||||
status = instance.status
|
||||
while status != 'ACTIVE' and count < 60:
|
||||
time.sleep(3)
|
||||
instance = nova.servers.get(instance.id)
|
||||
status = instance.status
|
||||
self.log.debug('instance status: {}'.format(status))
|
||||
count += 1
|
||||
|
||||
if status != 'ACTIVE':
|
||||
self.log.error('instance creation timed out')
|
||||
return None
|
||||
|
||||
return instance
|
||||
|
||||
def delete_instance(self, nova, instance):
|
||||
"""Delete the specified instance."""
|
||||
num_before = len(list(nova.servers.list()))
|
||||
nova.servers.delete(instance)
|
||||
|
||||
count = 1
|
||||
num_after = len(list(nova.servers.list()))
|
||||
while num_after != (num_before - 1) and count < 10:
|
||||
time.sleep(3)
|
||||
num_after = len(list(nova.servers.list()))
|
||||
self.log.debug('number of instances: {}'.format(num_after))
|
||||
count += 1
|
||||
|
||||
if num_after != (num_before - 1):
|
||||
self.log.error('instance deletion timed out')
|
||||
return False
|
||||
|
||||
return True
|
Loading…
Reference in New Issue