openstack
/
charm-rabbitmq-server
Code Issues Proposed changes

refreshed charmhelpers

This commit is contained in:
yolanda.robla@canonical.com 2014-01-24 14:05:44 +01:00
parent df17eabb36
commit e5df517c74
50 changed files with 131 additions and 2841 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Makefile Normal file
View File

@ -0,0 +1,10 @@
#!/usr/bin/make
PYTHON := /usr/bin/env python
lint:
@flake8 --exclude lib/charmhelpers hooks
@charm proof
sync:
@charm-helper-sync -c charm-helpers.yaml

1
charm-helpers.yaml
View File

@ -4,3 +4,4 @@ include:
- core
- contrib.charmsupport
- contrib.openstack
- contrib.unison

57
lib/charmhelpers/cli/README.rst
View File

@ -1,57 +0,0 @@
==========
Commandant
==========
-----------------------------------------------------
Automatic command-line interfaces to Python functions
-----------------------------------------------------
One of the benefits of ``libvirt`` is the uniformity of the interface: the C API (as well as the bindings in other languages) is a set of functions that accept parameters that are nearly identical to the command-line arguments. If you run ``virsh``, you get an interactive command prompt that supports all of the same commands that your shell scripts use as ``virsh`` subcommands.
Command execution and stdio manipulation is the greatest common factor across all development systems in the POSIX environment. By exposing your functions as commands that manipulate streams of text, you can make life easier for all the Ruby and Erlang and Go programmers in your life.
Goals
=====
* Single decorator to expose a function as a command.
* now two decorators - one "automatic" and one that allows authors to manipulate the arguments for fine-grained control.(MW)
* Automatic analysis of function signature through ``inspect.getargspec()``
* Command argument parser built automatically with ``argparse``
* Interactive interpreter loop object made with ``Cmd``
* Options to output structured return value data via ``pprint``, ``yaml`` or ``json`` dumps.
Other Important Features that need writing
------------------------------------------
* Help and Usage documentation can be automatically generated, but it will be important to let users override this behaviour
* The decorator should allow specifying further parameters to the parser's add_argument() calls, to specify types or to make arguments behave as boolean flags, etc.
- Filename arguments are important, as good practice is for functions to accept file objects as parameters.
- choices arguments help to limit bad input before the function is called
* Some automatic behaviour could make for better defaults, once the user can override them.
- We could automatically detect arguments that default to False or True, and automatically support --no-foo for foo=True.
- We could automatically support hyphens as alternates for underscores
- Arguments defaulting to sequence types could support the ``append`` action.
-----------------------------------------------------
Implementing subcommands
-----------------------------------------------------
(WIP)
So as to avoid dependencies on the cli module, subcommands should be defined separately from their implementations. The recommmendation would be to place definitions into separate modules near the implementations which they expose.
Some examples::
from charmhelpers.cli import CommandLine
from charmhelpers.payload import execd
from charmhelpers.foo import bar
cli = CommandLine()
cli.subcommand(execd.execd_run)
@cli.subcommand_builder("bar", help="Bar baz qux")
def barcmd_builder(subparser):
subparser.add_argument('argument1', help="yackety")
return bar

147
lib/charmhelpers/cli/__init__.py
View File

@ -1,147 +0,0 @@
import inspect
import itertools
import argparse
import sys
class OutputFormatter(object):
def __init__(self, outfile=sys.stdout):
self.formats = (
"raw",
"json",
"py",
"yaml",
"csv",
"tab",
)
self.outfile = outfile
def add_arguments(self, argument_parser):
formatgroup = argument_parser.add_mutually_exclusive_group()
choices = self.supported_formats
formatgroup.add_argument("--format", metavar='FMT',
help="Select output format for returned data, "
"where FMT is one of: {}".format(choices),
choices=choices, default='raw')
for fmt in self.formats:
fmtfunc = getattr(self, fmt)
formatgroup.add_argument("-{}".format(fmt[0]),
"--{}".format(fmt), action='store_const',
const=fmt, dest='format',
help=fmtfunc.__doc__)
@property
def supported_formats(self):
return self.formats
def raw(self, output):
"""Output data as raw string (default)"""
self.outfile.write(str(output))
def py(self, output):
"""Output data as a nicely-formatted python data structure"""
import pprint
pprint.pprint(output, stream=self.outfile)
def json(self, output):
"""Output data in JSON format"""
import json
json.dump(output, self.outfile)
def yaml(self, output):
"""Output data in YAML format"""
import yaml
yaml.safe_dump(output, self.outfile)
def csv(self, output):
"""Output data as excel-compatible CSV"""
import csv
csvwriter = csv.writer(self.outfile)
csvwriter.writerows(output)
def tab(self, output):
"""Output data in excel-compatible tab-delimited format"""
import csv
csvwriter = csv.writer(self.outfile, dialect=csv.excel_tab)
csvwriter.writerows(output)
def format_output(self, output, fmt='raw'):
fmtfunc = getattr(self, fmt)
fmtfunc(output)
class CommandLine(object):
argument_parser = None
subparsers = None
formatter = None
def __init__(self):
if not self.argument_parser:
self.argument_parser = argparse.ArgumentParser(description='Perform common charm tasks')
if not self.formatter:
self.formatter = OutputFormatter()
self.formatter.add_arguments(self.argument_parser)
if not self.subparsers:
self.subparsers = self.argument_parser.add_subparsers(help='Commands')
def subcommand(self, command_name=None):
"""
Decorate a function as a subcommand. Use its arguments as the
command-line arguments"""
def wrapper(decorated):
cmd_name = command_name or decorated.__name__
subparser = self.subparsers.add_parser(cmd_name,
description=decorated.__doc__)
for args, kwargs in describe_arguments(decorated):
subparser.add_argument(*args, **kwargs)
subparser.set_defaults(func=decorated)
return decorated
return wrapper
def subcommand_builder(self, command_name, description=None):
"""
Decorate a function that builds a subcommand. Builders should accept a
single argument (the subparser instance) and return the function to be
run as the command."""
def wrapper(decorated):
subparser = self.subparsers.add_parser(command_name)
func = decorated(subparser)
subparser.set_defaults(func=func)
subparser.description = description or func.__doc__
return wrapper
def run(self):
"Run cli, processing arguments and executing subcommands."
arguments = self.argument_parser.parse_args()
argspec = inspect.getargspec(arguments.func)
vargs = []
kwargs = {}
if argspec.varargs:
vargs = getattr(arguments, argspec.varargs)
for arg in argspec.args:
kwargs[arg] = getattr(arguments, arg)
self.formatter.format_output(arguments.func(*vargs, **kwargs), arguments.format)
cmdline = CommandLine()
def describe_arguments(func):
"""
Analyze a function's signature and return a data structure suitable for
passing in as arguments to an argparse parser's add_argument() method."""
argspec = inspect.getargspec(func)
# we should probably raise an exception somewhere if func includes **kwargs
if argspec.defaults:
positional_args = argspec.args[:-len(argspec.defaults)]
keyword_names = argspec.args[-len(argspec.defaults):]
for arg, default in itertools.izip(keyword_names, argspec.defaults):
yield ('--{}'.format(arg),), {'default': default}
else:
positional_args = argspec.args
for arg in positional_args:
yield (arg,), {}
if argspec.varargs:
yield (argspec.varargs,), {'nargs': '*'}

2
lib/charmhelpers/cli/commands.py
View File

@ -1,2 +0,0 @@
from . import CommandLine
import host

15
lib/charmhelpers/cli/host.py
View File

@ -1,15 +0,0 @@
from . import cmdline
from charmhelpers.core import host
@cmdline.subcommand()
def mounts():
"List mounts"
return host.mounts()
@cmdline.subcommand_builder('service', description="Control system services")
def service(subparser):
subparser.add_argument("action", help="The action to perform (start, stop, etc...)")
subparser.add_argument("service_name", help="Name of the service to control")
return host.service

165
lib/charmhelpers/contrib/ansible/__init__.py
View File

@ -1,165 +0,0 @@
# Copyright 2013 Canonical Ltd.
#
# Authors:
# Charm Helpers Developers <juju@lists.ubuntu.com>
"""Charm Helpers ansible - declare the state of your machines.
This helper enables you to declare your machine state, rather than
program it procedurally (and have to test each change to your procedures).
Your install hook can be as simple as:
{{{
import charmhelpers.contrib.ansible
def install():
charmhelpers.contrib.ansible.install_ansible_support()
charmhelpers.contrib.ansible.apply_playbook('playbooks/install.yaml')
}}}
and won't need to change (nor will its tests) when you change the machine
state.
All of your juju config and relation-data are available as template
variables within your playbooks and templates. An install playbook looks
something like:
{{{
---
- hosts: localhost
user: root
tasks:
- name: Add private repositories.
template:
src: ../templates/private-repositories.list.jinja2
dest: /etc/apt/sources.list.d/private.list
- name: Update the cache.
apt: update_cache=yes
- name: Install dependencies.
apt: pkg={{ item }}
with_items:
- python-mimeparse
- python-webob
- sunburnt
- name: Setup groups.
group: name={{ item.name }} gid={{ item.gid }}
with_items:
- { name: 'deploy_user', gid: 1800 }
- { name: 'service_user', gid: 1500 }
...
}}}
Read more online about playbooks[1] and standard ansible modules[2].
[1] http://www.ansibleworks.com/docs/playbooks.html
[2] http://www.ansibleworks.com/docs/modules.html
"""
import os
import subprocess
import charmhelpers.contrib.templating.contexts
import charmhelpers.core.host
import charmhelpers.core.hookenv
import charmhelpers.fetch
charm_dir = os.environ.get('CHARM_DIR', '')
ansible_hosts_path = '/etc/ansible/hosts'
# Ansible will automatically include any vars in the following
# file in its inventory when run locally.
ansible_vars_path = '/etc/ansible/host_vars/localhost'
def install_ansible_support(from_ppa=True):
"""Installs the ansible package.
By default it is installed from the PPA [1] linked from
the ansible website [2].
[1] https://launchpad.net/~rquillo/+archive/ansible
[2] http://www.ansibleworks.com/docs/gettingstarted.html#ubuntu-and-debian
If from_ppa is false, you must ensure that the package is available
from a configured repository.
"""
if from_ppa:
charmhelpers.fetch.add_source('ppa:rquillo/ansible')
charmhelpers.fetch.apt_update(fatal=True)
charmhelpers.fetch.apt_install('ansible')
with open(ansible_hosts_path, 'w+') as hosts_file:
hosts_file.write('localhost ansible_connection=local')
def apply_playbook(playbook, tags=None):
tags = tags or []
tags = ",".join(tags)
charmhelpers.contrib.templating.contexts.juju_state_to_yaml(
ansible_vars_path, namespace_separator='__',
allow_hyphens_in_keys=False)
call = [
'ansible-playbook',
'-c',
'local',
playbook,
]
if tags:
call.extend(['--tags', '{}'.format(tags)])
subprocess.check_call(call)
class AnsibleHooks(charmhelpers.core.hookenv.Hooks):
"""Run a playbook with the hook-name as the tag.
This helper builds on the standard hookenv.Hooks helper,
but additionally runs the playbook with the hook-name specified
using --tags (ie. running all the tasks tagged with the hook-name).
Example:
hooks = AnsibleHooks(playbook_path='playbooks/my_machine_state.yaml')
# All the tasks within my_machine_state.yaml tagged with 'install'
# will be run automatically after do_custom_work()
@hooks.hook()
def install():
do_custom_work()
# For most of your hooks, you won't need to do anything other
# than run the tagged tasks for the hook:
@hooks.hook('config-changed', 'start', 'stop')
def just_use_playbook():
pass
# As a convenience, you can avoid the above noop function by specifying
# the hooks which are handled by ansible-only and they'll be registered
# for you:
# hooks = AnsibleHooks(
# 'playbooks/my_machine_state.yaml',
# default_hooks=['config-changed', 'start', 'stop'])
if __name__ == "__main__":
# execute a hook based on the name the program is called by
hooks.execute(sys.argv)
"""
def __init__(self, playbook_path, default_hooks=None):
"""Register any hooks handled by ansible."""
super(AnsibleHooks, self).__init__()
self.playbook_path = playbook_path
default_hooks = default_hooks or []
noop = lambda *args, **kwargs: None
for hook in default_hooks:
self.register(hook, noop)
def execute(self, args):
"""Execute the hook followed by the playbook using the hook as tag."""
super(AnsibleHooks, self).execute(args)
hook_name = os.path.basename(args[0])
charmhelpers.contrib.ansible.apply_playbook(
self.playbook_path, tags=[hook_name])

4
lib/charmhelpers/contrib/charmhelpers/IMPORT
View File

@ -1,4 +0,0 @@
Source lp:charm-tools/trunk
charm-tools/helpers/python/charmhelpers/__init__.py -> charmhelpers/charmhelpers/contrib/charmhelpers/__init__.py
charm-tools/helpers/python/charmhelpers/tests/test_charmhelpers.py -> charmhelpers/tests/contrib/charmhelpers/test_charmhelpers.py

184
lib/charmhelpers/contrib/charmhelpers/__init__.py
View File

@ -1,184 +0,0 @@
# Copyright 2012 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
import warnings
warnings.warn("contrib.charmhelpers is deprecated", DeprecationWarning)
"""Helper functions for writing Juju charms in Python."""
__metaclass__ = type
__all__ = [
#'get_config', # core.hookenv.config()
#'log', # core.hookenv.log()
#'log_entry', # core.hookenv.log()
#'log_exit', # core.hookenv.log()
#'relation_get', # core.hookenv.relation_get()
#'relation_set', # core.hookenv.relation_set()
#'relation_ids', # core.hookenv.relation_ids()
#'relation_list', # core.hookenv.relation_units()
#'config_get', # core.hookenv.config()
#'unit_get', # core.hookenv.unit_get()
#'open_port', # core.hookenv.open_port()
#'close_port', # core.hookenv.close_port()
#'service_control', # core.host.service()
'unit_info', # client-side, NOT IMPLEMENTED
'wait_for_machine', # client-side, NOT IMPLEMENTED
'wait_for_page_contents', # client-side, NOT IMPLEMENTED
'wait_for_relation', # client-side, NOT IMPLEMENTED
'wait_for_unit', # client-side, NOT IMPLEMENTED
]
import operator
from shelltoolbox import (
command,
)
import tempfile
import time
import urllib2
import yaml
SLEEP_AMOUNT = 0.1
# We create a juju_status Command here because it makes testing much,
# much easier.
juju_status = lambda: command('juju')('status')
# re-implemented as charmhelpers.fetch.configure_sources()
#def configure_source(update=False):
# source = config_get('source')
# if ((source.startswith('ppa:') or
# source.startswith('cloud:') or
# source.startswith('http:'))):
# run('add-apt-repository', source)
# if source.startswith("http:"):
# run('apt-key', 'import', config_get('key'))
# if update:
# run('apt-get', 'update')
# DEPRECATED: client-side only
def make_charm_config_file(charm_config):
charm_config_file = tempfile.NamedTemporaryFile()
charm_config_file.write(yaml.dump(charm_config))
charm_config_file.flush()
# The NamedTemporaryFile instance is returned instead of just the name
# because we want to take advantage of garbage collection-triggered
# deletion of the temp file when it goes out of scope in the caller.
return charm_config_file
# DEPRECATED: client-side only
def unit_info(service_name, item_name, data=None, unit=None):
if data is None:
data = yaml.safe_load(juju_status())
service = data['services'].get(service_name)
if service is None:
# XXX 2012-02-08 gmb:
# This allows us to cope with the race condition that we
# have between deploying a service and having it come up in
# `juju status`. We could probably do with cleaning it up so
# that it fails a bit more noisily after a while.
return ''
units = service['units']
if unit is not None:
item = units[unit][item_name]
else:
# It might seem odd to sort the units here, but we do it to
# ensure that when no unit is specified, the first unit for the
# service (or at least the one with the lowest number) is the
# one whose data gets returned.
sorted_unit_names = sorted(units.keys())
item = units[sorted_unit_names[0]][item_name]
return item
# DEPRECATED: client-side only
def get_machine_data():
return yaml.safe_load(juju_status())['machines']
# DEPRECATED: client-side only
def wait_for_machine(num_machines=1, timeout=300):
"""Wait `timeout` seconds for `num_machines` machines to come up.
This wait_for... function can be called by other wait_for functions
whose timeouts might be too short in situations where only a bare
Juju setup has been bootstrapped.
:return: A tuple of (num_machines, time_taken). This is used for
testing.
"""
# You may think this is a hack, and you'd be right. The easiest way
# to tell what environment we're working in (LXC vs EC2) is to check
# the dns-name of the first machine. If it's localhost we're in LXC
# and we can just return here.
if get_machine_data()[0]['dns-name'] == 'localhost':
return 1, 0
start_time = time.time()
while True:
# Drop the first machine, since it's the Zookeeper and that's
# not a machine that we need to wait for. This will only work
# for EC2 environments, which is why we return early above if
# we're in LXC.
machine_data = get_machine_data()
non_zookeeper_machines = [
machine_data[key] for key in machine_data.keys()[1:]]
if len(non_zookeeper_machines) >= num_machines:
all_machines_running = True
for machine in non_zookeeper_machines:
if machine.get('instance-state') != 'running':
all_machines_running = False
break
if all_machines_running:
break
if time.time() - start_time >= timeout:
raise RuntimeError('timeout waiting for service to start')
time.sleep(SLEEP_AMOUNT)
return num_machines, time.time() - start_time
# DEPRECATED: client-side only
def wait_for_unit(service_name, timeout=480):
"""Wait `timeout` seconds for a given service name to come up."""
wait_for_machine(num_machines=1)
start_time = time.time()
while True:
state = unit_info(service_name, 'agent-state')
if 'error' in state or state == 'started':
break
if time.time() - start_time >= timeout:
raise RuntimeError('timeout waiting for service to start')
time.sleep(SLEEP_AMOUNT)
if state != 'started':
raise RuntimeError('unit did not start, agent-state: ' + state)
# DEPRECATED: client-side only
def wait_for_relation(service_name, relation_name, timeout=120):
"""Wait `timeout` seconds for a given relation to come up."""
start_time = time.time()
while True:
relation = unit_info(service_name, 'relations').get(relation_name)
if relation is not None and relation['state'] == 'up':
break
if time.time() - start_time >= timeout:
raise RuntimeError('timeout waiting for relation to be up')
time.sleep(SLEEP_AMOUNT)
# DEPRECATED: client-side only
def wait_for_page_contents(url, contents, timeout=120, validate=None):
if validate is None:
validate = operator.contains
start_time = time.time()
while True:
try:
stream = urllib2.urlopen(url)
except (urllib2.HTTPError, urllib2.URLError):
pass
else:
page = stream.read()
if validate(page, contents):
return page
if time.time() - start_time >= timeout:
raise RuntimeError('timeout waiting for contents of ' + url)
time.sleep(SLEEP_AMOUNT)

14
lib/charmhelpers/contrib/charmsupport/IMPORT Normal file
View File

@ -0,0 +1,14 @@
Source: lp:charmsupport/trunk
charmsupport/charmsupport/execd.py -> charm-helpers/charmhelpers/contrib/charmsupport/execd.py
charmsupport/charmsupport/hookenv.py -> charm-helpers/charmhelpers/contrib/charmsupport/hookenv.py
charmsupport/charmsupport/host.py -> charm-helpers/charmhelpers/contrib/charmsupport/host.py
charmsupport/charmsupport/nrpe.py -> charm-helpers/charmhelpers/contrib/charmsupport/nrpe.py
charmsupport/charmsupport/volumes.py -> charm-helpers/charmhelpers/contrib/charmsupport/volumes.py
charmsupport/tests/test_execd.py -> charm-helpers/tests/contrib/charmsupport/test_execd.py
charmsupport/tests/test_hookenv.py -> charm-helpers/tests/contrib/charmsupport/test_hookenv.py
charmsupport/tests/test_host.py -> charm-helpers/tests/contrib/charmsupport/test_host.py
charmsupport/tests/test_nrpe.py -> charm-helpers/tests/contrib/charmsupport/test_nrpe.py
charmsupport/bin/charmsupport -> charm-helpers/bin/contrib/charmsupport/charmsupport

BIN
lib/charmhelpers/contrib/charmsupport/__init__.pyc Normal file
View File

Binary file not shown.

BIN
lib/charmhelpers/contrib/charmsupport/nrpe.pyc Normal file
View File

Binary file not shown.

0
lib/charmhelpers/contrib/hahelpers/__init__.py
View File

58
lib/charmhelpers/contrib/hahelpers/apache.py
View File

@ -1,58 +0,0 @@
#
# Copyright 2012 Canonical Ltd.
#
# This file is sourced from lp:openstack-charm-helpers
#
# Authors:
# James Page <james.page@ubuntu.com>
# Adam Gandelman <adamg@ubuntu.com>
#
import subprocess
from charmhelpers.core.hookenv import (
config as config_get,
relation_get,
relation_ids,
related_units as relation_list,
log,
INFO,
)
def get_cert():
cert = config_get('ssl_cert')
key = config_get('ssl_key')
if not (cert and key):
log("Inspecting identity-service relations for SSL certificate.",
level=INFO)
cert = key = None
for r_id in relation_ids('identity-service'):
for unit in relation_list(r_id):
if not cert:
cert = relation_get('ssl_cert',
rid=r_id, unit=unit)
if not key:
key = relation_get('ssl_key',
rid=r_id, unit=unit)
return (cert, key)
def get_ca_cert():
ca_cert = None
log("Inspecting identity-service relations for CA SSL certificate.",
level=INFO)
for r_id in relation_ids('identity-service'):
for unit in relation_list(r_id):
if not ca_cert:
ca_cert = relation_get('ca_cert',
rid=r_id, unit=unit)
return ca_cert
def install_ca_cert(ca_cert):
if ca_cert:
with open('/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt',
'w') as crt:
crt.write(ca_cert)
subprocess.check_call(['update-ca-certificates', '--fresh'])

183
lib/charmhelpers/contrib/hahelpers/cluster.py
View File

@ -1,183 +0,0 @@
#
# Copyright 2012 Canonical Ltd.
#
# Authors:
# James Page <james.page@ubuntu.com>
# Adam Gandelman <adamg@ubuntu.com>
#
import subprocess
import os
from socket import gethostname as get_unit_hostname
from charmhelpers.core.hookenv import (
log,
relation_ids,
related_units as relation_list,
relation_get,
config as config_get,
INFO,
ERROR,
unit_get,
)
class HAIncompleteConfig(Exception):
pass
def is_clustered():
for r_id in (relation_ids('ha') or []):
for unit in (relation_list(r_id) or []):
clustered = relation_get('clustered',
rid=r_id,
unit=unit)
if clustered:
return True
return False
def is_leader(resource):
cmd = [
"crm", "resource",
"show", resource
]
try:
status = subprocess.check_output(cmd)
except subprocess.CalledProcessError:
return False
else:
if get_unit_hostname() in status:
return True
else:
return False
def peer_units():
peers = []
for r_id in (relation_ids('cluster') or []):
for unit in (relation_list(r_id) or []):
peers.append(unit)
return peers
def oldest_peer(peers):
local_unit_no = int(os.getenv('JUJU_UNIT_NAME').split('/')[1])
for peer in peers:
remote_unit_no = int(peer.split('/')[1])
if remote_unit_no < local_unit_no:
return False
return True
def eligible_leader(resource):
if is_clustered():
if not is_leader(resource):
log('Deferring action to CRM leader.', level=INFO)
return False
else:
peers = peer_units()
if peers and not oldest_peer(peers):
log('Deferring action to oldest service unit.', level=INFO)
return False
return True
def https():
'''
Determines whether enough data has been provided in configuration
or relation data to configure HTTPS
.
returns: boolean
'''
if config_get('use-https') == "yes":
return True
if config_get('ssl_cert') and config_get('ssl_key'):
return True
for r_id in relation_ids('identity-service'):
for unit in relation_list(r_id):
rel_state = [
relation_get('https_keystone', rid=r_id, unit=unit),
relation_get('ssl_cert', rid=r_id, unit=unit),
relation_get('ssl_key', rid=r_id, unit=unit),
relation_get('ca_cert', rid=r_id, unit=unit),
]
# NOTE: works around (LP: #1203241)
if (None not in rel_state) and ('' not in rel_state):
return True
return False
def determine_api_port(public_port):
'''
Determine correct API server listening port based on
existence of HTTPS reverse proxy and/or haproxy.
public_port: int: standard public port for given service
returns: int: the correct listening port for the API service
'''
i = 0
if len(peer_units()) > 0 or is_clustered():
i += 1
if https():
i += 1
return public_port - (i * 10)
def determine_haproxy_port(public_port):
'''
Description: Determine correct proxy listening port based on public IP +
existence of HTTPS reverse proxy.
public_port: int: standard public port for given service
returns: int: the correct listening port for the HAProxy service
'''
i = 0
if https():
i += 1
return public_port - (i * 10)
def get_hacluster_config():
'''
Obtains all relevant configuration from charm configuration required
for initiating a relation to hacluster:
ha-bindiface, ha-mcastport, vip, vip_iface, vip_cidr
returns: dict: A dict containing settings keyed by setting name.
raises: HAIncompleteConfig if settings are missing.
'''
settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'vip_iface', 'vip_cidr']
conf = {}
for setting in settings:
conf[setting] = config_get(setting)
missing = []
[missing.append(s) for s, v in conf.iteritems() if v is None]
if missing:
log('Insufficient config data to configure hacluster.', level=ERROR)
raise HAIncompleteConfig
return conf
def canonical_url(configs, vip_setting='vip'):
'''
Returns the correct HTTP URL to this host given the state of HTTPS
configuration and hacluster.
:configs : OSTemplateRenderer: A config tempating object to inspect for
a complete https context.
:vip_setting: str: Setting in charm config that specifies
VIP address.
'''
scheme = 'http'
if 'https' in configs.complete_contexts():
scheme = 'https'
if is_clustered():
addr = config_get(vip_setting)
else:
addr = unit_get('private-address')
return '%s://%s' % (scheme, addr)

4
lib/charmhelpers/contrib/jujugui/IMPORT
View File

@ -1,4 +0,0 @@
Source: lp:charms/juju-gui
juju-gui/hooks/utils.py -> charm-helpers/charmhelpers/contrib/jujugui/utils.py
juju-gui/tests/test_utils.py -> charm-helpers/tests/contrib/jujugui/test_utils.py

0
lib/charmhelpers/contrib/jujugui/__init__.py
View File

602
lib/charmhelpers/contrib/jujugui/utils.py
View File

@ -1,602 +0,0 @@
"""Juju GUI charm utilities."""
__all__ = [
'AGENT',
'APACHE',
'API_PORT',
'CURRENT_DIR',
'HAPROXY',
'IMPROV',
'JUJU_DIR',
'JUJU_GUI_DIR',
'JUJU_GUI_SITE',
'JUJU_PEM',
'WEB_PORT',
'bzr_checkout',
'chain',
'cmd_log',
'fetch_api',
'fetch_gui',
'find_missing_packages',
'first_path_in_dir',
'get_api_address',
'get_npm_cache_archive_url',
'get_release_file_url',
'get_staging_dependencies',
'get_zookeeper_address',
'legacy_juju',
'log_hook',
'merge',
'parse_source',
'prime_npm_cache',
'render_to_file',
'save_or_create_certificates',
'setup_apache',
'setup_gui',
'start_agent',
'start_gui',
'start_improv',
'write_apache_config',
]
from contextlib import contextmanager
import errno
import json
import os
import logging
import shutil
from subprocess import CalledProcessError
import tempfile
from urlparse import urlparse
import apt
import tempita
from launchpadlib.launchpad import Launchpad
from shelltoolbox import (
Serializer,
apt_get_install,
command,
environ,
install_extra_repositories,
run,
script_name,
search_file,
su,
)
from charmhelpers.core.host import (
service_start,
)
from charmhelpers.core.hookenv import (
log,
config,
unit_get,
)
AGENT = 'juju-api-agent'
APACHE = 'apache2'
IMPROV = 'juju-api-improv'
HAPROXY = 'haproxy'
API_PORT = 8080
WEB_PORT = 8000
CURRENT_DIR = os.getcwd()
JUJU_DIR = os.path.join(CURRENT_DIR, 'juju')
JUJU_GUI_DIR = os.path.join(CURRENT_DIR, 'juju-gui')
JUJU_GUI_SITE = '/etc/apache2/sites-available/juju-gui'
JUJU_GUI_PORTS = '/etc/apache2/ports.conf'
JUJU_PEM = 'juju.includes-private-key.pem'
BUILD_REPOSITORIES = ('ppa:chris-lea/node.js-legacy',)
DEB_BUILD_DEPENDENCIES = (
'bzr', 'imagemagick', 'make', 'nodejs', 'npm',
)
DEB_STAGE_DEPENDENCIES = (
'zookeeper',
)
# Store the configuration from on invocation to the next.
config_json = Serializer('/tmp/config.json')
# Bazaar checkout command.
bzr_checkout = command('bzr', 'co', '--lightweight')
# Whether or not the charm is deployed using juju-core.
# If juju-core has been used to deploy the charm, an agent.conf file must
# be present in the charm parent directory.
legacy_juju = lambda: not os.path.exists(
os.path.join(CURRENT_DIR, '..', 'agent.conf'))
def _get_build_dependencies():
"""Install deb dependencies for building."""
log('Installing build dependencies.')
cmd_log(install_extra_repositories(*BUILD_REPOSITORIES))
cmd_log(apt_get_install(*DEB_BUILD_DEPENDENCIES))
def get_api_address(unit_dir):
"""Return the Juju API address stored in the uniter agent.conf file."""
import yaml # python-yaml is only installed if juju-core is used.
# XXX 2013-03-27 frankban bug=1161443:
# currently the uniter agent.conf file does not include the API
# address. For now retrieve it from the machine agent file.
base_dir = os.path.abspath(os.path.join(unit_dir, '..'))
for dirname in os.listdir(base_dir):
if dirname.startswith('machine-'):
agent_conf = os.path.join(base_dir, dirname, 'agent.conf')
break
else:
raise IOError('Juju agent configuration file not found.')
contents = yaml.load(open(agent_conf))
return contents['apiinfo']['addrs'][0]
def get_staging_dependencies():
"""Install deb dependencies for the stage (improv) environment."""
log('Installing stage dependencies.')
cmd_log(apt_get_install(*DEB_STAGE_DEPENDENCIES))
def first_path_in_dir(directory):
"""Return the full path of the first file/dir in *directory*."""
return os.path.join(directory, os.listdir(directory)[0])
def _get_by_attr(collection, attr, value):
"""Return the first item in collection having attr == value.
Return None if the item is not found.
"""
for item in collection:
if getattr(item, attr) == value:
return item
def get_release_file_url(project, series_name, release_version):
"""Return the URL of the release file hosted in Launchpad.
The returned URL points to a release file for the given project, series
name and release version.
The argument *project* is a project object as returned by launchpadlib.
The arguments *series_name* and *release_version* are strings. If
*release_version* is None, the URL of the latest release will be returned.
"""
series = _get_by_attr(project.series, 'name', series_name)
if series is None:
raise ValueError('%r: series not found' % series_name)
# Releases are returned by Launchpad in reverse date order.
releases = list(series.releases)
if not releases:
raise ValueError('%r: series does not contain releases' % series_name)
if release_version is not None:
release = _get_by_attr(releases, 'version', release_version)
if release is None:
raise ValueError('%r: release not found' % release_version)
releases = [release]
for release in releases:
for file_ in release.files:
if str(file_).endswith('.tgz'):
return file_.file_link
raise ValueError('%r: file not found' % release_version)
def get_zookeeper_address(agent_file_path):
"""Retrieve the Zookeeper address contained in the given *agent_file_path*.
The *agent_file_path* is a path to a file containing a line similar to the
following::
env JUJU_ZOOKEEPER="address"
"""
line = search_file('JUJU_ZOOKEEPER', agent_file_path).strip()
return line.split('=')[1].strip('"')
@contextmanager
def log_hook():
"""Log when a hook starts and stops its execution.
Also log to stdout possible CalledProcessError exceptions raised executing
the hook.
"""
script = script_name()
log(">>> Entering {}".format(script))
try:
yield
except CalledProcessError as err:
log('Exception caught:')
log(err.output)
raise
finally:
log("<<< Exiting {}".format(script))
def parse_source(source):
"""Parse the ``juju-gui-source`` option.
Return a tuple of two elements representing info on how to deploy Juju GUI.
Examples:
- ('stable', None): latest stable release;
- ('stable', '0.1.0'): stable release v0.1.0;
- ('trunk', None): latest trunk release;
- ('trunk', '0.1.0+build.1'): trunk release v0.1.0 bzr revision 1;
- ('branch', 'lp:juju-gui'): release is made from a branch;
- ('url', 'http://example.com/gui'): release from a downloaded file.
"""
if source.startswith('url:'):
source = source[4:]
# Support file paths, including relative paths.
if urlparse(source).scheme == '':
if not source.startswith('/'):
source = os.path.join(os.path.abspath(CURRENT_DIR), source)
source = "file://%s" % source
return 'url', source
if source in ('stable', 'trunk'):
return source, None
if source.startswith('lp:') or source.startswith('http://'):
return 'branch', source
if 'build' in source:
return 'trunk', source
return 'stable', source
def render_to_file(template_name, context, destination):
"""Render the given *template_name* into *destination* using *context*.
The tempita template language is used to render contents
(see http://pythonpaste.org/tempita/).
The argument *template_name* is the name or path of the template file:
it may be either a path relative to ``../config`` or an absolute path.
The argument *destination* is a file path.
The argument *context* is a dict-like object.
"""
template_path = os.path.abspath(template_name)
template = tempita.Template.from_filename(template_path)
with open(destination, 'w') as stream:
stream.write(template.substitute(context))
results_log = None
def _setupLogging():
global results_log
if results_log is not None:
return
cfg = config()
logging.basicConfig(
filename=cfg['command-log-file'],
level=logging.INFO,
format="%(asctime)s: %(name)s@%(levelname)s %(message)s")
results_log = logging.getLogger('juju-gui')
def cmd_log(results):
global results_log
if not results:
return
if results_log is None:
_setupLogging()
# Since 'results' may be multi-line output, start it on a separate line
# from the logger timestamp, etc.
results_log.info('\n' + results)
def start_improv(staging_env, ssl_cert_path,
config_path='/etc/init/juju-api-improv.conf'):
"""Start a simulated juju environment using ``improv.py``."""
log('Setting up staging start up script.')
context = {
'juju_dir': JUJU_DIR,
'keys': ssl_cert_path,
'port': API_PORT,
'staging_env': staging_env,
}
render_to_file('config/juju-api-improv.conf.template', context, config_path)
log('Starting the staging backend.')
with su('root'):
service_start(IMPROV)
def start_agent(
ssl_cert_path, config_path='/etc/init/juju-api-agent.conf',
read_only=False):
"""Start the Juju agent and connect to the current environment."""
# Retrieve the Zookeeper address from the start up script.
unit_dir = os.path.realpath(os.path.join(CURRENT_DIR, '..'))
agent_file = '/etc/init/juju-{0}.conf'.format(os.path.basename(unit_dir))
zookeeper = get_zookeeper_address(agent_file)
log('Setting up API agent start up script.')
context = {
'juju_dir': JUJU_DIR,
'keys': ssl_cert_path,
'port': API_PORT,
'zookeeper': zookeeper,
'read_only': read_only
}
render_to_file('config/juju-api-agent.conf.template', context, config_path)
log('Starting API agent.')
with su('root'):
service_start(AGENT)
def start_gui(
console_enabled, login_help, readonly, in_staging, ssl_cert_path,
charmworld_url, serve_tests, haproxy_path='/etc/haproxy/haproxy.cfg',
config_js_path=None, secure=True, sandbox=False):
"""Set up and start the Juju GUI server."""
with su('root'):
run('chown', '-R', 'ubuntu:', JUJU_GUI_DIR)
# XXX 2013-02-05 frankban bug=1116320:
# External insecure resources are still loaded when testing in the
# debug environment. For now, switch to the production environment if
# the charm is configured to serve tests.
if in_staging and not serve_tests:
build_dirname = 'build-debug'
else:
build_dirname = 'build-prod'
build_dir = os.path.join(JUJU_GUI_DIR, build_dirname)
log('Generating the Juju GUI configuration file.')
is_legacy_juju = legacy_juju()
user, password = None, None
if (is_legacy_juju and in_staging) or sandbox:
user, password = 'admin', 'admin'
else:
user, password = None, None
api_backend = 'python' if is_legacy_juju else 'go'
if secure:
protocol = 'wss'
else:
log('Running in insecure mode! Port 80 will serve unencrypted.')
protocol = 'ws'
context = {
'raw_protocol': protocol,
'address': unit_get('public-address'),
'console_enabled': json.dumps(console_enabled),
'login_help': json.dumps(login_help),
'password': json.dumps(password),
'api_backend': json.dumps(api_backend),
'readonly': json.dumps(readonly),
'user': json.dumps(user),
'protocol': json.dumps(protocol),
'sandbox': json.dumps(sandbox),
'charmworld_url': json.dumps(charmworld_url),
}
if config_js_path is None:
config_js_path = os.path.join(
build_dir, 'juju-ui', 'assets', 'config.js')
render_to_file('config/config.js.template', context, config_js_path)
write_apache_config(build_dir, serve_tests)
log('Generating haproxy configuration file.')
if is_legacy_juju:
# The PyJuju API agent is listening on localhost.
api_address = '127.0.0.1:{0}'.format(API_PORT)
else:
# Retrieve the juju-core API server address.
api_address = get_api_address(os.path.join(CURRENT_DIR, '..'))
context = {
'api_address': api_address,
'api_pem': JUJU_PEM,
'legacy_juju': is_legacy_juju,
'ssl_cert_path': ssl_cert_path,
# In PyJuju environments, use the same certificate for both HTTPS and
# WebSocket connections. In juju-core the system already has the proper
# certificate installed.
'web_pem': JUJU_PEM,
'web_port': WEB_PORT,
'secure': secure
}
render_to_file('config/haproxy.cfg.template', context, haproxy_path)
log('Starting Juju GUI.')
def write_apache_config(build_dir, serve_tests=False):
log('Generating the apache site configuration file.')
context = {
'port': WEB_PORT,
'serve_tests': serve_tests,
'server_root': build_dir,
'tests_root': os.path.join(JUJU_GUI_DIR, 'test', ''),
}
render_to_file('config/apache-ports.template', context, JUJU_GUI_PORTS)
render_to_file('config/apache-site.template', context, JUJU_GUI_SITE)
def get_npm_cache_archive_url(Launchpad=Launchpad):
"""Figure out the URL of the most recent NPM cache archive on Launchpad."""
launchpad = Launchpad.login_anonymously('Juju GUI charm', 'production')
project = launchpad.projects['juju-gui']
# Find the URL of the most recently created NPM cache archive.
npm_cache_url = get_release_file_url(project, 'npm-cache', None)
return npm_cache_url
def prime_npm_cache(npm_cache_url):
"""Download NPM cache archive and prime the NPM cache with it."""
# Download the cache archive and then uncompress it into the NPM cache.
npm_cache_archive = os.path.join(CURRENT_DIR, 'npm-cache.tgz')
cmd_log(run('curl', '-L', '-o', npm_cache_archive, npm_cache_url))
npm_cache_dir = os.path.expanduser('~/.npm')
# The NPM cache directory probably does not exist, so make it if not.
try:
os.mkdir(npm_cache_dir)
except OSError, e:
# If the directory already exists then ignore the error.
if e.errno != errno.EEXIST: # File exists.
raise
uncompress = command('tar', '-x', '-z', '-C', npm_cache_dir, '-f')
cmd_log(uncompress(npm_cache_archive))
def fetch_gui(juju_gui_source, logpath):
"""Retrieve the Juju GUI release/branch."""
# Retrieve a Juju GUI release.
origin, version_or_branch = parse_source(juju_gui_source)
if origin == 'branch':
# Make sure we have the dependencies necessary for us to actually make
# a build.
_get_build_dependencies()
# Create a release starting from a branch.
juju_gui_source_dir = os.path.join(CURRENT_DIR, 'juju-gui-source')
log('Retrieving Juju GUI source checkout from %s.' % version_or_branch)
cmd_log(run('rm', '-rf', juju_gui_source_dir))
cmd_log(bzr_checkout(version_or_branch, juju_gui_source_dir))
log('Preparing a Juju GUI release.')
logdir = os.path.dirname(logpath)
fd, name = tempfile.mkstemp(prefix='make-distfile-', dir=logdir)
log('Output from "make distfile" sent to %s' % name)
with environ(NO_BZR='1'):
run('make', '-C', juju_gui_source_dir, 'distfile',
stdout=fd, stderr=fd)
release_tarball = first_path_in_dir(
os.path.join(juju_gui_source_dir, 'releases'))
else:
log('Retrieving Juju GUI release.')
if origin == 'url':
file_url = version_or_branch
else:
# Retrieve a release from Launchpad.
launchpad = Launchpad.login_anonymously(
'Juju GUI charm', 'production')
project = launchpad.projects['juju-gui']
file_url = get_release_file_url(project, origin, version_or_branch)
log('Downloading release file from %s.' % file_url)
release_tarball = os.path.join(CURRENT_DIR, 'release.tgz')
cmd_log(run('curl', '-L', '-o', release_tarball, file_url))
return release_tarball
def fetch_api(juju_api_branch):
"""Retrieve the Juju branch."""
# Retrieve Juju API source checkout.
log('Retrieving Juju API source checkout.')
cmd_log(run('rm', '-rf', JUJU_DIR))
cmd_log(bzr_checkout(juju_api_branch, JUJU_DIR))
def setup_gui(release_tarball):
"""Set up Juju GUI."""
# Uncompress the release tarball.
log('Installing Juju GUI.')
release_dir = os.path.join(CURRENT_DIR, 'release')
cmd_log(run('rm', '-rf', release_dir))
os.mkdir(release_dir)
uncompress = command('tar', '-x', '-z', '-C', release_dir, '-f')
cmd_log(uncompress(release_tarball))
# Link the Juju GUI dir to the contents of the release tarball.
cmd_log(run('ln', '-sf', first_path_in_dir(release_dir), JUJU_GUI_DIR))
def setup_apache():
"""Set up apache."""
log('Setting up apache.')
if not os.path.exists(JUJU_GUI_SITE):
cmd_log(run('touch', JUJU_GUI_SITE))
cmd_log(run('chown', 'ubuntu:', JUJU_GUI_SITE))
cmd_log(
run('ln', '-s', JUJU_GUI_SITE,
'/etc/apache2/sites-enabled/juju-gui'))
if not os.path.exists(JUJU_GUI_PORTS):
cmd_log(run('touch', JUJU_GUI_PORTS))
cmd_log(run('chown', 'ubuntu:', JUJU_GUI_PORTS))
with su('root'):
run('a2dissite', 'default')
run('a2ensite', 'juju-gui')
def save_or_create_certificates(
ssl_cert_path, ssl_cert_contents, ssl_key_contents):
"""Generate the SSL certificates.
If both *ssl_cert_contents* and *ssl_key_contents* are provided, use them
as certificates; otherwise, generate them.
Also create a pem file, suitable for use in the haproxy configuration,
concatenating the key and the certificate files.
"""
crt_path = os.path.join(ssl_cert_path, 'juju.crt')
key_path = os.path.join(ssl_cert_path, 'juju.key')
if not os.path.exists(ssl_cert_path):
os.makedirs(ssl_cert_path)
if ssl_cert_contents and ssl_key_contents:
# Save the provided certificates.
with open(crt_path, 'w') as cert_file:
cert_file.write(ssl_cert_contents)
with open(key_path, 'w') as key_file:
key_file.write(ssl_key_contents)
else:
# Generate certificates.
# See http://superuser.com/questions/226192/openssl-without-prompt
cmd_log(run(
'openssl', 'req', '-new', '-newkey', 'rsa:4096',
'-days', '365', '-nodes', '-x509', '-subj',
# These are arbitrary test values for the certificate.
'/C=GB/ST=Juju/L=GUI/O=Ubuntu/CN=juju.ubuntu.com',
'-keyout', key_path, '-out', crt_path))
# Generate the pem file.
pem_path = os.path.join(ssl_cert_path, JUJU_PEM)
if os.path.exists(pem_path):
os.remove(pem_path)
with open(pem_path, 'w') as pem_file:
shutil.copyfileobj(open(key_path), pem_file)
shutil.copyfileobj(open(crt_path), pem_file)
def find_missing_packages(*packages):
"""Given a list of packages, return the packages which are not installed.
"""
cache = apt.Cache()
missing = set()
for pkg_name in packages:
try:
pkg = cache[pkg_name]
except KeyError:
missing.add(pkg_name)
continue
if pkg.is_installed:
continue
missing.add(pkg_name)
return missing
## Backend support decorators
def chain(name):
"""Helper method to compose a set of mixin objects into a callable.
Each method is called in the context of its mixin instance, and its
argument is the Backend instance.
"""
# Chain method calls through all implementing mixins.
def method(self):
for mixin in self.mixins:
a_callable = getattr(type(mixin), name, None)
if a_callable:
a_callable(mixin, self)
method.__name__ = name
return method
def merge(name):
"""Helper to merge a property from a set of strategy objects
into a unified set.
"""
# Return merged property from every providing mixin as a set.
@property
def method(self):
result = set()
for mixin in self.mixins:
segment = getattr(type(mixin), name, None)
if segment and isinstance(segment, (list, tuple, set)):
result |= set(segment)
return result
return method

0
lib/charmhelpers/contrib/network/__init__.py
View File

75
lib/charmhelpers/contrib/network/ovs/__init__.py
View File

@ -1,75 +0,0 @@
''' Helpers for interacting with OpenvSwitch '''
import subprocess
import os
from charmhelpers.core.hookenv import (
log, WARNING
)
from charmhelpers.core.host import (
service
)
def add_bridge(name):
''' Add the named bridge to openvswitch '''
log('Creating bridge {}'.format(name))
subprocess.check_call(["ovs-vsctl", "--", "--may-exist", "add-br", name])
def del_bridge(name):
''' Delete the named bridge from openvswitch '''
log('Deleting bridge {}'.format(name))
subprocess.check_call(["ovs-vsctl", "--", "--if-exists", "del-br", name])
def add_bridge_port(name, port):
''' Add a port to the named openvswitch bridge '''
log('Adding port {} to bridge {}'.format(port, name))
subprocess.check_call(["ovs-vsctl", "--", "--may-exist", "add-port",
name, port])
subprocess.check_call(["ip", "link", "set", port, "up"])
def del_bridge_port(name, port):
''' Delete a port from the named openvswitch bridge '''
log('Deleting port {} from bridge {}'.format(port, name))
subprocess.check_call(["ovs-vsctl", "--", "--if-exists", "del-port",
name, port])
subprocess.check_call(["ip", "link", "set", port, "down"])
def set_manager(manager):
''' Set the controller for the local openvswitch '''
log('Setting manager for local ovs to {}'.format(manager))
subprocess.check_call(['ovs-vsctl', 'set-manager',
'ssl:{}'.format(manager)])
CERT_PATH = '/etc/openvswitch/ovsclient-cert.pem'
def get_certificate():
''' Read openvswitch certificate from disk '''
if os.path.exists(CERT_PATH):
log('Reading ovs certificate from {}'.format(CERT_PATH))
with open(CERT_PATH, 'r') as cert:
full_cert = cert.read()
begin_marker = "-----BEGIN CERTIFICATE-----"
end_marker = "-----END CERTIFICATE-----"
begin_index = full_cert.find(begin_marker)
end_index = full_cert.rfind(end_marker)
if end_index == -1 or begin_index == -1:
raise RuntimeError("Certificate does not contain valid begin"
" and end markers.")
full_cert = full_cert[begin_index:(end_index + len(end_marker))]
return full_cert
else:
log('Certificate not found', level=WARNING)
return None
def full_restart():
''' Full restart and reload of openvswitch '''
if os.path.exists('/etc/init/openvswitch-force-reload-kmod.conf'):
service('start', 'openvswitch-force-reload-kmod')
else:
service('force-reload-kmod', 'openvswitch-switch')

BIN
lib/charmhelpers/contrib/openstack/__init__.pyc Normal file
View File

Binary file not shown.

BIN
lib/charmhelpers/contrib/openstack/alternatives.pyc Normal file
View File

Binary file not shown.

BIN
lib/charmhelpers/contrib/openstack/context.pyc Normal file
View File

Binary file not shown.

BIN
lib/charmhelpers/contrib/openstack/neutron.pyc Normal file
View File

Binary file not shown.

11
lib/charmhelpers/contrib/openstack/templates/ceph.conf Normal file
View File

@ -0,0 +1,11 @@
###############################################################################
# [ WARNING ]
# cinder configuration file maintained by Juju
# local changes may be overwritten.
###############################################################################
{% if auth -%}
[global]
auth_supported = {{ auth }}
keyring = /etc/ceph/$cluster.$name.keyring
mon host = {{ mon_hosts }}
{% endif -%}

37
lib/charmhelpers/contrib/openstack/templates/haproxy.cfg Normal file
View File

@ -0,0 +1,37 @@
global
log 127.0.0.1 local0
log 127.0.0.1 local1 notice
maxconn 20000
user haproxy
group haproxy
spread-checks 0
defaults
log global
mode http
option httplog
option dontlognull
retries 3
timeout queue 1000
timeout connect 1000
timeout client 30000
timeout server 30000
listen stats :8888
mode http
stats enable
stats hide-version
stats realm Haproxy\ Statistics
stats uri /
stats auth admin:password
{% if units -%}
{% for service, ports in service_ports.iteritems() -%}
listen {{ service }} 0.0.0.0:{{ ports[0] }}
balance roundrobin
option tcplog
{% for unit, address in units.iteritems() -%}
server {{ unit }} {{ address }}:{{ ports[1] }} check
{% endfor %}
{% endfor -%}
{% endif -%}

23
lib/charmhelpers/contrib/openstack/templates/openstack_https_frontend Normal file
View File

@ -0,0 +1,23 @@
{% if endpoints -%}
{% for ext, int in endpoints -%}
Listen {{ ext }}
NameVirtualHost *:{{ ext }}
<VirtualHost *:{{ ext }}>
ServerName {{ private_address }}
SSLEngine on
SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert
SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key
ProxyPass / http://localhost:{{ int }}/
ProxyPassReverse / http://localhost:{{ int }}/
ProxyPreserveHost on
</VirtualHost>
<Proxy *>
Order deny,allow
Allow from all
</Proxy>
<Location />
Order allow,deny
Allow from all
</Location>
{% endfor -%}
{% endif -%}

1
lib/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf Symbolic link
View File

@ -0,0 +1 @@
openstack_https_frontend

BIN
lib/charmhelpers/contrib/openstack/templating.pyc Normal file
View File

Binary file not shown.

14
lib/charmhelpers/contrib/openstack/utils.py
View File

@ -415,7 +415,7 @@ def get_host_ip(hostname):
return ns_query(hostname)
def get_hostname(address):
def get_hostname(address, fqdn=True):
"""
Resolves hostname for given IP, or returns the input
if it is already a hostname.
@ -434,7 +434,11 @@ def get_hostname(address):
if not result:
return None
# strip trailing .
if result.endswith('.'):
return result[:-1]
return result
if fqdn:
# strip trailing .
if result.endswith('.'):
return result[:-1]
else:
return result
else:
return result.split('.')[0]

BIN
lib/charmhelpers/contrib/openstack/utils.pyc Normal file
View File

Binary file not shown.

102
lib/charmhelpers/contrib/saltstack/__init__.py
View File

@ -1,102 +0,0 @@
"""Charm Helpers saltstack - declare the state of your machines.
This helper enables you to declare your machine state, rather than
program it procedurally (and have to test each change to your procedures).
Your install hook can be as simple as:
{{{
from charmhelpers.contrib.saltstack import (
install_salt_support,
update_machine_state,
)
def install():
install_salt_support()
update_machine_state('machine_states/dependencies.yaml')
update_machine_state('machine_states/installed.yaml')
}}}
and won't need to change (nor will its tests) when you change the machine
state.
It's using a python package called salt-minion which allows various formats for
specifying resources, such as:
{{{
/srv/{{ basedir }}:
file.directory:
- group: ubunet
- user: ubunet
- require:
- user: ubunet
- recurse:
- user
- group
ubunet:
group.present:
- gid: 1500
user.present:
- uid: 1500
- gid: 1500
- createhome: False
- require:
- group: ubunet
}}}
The docs for all the different state definitions are at:
http://docs.saltstack.com/ref/states/all/
TODO:
* Add test helpers which will ensure that machine state definitions
are functionally (but not necessarily logically) correct (ie. getting
salt to parse all state defs.
* Add a link to a public bootstrap charm example / blogpost.
* Find a way to obviate the need to use the grains['charm_dir'] syntax
in templates.
"""
# Copyright 2013 Canonical Ltd.
#
# Authors:
# Charm Helpers Developers <juju@lists.ubuntu.com>
import subprocess
import charmhelpers.contrib.templating.contexts
import charmhelpers.core.host
import charmhelpers.core.hookenv
salt_grains_path = '/etc/salt/grains'
def install_salt_support(from_ppa=True):
"""Installs the salt-minion helper for machine state.
By default the salt-minion package is installed from
the saltstack PPA. If from_ppa is False you must ensure
that the salt-minion package is available in the apt cache.
"""
if from_ppa:
subprocess.check_call([
'/usr/bin/add-apt-repository',
'--yes',
'ppa:saltstack/salt',
])
subprocess.check_call(['/usr/bin/apt-get', 'update'])
# We install salt-common as salt-minion would run the salt-minion
# daemon.
charmhelpers.fetch.apt_install('salt-common')
def update_machine_state(state_path):
"""Update the machine state using the provided state declaration."""
charmhelpers.contrib.templating.contexts.juju_state_to_yaml(
salt_grains_path)
subprocess.check_call([
'salt-call',
'--local',
'state.template',
state_path,
])

78
lib/charmhelpers/contrib/ssl/__init__.py
View File

@ -1,78 +0,0 @@
import subprocess
from charmhelpers.core import hookenv
def generate_selfsigned(keyfile, certfile, keysize="1024", config=None, subject=None, cn=None):
"""Generate selfsigned SSL keypair
You must provide one of the 3 optional arguments:
config, subject or cn
If more than one is provided the leftmost will be used
Arguments:
keyfile -- (required) full path to the keyfile to be created
certfile -- (required) full path to the certfile to be created
keysize -- (optional) SSL key length
config -- (optional) openssl configuration file
subject -- (optional) dictionary with SSL subject variables
cn -- (optional) cerfificate common name
Required keys in subject dict:
cn -- Common name (eq. FQDN)
Optional keys in subject dict
country -- Country Name (2 letter code)
state -- State or Province Name (full name)
locality -- Locality Name (eg, city)
organization -- Organization Name (eg, company)
organizational_unit -- Organizational Unit Name (eg, section)
email -- Email Address
"""
cmd = []
if config:
cmd = ["/usr/bin/openssl", "req", "-new", "-newkey",
"rsa:{}".format(keysize), "-days", "365", "-nodes", "-x509",
"-keyout", keyfile,
"-out", certfile, "-config", config]
elif subject:
ssl_subject = ""
if "country" in subject:
ssl_subject = ssl_subject + "/C={}".format(subject["country"])
if "state" in subject:
ssl_subject = ssl_subject + "/ST={}".format(subject["state"])
if "locality" in subject:
ssl_subject = ssl_subject + "/L={}".format(subject["locality"])
if "organization" in subject:
ssl_subject = ssl_subject + "/O={}".format(subject["organization"])
if "organizational_unit" in subject:
ssl_subject = ssl_subject + "/OU={}".format(subject["organizational_unit"])
if "cn" in subject:
ssl_subject = ssl_subject + "/CN={}".format(subject["cn"])
else:
hookenv.log("When using \"subject\" argument you must "
"provide \"cn\" field at very least")
return False
if "email" in subject:
ssl_subject = ssl_subject + "/emailAddress={}".format(subject["email"])
cmd = ["/usr/bin/openssl", "req", "-new", "-newkey",
"rsa:{}".format(keysize), "-days", "365", "-nodes", "-x509",
"-keyout", keyfile,
"-out", certfile, "-subj", ssl_subject]
elif cn:
cmd = ["/usr/bin/openssl", "req", "-new", "-newkey",
"rsa:{}".format(keysize), "-days", "365", "-nodes", "-x509",
"-keyout", keyfile,
"-out", certfile, "-subj", "/CN={}".format(cn)]
if not cmd:
hookenv.log("No config, subject or cn provided,"
"unable to generate self signed SSL certificates")
return False
try:
subprocess.check_call(cmd)
return True
except Exception as e:
print "Execution of openssl command failed:\n{}".format(e)
return False

0
lib/charmhelpers/contrib/storage/__init__.py
View File

0
lib/charmhelpers/contrib/storage/linux/__init__.py
View File

383
lib/charmhelpers/contrib/storage/linux/ceph.py
View File

@ -1,383 +0,0 @@
#
# Copyright 2012 Canonical Ltd.
#
# This file is sourced from lp:openstack-charm-helpers
#
# Authors:
# James Page <james.page@ubuntu.com>
# Adam Gandelman <adamg@ubuntu.com>
#
import os
import shutil
import json
import time
from subprocess import (
check_call,
check_output,
CalledProcessError
)
from charmhelpers.core.hookenv import (
relation_get,
relation_ids,
related_units,
log,
INFO,
WARNING,
ERROR
)
from charmhelpers.core.host import (
mount,
mounts,
service_start,
service_stop,
service_running,
umount,
)
from charmhelpers.fetch import (
apt_install,
)
KEYRING = '/etc/ceph/ceph.client.{}.keyring'
KEYFILE = '/etc/ceph/ceph.client.{}.key'
CEPH_CONF = """[global]
auth supported = {auth}
keyring = {keyring}
mon host = {mon_hosts}
"""
def install():
''' Basic Ceph client installation '''
ceph_dir = "/etc/ceph"
if not os.path.exists(ceph_dir):
os.mkdir(ceph_dir)
apt_install('ceph-common', fatal=True)
def rbd_exists(service, pool, rbd_img):
''' Check to see if a RADOS block device exists '''
try:
out = check_output(['rbd', 'list', '--id', service,
'--pool', pool])
except CalledProcessError:
return False
else:
return rbd_img in out
def create_rbd_image(service, pool, image, sizemb):
''' Create a new RADOS block device '''
cmd = [
'rbd',
'create',
image,
'--size',
str(sizemb),
'--id',
service,
'--pool',
pool
]
check_call(cmd)
def pool_exists(service, name):
''' Check to see if a RADOS pool already exists '''
try:
out = check_output(['rados', '--id', service, 'lspools'])
except CalledProcessError:
return False
else:
return name in out
def get_osds(service):
'''
Return a list of all Ceph Object Storage Daemons
currently in the cluster
'''
version = ceph_version()
if version and version >= '0.56':
return json.loads(check_output(['ceph', '--id', service,
'osd', 'ls', '--format=json']))
else:
return None
def create_pool(service, name, replicas=2):
''' Create a new RADOS pool '''
if pool_exists(service, name):
log("Ceph pool {} already exists, skipping creation".format(name),
level=WARNING)
return
# Calculate the number of placement groups based
# on upstream recommended best practices.
osds = get_osds(service)
if osds:
pgnum = (len(osds) * 100 / replicas)
else:
# NOTE(james-page): Default to 200 for older ceph versions
# which don't support OSD query from cli
pgnum = 200
cmd = [
'ceph', '--id', service,
'osd', 'pool', 'create',
name, str(pgnum)
]
check_call(cmd)
cmd = [
'ceph', '--id', service,
'osd', 'pool', 'set', name,
'size', str(replicas)
]
check_call(cmd)
def delete_pool(service, name):
''' Delete a RADOS pool from ceph '''
cmd = [
'ceph', '--id', service,
'osd', 'pool', 'delete',
name, '--yes-i-really-really-mean-it'
]
check_call(cmd)
def _keyfile_path(service):
return KEYFILE.format(service)
def _keyring_path(service):
return KEYRING.format(service)
def create_keyring(service, key):
''' Create a new Ceph keyring containing key'''
keyring = _keyring_path(service)
if os.path.exists(keyring):
log('ceph: Keyring exists at %s.' % keyring, level=WARNING)
return
cmd = [
'ceph-authtool',
keyring,
'--create-keyring',
'--name=client.{}'.format(service),
'--add-key={}'.format(key)
]
check_call(cmd)
log('ceph: Created new ring at %s.' % keyring, level=INFO)
def create_key_file(service, key):
''' Create a file containing key '''
keyfile = _keyfile_path(service)
if os.path.exists(keyfile):
log('ceph: Keyfile exists at %s.' % keyfile, level=WARNING)
return
with open(keyfile, 'w') as fd:
fd.write(key)
log('ceph: Created new keyfile at %s.' % keyfile, level=INFO)
def get_ceph_nodes():
''' Query named relation 'ceph' to detemine current nodes '''
hosts = []
for r_id in relation_ids('ceph'):
for unit in related_units(r_id):
hosts.append(relation_get('private-address', unit=unit, rid=r_id))
return hosts
def configure(service, key, auth):
''' Perform basic configuration of Ceph '''
create_keyring(service, key)
create_key_file(service, key)
hosts = get_ceph_nodes()
with open('/etc/ceph/ceph.conf', 'w') as ceph_conf:
ceph_conf.write(CEPH_CONF.format(auth=auth,
keyring=_keyring_path(service),
mon_hosts=",".join(map(str, hosts))))
modprobe('rbd')
def image_mapped(name):
''' Determine whether a RADOS block device is mapped locally '''
try:
out = check_output(['rbd', 'showmapped'])
except CalledProcessError:
return False
else:
return name in out
def map_block_storage(service, pool, image):
''' Map a RADOS block device for local use '''
cmd = [
'rbd',
'map',
'{}/{}'.format(pool, image),
'--user',
service,
'--secret',
_keyfile_path(service),
]
check_call(cmd)
def filesystem_mounted(fs):
''' Determine whether a filesytems is already mounted '''
return fs in [f for f, m in mounts()]
def make_filesystem(blk_device, fstype='ext4', timeout=10):
''' Make a new filesystem on the specified block device '''
count = 0
e_noent = os.errno.ENOENT
while not os.path.exists(blk_device):
if count >= timeout:
log('ceph: gave up waiting on block device %s' % blk_device,
level=ERROR)
raise IOError(e_noent, os.strerror(e_noent), blk_device)
log('ceph: waiting for block device %s to appear' % blk_device,
level=INFO)
count += 1
time.sleep(1)
else:
log('ceph: Formatting block device %s as filesystem %s.' %
(blk_device, fstype), level=INFO)
check_call(['mkfs', '-t', fstype, blk_device])
def place_data_on_block_device(blk_device, data_src_dst):
''' Migrate data in data_src_dst to blk_device and then remount '''
# mount block device into /mnt
mount(blk_device, '/mnt')
# copy data to /mnt
copy_files(data_src_dst, '/mnt')
# umount block device
umount('/mnt')
# Grab user/group ID's from original source
_dir = os.stat(data_src_dst)
uid = _dir.st_uid
gid = _dir.st_gid
# re-mount where the data should originally be
# TODO: persist is currently a NO-OP in core.host
mount(blk_device, data_src_dst, persist=True)
# ensure original ownership of new mount.
os.chown(data_src_dst, uid, gid)
# TODO: re-use
def modprobe(module):
''' Load a kernel module and configure for auto-load on reboot '''
log('ceph: Loading kernel module', level=INFO)
cmd = ['modprobe', module]
check_call(cmd)
with open('/etc/modules', 'r+') as modules:
if module not in modules.read():
modules.write(module)
def copy_files(src, dst, symlinks=False, ignore=None):
''' Copy files from src to dst '''
for item in os.listdir(src):
s = os.path.join(src, item)
d = os.path.join(dst, item)
if os.path.isdir(s):
shutil.copytree(s, d, symlinks, ignore)
else:
shutil.copy2(s, d)
def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
blk_device, fstype, system_services=[]):
"""
NOTE: This function must only be called from a single service unit for
the same rbd_img otherwise data loss will occur.
Ensures given pool and RBD image exists, is mapped to a block device,
and the device is formatted and mounted at the given mount_point.
If formatting a device for the first time, data existing at mount_point
will be migrated to the RBD device before being re-mounted.
All services listed in system_services will be stopped prior to data
migration and restarted when complete.
"""
# Ensure pool, RBD image, RBD mappings are in place.
if not pool_exists(service, pool):
log('ceph: Creating new pool {}.'.format(pool))
create_pool(service, pool)
if not rbd_exists(service, pool, rbd_img):
log('ceph: Creating RBD image ({}).'.format(rbd_img))
create_rbd_image(service, pool, rbd_img, sizemb)
if not image_mapped(rbd_img):
log('ceph: Mapping RBD Image {} as a Block Device.'.format(rbd_img))
map_block_storage(service, pool, rbd_img)
# make file system
# TODO: What happens if for whatever reason this is run again and
# the data is already in the rbd device and/or is mounted??
# When it is mounted already, it will fail to make the fs
# XXX: This is really sketchy! Need to at least add an fstab entry
# otherwise this hook will blow away existing data if its executed
# after a reboot.
if not filesystem_mounted(mount_point):
make_filesystem(blk_device, fstype)
for svc in system_services:
if service_running(svc):
log('ceph: Stopping services {} prior to migrating data.'
.format(svc))
service_stop(svc)
place_data_on_block_device(blk_device, mount_point)
for svc in system_services:
log('ceph: Starting service {} after migrating data.'
.format(svc))
service_start(svc)
def ensure_ceph_keyring(service, user=None, group=None):
'''
Ensures a ceph keyring is created for a named service
and optionally ensures user and group ownership.
Returns False if no ceph key is available in relation state.
'''
key = None
for rid in relation_ids('ceph'):
for unit in related_units(rid):
key = relation_get('key', rid=rid, unit=unit)
if key:
break
if not key:
return False
create_keyring(service=service, key=key)
keyring = _keyring_path(service)
if user and group:
check_call(['chown', '%s.%s' % (user, group), keyring])
return True
def ceph_version():
''' Retrieve the local version of ceph '''
if os.path.exists('/usr/bin/ceph'):
cmd = ['ceph', '-v']
output = check_output(cmd)
output = output.split()
if len(output) > 3:
return output[2]
else:
return None
else:
return None

62
lib/charmhelpers/contrib/storage/linux/loopback.py
View File

@ -1,62 +0,0 @@
import os
import re
from subprocess import (
check_call,
check_output,
)
##################################################
# loopback device helpers.
##################################################
def loopback_devices():
'''
Parse through 'losetup -a' output to determine currently mapped
loopback devices. Output is expected to look like:
/dev/loop0: [0807]:961814 (/tmp/my.img)
:returns: dict: a dict mapping {loopback_dev: backing_file}
'''
loopbacks = {}
cmd = ['losetup', '-a']
devs = [d.strip().split(' ') for d in
check_output(cmd).splitlines() if d != '']
for dev, _, f in devs:
loopbacks[dev.replace(':', '')] = re.search('\((\S+)\)', f).groups()[0]
return loopbacks
def create_loopback(file_path):
'''
Create a loopback device for a given backing file.
:returns: str: Full path to new loopback device (eg, /dev/loop0)
'''
file_path = os.path.abspath(file_path)
check_call(['losetup', '--find', file_path])
for d, f in loopback_devices().iteritems():
if f == file_path:
return d
def ensure_loopback_device(path, size):
'''
Ensure a loopback device exists for a given backing file path and size.
If it a loopback device is not mapped to file, a new one will be created.
TODO: Confirm size of found loopback device.
:returns: str: Full path to the ensured loopback device (eg, /dev/loop0)
'''
for d, f in loopback_devices().iteritems():
if f == path:
return d
if not os.path.exists(path):
cmd = ['truncate', '--size', size, path]
check_call(cmd)
return create_loopback(path)

88
lib/charmhelpers/contrib/storage/linux/lvm.py
View File

@ -1,88 +0,0 @@
from subprocess import (
CalledProcessError,
check_call,
check_output,
Popen,
PIPE,
)
##################################################
# LVM helpers.
##################################################
def deactivate_lvm_volume_group(block_device):
'''
Deactivate any volume gruop associated with an LVM physical volume.
:param block_device: str: Full path to LVM physical volume
'''
vg = list_lvm_volume_group(block_device)
if vg:
cmd = ['vgchange', '-an', vg]
check_call(cmd)
def is_lvm_physical_volume(block_device):
'''
Determine whether a block device is initialized as an LVM PV.
:param block_device: str: Full path of block device to inspect.
:returns: boolean: True if block device is a PV, False if not.
'''
try:
check_output(['pvdisplay', block_device])
return True
except CalledProcessError:
return False
def remove_lvm_physical_volume(block_device):
'''
Remove LVM PV signatures from a given block device.
:param block_device: str: Full path of block device to scrub.
'''
p = Popen(['pvremove', '-ff', block_device],
stdin=PIPE)
p.communicate(input='y\n')
def list_lvm_volume_group(block_device):
'''
List LVM volume group associated with a given block device.
Assumes block device is a valid LVM PV.
:param block_device: str: Full path of block device to inspect.
:returns: str: Name of volume group associated with block device or None
'''
vg = None
pvd = check_output(['pvdisplay', block_device]).splitlines()
for l in pvd:
if l.strip().startswith('VG Name'):
vg = ' '.join(l.split()).split(' ').pop()
return vg
def create_lvm_physical_volume(block_device):
'''
Initialize a block device as an LVM physical volume.
:param block_device: str: Full path of block device to initialize.
'''
check_call(['pvcreate', block_device])
def create_lvm_volume_group(volume_group, block_device):
'''
Create an LVM volume group backed by a given block device.
Assumes block device has already been initialized as an LVM PV.
:param volume_group: str: Name of volume group to create.
:block_device: str: Full path of PV-initialized block device.
'''
check_call(['vgcreate', volume_group, block_device])

25
lib/charmhelpers/contrib/storage/linux/utils.py
View File

@ -1,25 +0,0 @@
from os import stat
from stat import S_ISBLK
from subprocess import (
check_call
)
def is_block_device(path):
'''
Confirm device at path is a valid block device node.
:returns: boolean: True if path is a block device, False if not.
'''
return S_ISBLK(stat(path).st_mode)
def zap_disk(block_device):
'''
Clear a block device of partition table. Relies on sgdisk, which is
installed as pat of the 'gdisk' package in Ubuntu.
:param block_device: str: Full path of block device to clean.
'''
check_call(['sgdisk', '--zap-all', '--mbrtogpt', block_device])

0
lib/charmhelpers/contrib/templating/__init__.py
View File

73
lib/charmhelpers/contrib/templating/contexts.py
View File

@ -1,73 +0,0 @@
# Copyright 2013 Canonical Ltd.
#
# Authors:
# Charm Helpers Developers <juju@lists.ubuntu.com>
"""A helper to create a yaml cache of config with namespaced relation data."""
import os
import yaml
import charmhelpers.core.hookenv
charm_dir = os.environ.get('CHARM_DIR', '')
def juju_state_to_yaml(yaml_path, namespace_separator=':',
allow_hyphens_in_keys=True):
"""Update the juju config and state in a yaml file.
This includes any current relation-get data, and the charm
directory.
This function was created for the ansible and saltstack
support, as those libraries can use a yaml file to supply
context to templates, but it may be useful generally to
create and update an on-disk cache of all the config, including
previous relation data.
By default, hyphens are allowed in keys as this is supported
by yaml, but for tools like ansible, hyphens are not valid [1].
[1] http://www.ansibleworks.com/docs/playbooks_variables.html#what-makes-a-valid-variable-name
"""
config = charmhelpers.core.hookenv.config()
# Add the charm_dir which we will need to refer to charm
# file resources etc.
config['charm_dir'] = charm_dir
config['local_unit'] = charmhelpers.core.hookenv.local_unit()
# Add any relation data prefixed with the relation type.
relation_type = charmhelpers.core.hookenv.relation_type()
if relation_type is not None:
relation_data = charmhelpers.core.hookenv.relation_get()
relation_data = dict(
("{relation_type}{namespace_separator}{key}".format(
relation_type=relation_type.replace('-', '_'),
key=key,
namespace_separator=namespace_separator), val)
for key, val in relation_data.items())
config.update(relation_data)
# Don't use non-standard tags for unicode which will not
# work when salt uses yaml.load_safe.
yaml.add_representer(unicode, lambda dumper,
value: dumper.represent_scalar(
u'tag:yaml.org,2002:str', value))
yaml_dir = os.path.dirname(yaml_path)
if not os.path.exists(yaml_dir):
os.makedirs(yaml_dir)
if os.path.exists(yaml_path):
with open(yaml_path, "r") as existing_vars_file:
existing_vars = yaml.load(existing_vars_file.read())
else:
existing_vars = {}
if not allow_hyphens_in_keys:
config = dict(
(key.replace('-', '_'), val) for key, val in config.items())
existing_vars.update(config)
with open(yaml_path, "w+") as fp:
fp.write(yaml.dump(existing_vars))

13
lib/charmhelpers/contrib/templating/pyformat.py
View File

@ -1,13 +0,0 @@
'''
Templating using standard Python str.format() method.
'''
from charmhelpers.core import hookenv
def render(template, extra={}, **kwargs):
"""Return the template rendered using Python's str.format()."""
context = hookenv.execution_environment()
context.update(extra)
context.update(kwargs)
return template.format(**context)

55
lib/charmhelpers/contrib/unison/utils.py
View File

@ -47,9 +47,7 @@ import subprocess
import sys
from charmhelpers.core.hookenv import (
config,
log as juju_log,
charm_dir,
log,
ERROR,
INFO,
relation_get,
@ -59,13 +57,13 @@ from charmhelpers.core.hookenv import (
unit_get
)
def get_homedir(user):
try:
user = pwd.getpwnam(user)
return user.pw_dir
except KeyError:
juju_log('INFO',
'Could not get homedir for user %s: user exists?')
log('INFO', 'Could not get homedir for user %s: user exists?')
sys.exit(1)
@ -77,29 +75,28 @@ def get_keypair(user):
priv_key = os.path.join(ssh_dir, 'id_rsa')
if not os.path.isfile(priv_key):
juju_log('INFO', 'Generating new ssh key for user %s.' % user)
log('INFO', 'Generating new ssh key for user %s.' % user)
cmd = ['ssh-keygen', '-q', '-N', '', '-t', 'rsa', '-b', '2048',
'-f', priv_key]
subprocess.check_call(cmd)
pub_key = '%s.pub' % priv_key
if not os.path.isfile(pub_key):
juju_log('INFO', 'Generatring missing ssh public key @ %s.' % \
pub_key)
log('INFO', 'Generatring missing ssh public key @ %s.' % pub_key)
cmd = ['ssh-keygen', '-y', '-f', priv_key]
p = subprocess.check_output(cmd).strip()
with open(pub_key, 'wb') as out:
out.write(p)
subprocess.check_call(['chown', '-R', user, ssh_dir])
return open(priv_key, 'r').read().strip(), \
open(pub_key, 'r').read().strip()
open(pub_key, 'r').read().strip()
def write_authorized_keys(user, keys):
home_dir = get_homedir(user)
ssh_dir = os.path.join(home_dir, '.ssh')
auth_keys = os.path.join(ssh_dir, 'authorized_keys')
juju_log('INFO', 'Syncing authorized_keys @ %s.' % auth_keys)
log('INFO', 'Syncing authorized_keys @ %s.' % auth_keys)
with open(auth_keys, 'wb') as out:
for k in keys:
out.write('%s\n' % k)
@ -114,7 +111,7 @@ def write_known_hosts(user, hosts):
cmd = ['ssh-keyscan', '-H', '-t', 'rsa', host]
remote_key = subprocess.check_output(cmd).strip()
khosts.append(remote_key)
juju_log('INFO', 'Syncing known_hosts @ %s.' % known_hosts)
log('INFO', 'Syncing known_hosts @ %s.' % known_hosts)
with open(known_hosts, 'wb') as out:
for host in khosts:
out.write('%s\n' % host)
@ -125,7 +122,7 @@ def ensure_user(user, group=None):
try:
pwd.getpwnam(user)
except KeyError:
juju_log('INFO', 'Creating new user %s.%s.' % (user, group))
log('INFO', 'Creating new user %s.%s.' % (user, group))
cmd = ['adduser', '--system', '--shell', '/bin/bash', user]
if group:
try:
@ -136,7 +133,8 @@ def ensure_user(user, group=None):
subprocess.check_call(cmd)
def ssh_authorized_peers(peer_interface, user, group=None, ensure_local_user=False):
def ssh_authorized_peers(peer_interface, user, group=None,
ensure_local_user=False):
"""
Main setup function, should be called from both peer -changed and -joined
hooks with the same parameters.
@ -157,9 +155,8 @@ def ssh_authorized_peers(peer_interface, user, group=None, ensure_local_user=Fal
keys.append(settings['ssh_pub_key'])
hosts.append(settings['private-address'])
else:
juju_log('INFO',
'ssh_authorized_peers(): ssh_pub_key '\
'missing for unit %s, skipping.' % unit)
log('INFO', 'ssh_authorized_peers(): ssh_pub_key '
'missing for unit %s, skipping.' % unit)
write_authorized_keys(user, keys)
write_known_hosts(user, hosts)
authed_hosts = ':'.join(hosts)
@ -170,7 +167,7 @@ def _run_as_user(user):
try:
user = pwd.getpwnam(user)
except KeyError:
juju_log('INFO', 'Invalid user: %s' % user)
log('INFO', 'Invalid user: %s' % user)
sys.exit(1)
uid, gid = user.pw_uid, user.pw_gid
os.environ['HOME'] = user.pw_dir
@ -199,29 +196,24 @@ def sync_to_peer(host, user, paths=[], verbose=False):
path = path[:(len(path) - 1)]
try:
cmd = base_cmd + [path, 'ssh://%s@%s/%s' % (user, host, path)]
juju_log('INFO', 'Syncing local path %s to %s@%s:%s' %
(path, user, host, path))
log('INFO', 'Syncing local path %s to %s@%s:%s' %
(path, user, host, path))
run_as_user(user, cmd)
except:
# it may fail for permissions on some files
juju_log('INFO', 'Error syncing rabbit passwd files')
log('INFO', 'Error syncing rabbit passwd files')
def sync_to_peers(peer_interface, user, paths=[], verbose=False):
base_cmd = ['unison', '-auto', '-batch=true', '-confirmbigdel=false',
'-fastcheck=true', '-group=false', '-owner=false',
'-prefer=newer', '-times=true']
if not verbose:
base_cmd.append('-silent')
for r_id in (relation_ids(peer_interface) or []):
for unit in relation_list(r_id):
settings = relation_get(rid=r_id,unit=unit)
settings = relation_get(rid=r_id, unit=unit)
try:
authed_hosts = settings['ssh_authorized_hosts'].split(':')
except KeyError:
juju_log('INFO',
'unison sync_to_peers: peer has not authorized *any* hosts yet.')
log('INFO',
'unison sync_to_peers: peer has not authorized '
'*any* hosts yet.')
return
unit_hostname = unit_get('private-address')
@ -233,5 +225,6 @@ def sync_to_peers(peer_interface, user, paths=[], verbose=False):
# sync to this peer
sync_to_peer(settings['private-address'], user, paths, verbose)
else:
juju_log('INFO', 'unison sync_to_peers: peer (%s) has not authorized '\
'*this* host yet, skipping.' % settings['private-address'])
log('INFO', 'unison sync_to_peers: peer (%s) '
'has not authorized *this* host yet, skipping.' %
settings['private-address'])

279
lib/charmhelpers/fetch/__init__.py
View File

@ -1,279 +0,0 @@
import importlib
from yaml import safe_load
from charmhelpers.core.host import (
lsb_release
)
from urlparse import (
urlparse,
urlunparse,
)
import subprocess
from charmhelpers.core.hookenv import (
config,
log,
)
import apt_pkg
import os
CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
"""
PROPOSED_POCKET = """# Proposed
deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted
"""
CLOUD_ARCHIVE_POCKETS = {
# Folsom
'folsom': 'precise-updates/folsom',
'precise-folsom': 'precise-updates/folsom',
'precise-folsom/updates': 'precise-updates/folsom',
'precise-updates/folsom': 'precise-updates/folsom',
'folsom/proposed': 'precise-proposed/folsom',
'precise-folsom/proposed': 'precise-proposed/folsom',
'precise-proposed/folsom': 'precise-proposed/folsom',
# Grizzly
'grizzly': 'precise-updates/grizzly',
'precise-grizzly': 'precise-updates/grizzly',
'precise-grizzly/updates': 'precise-updates/grizzly',
'precise-updates/grizzly': 'precise-updates/grizzly',
'grizzly/proposed': 'precise-proposed/grizzly',
'precise-grizzly/proposed': 'precise-proposed/grizzly',
'precise-proposed/grizzly': 'precise-proposed/grizzly',
# Havana
'havana': 'precise-updates/havana',
'precise-havana': 'precise-updates/havana',
'precise-havana/updates': 'precise-updates/havana',
'precise-updates/havana': 'precise-updates/havana',
'havana/proposed': 'precise-proposed/havana',
'precise-havana/proposed': 'precise-proposed/havana',
'precise-proposed/havana': 'precise-proposed/havana',
# Icehouse
'icehouse': 'precise-updates/icehouse',
'precise-icehouse': 'precise-updates/icehouse',
'precise-icehouse/updates': 'precise-updates/icehouse',
'precise-updates/icehouse': 'precise-updates/icehouse',
'icehouse/proposed': 'precise-proposed/icehouse',
'precise-icehouse/proposed': 'precise-proposed/icehouse',
'precise-proposed/icehouse': 'precise-proposed/icehouse',
}
def filter_installed_packages(packages):
"""Returns a list of packages that require installation"""
apt_pkg.init()
cache = apt_pkg.Cache()
_pkgs = []
for package in packages:
try:
p = cache[package]
p.current_ver or _pkgs.append(package)
except KeyError:
log('Package {} has no installation candidate.'.format(package),
level='WARNING')
_pkgs.append(package)
return _pkgs
def apt_install(packages, options=None, fatal=False):
"""Install one or more packages"""
if options is None:
options = ['--option=Dpkg::Options::=--force-confold']
cmd = ['apt-get', '--assume-yes']
cmd.extend(options)
cmd.append('install')
if isinstance(packages, basestring):
cmd.append(packages)
else:
cmd.extend(packages)
log("Installing {} with options: {}".format(packages,
options))
env = os.environ.copy()
if 'DEBIAN_FRONTEND' not in env:
env['DEBIAN_FRONTEND'] = 'noninteractive'
if fatal:
subprocess.check_call(cmd, env=env)
else:
subprocess.call(cmd, env=env)
def apt_update(fatal=False):
"""Update local apt cache"""
cmd = ['apt-get', 'update']
if fatal:
subprocess.check_call(cmd)
else:
subprocess.call(cmd)
def apt_purge(packages, fatal=False):
"""Purge one or more packages"""
cmd = ['apt-get', '--assume-yes', 'purge']
if isinstance(packages, basestring):
cmd.append(packages)
else:
cmd.extend(packages)
log("Purging {}".format(packages))
if fatal:
subprocess.check_call(cmd)
else:
subprocess.call(cmd)
def apt_hold(packages, fatal=False):
"""Hold one or more packages"""
cmd = ['apt-mark', 'hold']
if isinstance(packages, basestring):
cmd.append(packages)
else:
cmd.extend(packages)
log("Holding {}".format(packages))
if fatal:
subprocess.check_call(cmd)
else:
subprocess.call(cmd)
def add_source(source, key=None):
if (source.startswith('ppa:') or
source.startswith('http:') or
source.startswith('deb ') or
source.startswith('cloud-archive:')):
subprocess.check_call(['add-apt-repository', '--yes', source])
elif source.startswith('cloud:'):
apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
fatal=True)
pocket = source.split(':')[-1]
if pocket not in CLOUD_ARCHIVE_POCKETS:
raise SourceConfigError(
'Unsupported cloud: source option %s' %
pocket)
actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
apt.write(CLOUD_ARCHIVE.format(actual_pocket))
elif source == 'proposed':
release = lsb_release()['DISTRIB_CODENAME']
with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
apt.write(PROPOSED_POCKET.format(release))
if key:
subprocess.check_call(['apt-key', 'import', key])
class SourceConfigError(Exception):
pass
def configure_sources(update=False,
sources_var='install_sources',
keys_var='install_keys'):
"""
Configure multiple sources from charm configuration
Example config:
install_sources:
- "ppa:foo"
- "http://example.com/repo precise main"
install_keys:
- null
- "a1b2c3d4"
Note that 'null' (a.k.a. None) should not be quoted.
"""
sources = safe_load(config(sources_var))
keys = config(keys_var)
if keys is not None:
keys = safe_load(keys)
if isinstance(sources, basestring) and (
keys is None or isinstance(keys, basestring)):
add_source(sources, keys)
else:
if not len(sources) == len(keys):
msg = 'Install sources and keys lists are different lengths'
raise SourceConfigError(msg)
for src_num in range(len(sources)):
add_source(sources[src_num], keys[src_num])
if update:
apt_update(fatal=True)
# The order of this list is very important. Handlers should be listed in from
# least- to most-specific URL matching.
FETCH_HANDLERS = (
'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
)
class UnhandledSource(Exception):
pass
def install_remote(source):
"""
Install a file tree from a remote source
The specified source should be a url of the form:
scheme://[host]/path[#[option=value][&...]]
Schemes supported are based on this modules submodules
Options supported are submodule-specific"""
# We ONLY check for True here because can_handle may return a string
# explaining why it can't handle a given source.
handlers = [h for h in plugins() if h.can_handle(source) is True]
installed_to = None
for handler in handlers:
try:
installed_to = handler.install(source)
except UnhandledSource:
pass
if not installed_to:
raise UnhandledSource("No handler found for source {}".format(source))
return installed_to
def install_from_config(config_var_name):
charm_config = config()
source = charm_config[config_var_name]
return install_remote(source)
class BaseFetchHandler(object):
"""Base class for FetchHandler implementations in fetch plugins"""
def can_handle(self, source):
"""Returns True if the source can be handled. Otherwise returns
a string explaining why it cannot"""
return "Wrong source type"
def install(self, source):
"""Try to download and unpack the source. Return the path to the
unpacked files or raise UnhandledSource."""
raise UnhandledSource("Wrong source type {}".format(source))
def parse_url(self, url):
return urlparse(url)
def base_url(self, url):
"""Return url without querystring or fragment"""
parts = list(self.parse_url(url))
parts[4:] = ['' for i in parts[4:]]
return urlunparse(parts)
def plugins(fetch_handlers=None):
if not fetch_handlers:
fetch_handlers = FETCH_HANDLERS
plugin_list = []
for handler_name in fetch_handlers:
package, classname = handler_name.rsplit('.', 1)
try:
handler_class = getattr(
importlib.import_module(package),
classname)
plugin_list.append(handler_class())
except (ImportError, AttributeError):
# Skip missing plugins so that they can be ommitted from
# installation if desired
log("FetchHandler {} not found, skipping plugin".format(
handler_name))
return plugin_list

48
lib/charmhelpers/fetch/archiveurl.py
View File

@ -1,48 +0,0 @@
import os
import urllib2
from charmhelpers.fetch import (
BaseFetchHandler,
UnhandledSource
)
from charmhelpers.payload.archive import (
get_archive_handler,
extract,
)
from charmhelpers.core.host import mkdir
class ArchiveUrlFetchHandler(BaseFetchHandler):
"""Handler for archives via generic URLs"""
def can_handle(self, source):
url_parts = self.parse_url(source)
if url_parts.scheme not in ('http', 'https', 'ftp', 'file'):
return "Wrong source type"
if get_archive_handler(self.base_url(source)):
return True
return False
def download(self, source, dest):
# propogate all exceptions
# URLError, OSError, etc
response = urllib2.urlopen(source)
try:
with open(dest, 'w') as dest_file:
dest_file.write(response.read())
except Exception as e:
if os.path.isfile(dest):
os.unlink(dest)
raise e
def install(self, source):
url_parts = self.parse_url(source)
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
if not os.path.exists(dest_dir):
mkdir(dest_dir, perms=0755)
dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path))
try:
self.download(source, dld_file)
except urllib2.URLError as e:
raise UnhandledSource(e.reason)
except OSError as e:
raise UnhandledSource(e.strerror)
return extract(dld_file)

49
lib/charmhelpers/fetch/bzrurl.py
View File

@ -1,49 +0,0 @@
import os
from charmhelpers.fetch import (
BaseFetchHandler,
UnhandledSource
)
from charmhelpers.core.host import mkdir
try:
from bzrlib.branch import Branch
except ImportError:
from charmhelpers.fetch import apt_install
apt_install("python-bzrlib")
from bzrlib.branch import Branch
class BzrUrlFetchHandler(BaseFetchHandler):
"""Handler for bazaar branches via generic and lp URLs"""
def can_handle(self, source):
url_parts = self.parse_url(source)
if url_parts.scheme not in ('bzr+ssh', 'lp'):
return False
else:
return True
def branch(self, source, dest):
url_parts = self.parse_url(source)
# If we use lp:branchname scheme we need to load plugins
if not self.can_handle(source):
raise UnhandledSource("Cannot handle {}".format(source))
if url_parts.scheme == "lp":
from bzrlib.plugin import load_plugins
load_plugins()
try:
remote_branch = Branch.open(source)
remote_branch.bzrdir.sprout(dest).open_branch()
except Exception as e:
raise e
def install(self, source):
url_parts = self.parse_url(source)
branch_name = url_parts.path.strip("/").split("/")[-1]
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", branch_name)
if not os.path.exists(dest_dir):
mkdir(dest_dir, perms=0755)
try:
self.branch(source, dest_dir)
except OSError as e:
raise UnhandledSource(e.strerror)
return dest_dir

1
lib/charmhelpers/payload/__init__.py
View File

@ -1 +0,0 @@
"Tools for working with files injected into a charm just before deployment."

57
lib/charmhelpers/payload/archive.py
View File

@ -1,57 +0,0 @@
import os
import tarfile
import zipfile
from charmhelpers.core import (
host,
hookenv,
)
class ArchiveError(Exception):
pass
def get_archive_handler(archive_name):
if os.path.isfile(archive_name):
if tarfile.is_tarfile(archive_name):
return extract_tarfile
elif zipfile.is_zipfile(archive_name):
return extract_zipfile
else:
# look at the file name
for ext in ('.tar', '.tar.gz', '.tgz', 'tar.bz2', '.tbz2', '.tbz'):
if archive_name.endswith(ext):
return extract_tarfile
for ext in ('.zip', '.jar'):
if archive_name.endswith(ext):
return extract_zipfile
def archive_dest_default(archive_name):
archive_file = os.path.basename(archive_name)
return os.path.join(hookenv.charm_dir(), "archives", archive_file)
def extract(archive_name, destpath=None):
handler = get_archive_handler(archive_name)
if handler:
if not destpath:
destpath = archive_dest_default(archive_name)
if not os.path.isdir(destpath):
host.mkdir(destpath)
handler(archive_name, destpath)
return destpath
else:
raise ArchiveError("No handler for archive")
def extract_tarfile(archive_name, destpath):
"Unpack a tar archive, optionally compressed"
archive = tarfile.open(archive_name)
archive.extractall(destpath)
def extract_zipfile(archive_name, destpath):
"Unpack a zip file"
archive = zipfile.ZipFile(archive_name)
archive.extractall(destpath)

50
lib/charmhelpers/payload/execd.py
View File

@ -1,50 +0,0 @@
#!/usr/bin/env python
import os
import sys
import subprocess
from charmhelpers.core import hookenv
def default_execd_dir():
return os.path.join(os.environ['CHARM_DIR'], 'exec.d')
def execd_module_paths(execd_dir=None):
"""Generate a list of full paths to modules within execd_dir."""
if not execd_dir:
execd_dir = default_execd_dir()
if not os.path.exists(execd_dir):
return
for subpath in os.listdir(execd_dir):
module = os.path.join(execd_dir, subpath)
if os.path.isdir(module):
yield module
def execd_submodule_paths(command, execd_dir=None):
"""Generate a list of full paths to the specified command within exec_dir.
"""
for module_path in execd_module_paths(execd_dir):
path = os.path.join(module_path, command)
if os.access(path, os.X_OK) and os.path.isfile(path):
yield path
def execd_run(command, execd_dir=None, die_on_error=False, stderr=None):
"""Run command for each module within execd_dir which defines it."""
for submodule_path in execd_submodule_paths(command, execd_dir):
try:
subprocess.check_call(submodule_path, shell=True, stderr=stderr)
except subprocess.CalledProcessError as e:
hookenv.log("Error ({}) running {}. Output: {}".format(
e.returncode, e.cmd, e.output))
if die_on_error:
sys.exit(e.returncode)
def execd_preinstall(execd_dir=None):
"""Run charm-pre-install for each module within execd_dir."""
execd_run('charm-pre-install', execd_dir=execd_dir)

2
revision
View File

@ -1 +1 @@
117
118