Charm Helpers resync
This commit is contained in:
parent
82d6f33d89
commit
486bdca45a
|
@ -0,0 +1,38 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# Bootstrap charm-helpers, installing its dependencies if necessary using
|
||||
# only standard libraries.
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
try:
|
||||
import six # flake8: noqa
|
||||
except ImportError:
|
||||
if sys.version_info.major == 2:
|
||||
subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
|
||||
else:
|
||||
subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
|
||||
import six # flake8: noqa
|
||||
|
||||
try:
|
||||
import yaml # flake8: noqa
|
||||
except ImportError:
|
||||
if sys.version_info.major == 2:
|
||||
subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
|
||||
else:
|
||||
subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
|
||||
import yaml # flake8: noqa
|
|
@ -0,0 +1,15 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
|
@ -0,0 +1,15 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
|
@ -1,3 +1,19 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
#
|
||||
# Copyright 2012 Canonical Ltd.
|
||||
#
|
||||
|
|
|
@ -1,3 +1,19 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
#
|
||||
# Copyright 2012 Canonical Ltd.
|
||||
#
|
||||
|
@ -16,6 +32,8 @@ import os
|
|||
|
||||
from socket import gethostname as get_unit_hostname
|
||||
|
||||
import six
|
||||
|
||||
from charmhelpers.core.hookenv import (
|
||||
log,
|
||||
relation_ids,
|
||||
|
@ -27,12 +45,24 @@ from charmhelpers.core.hookenv import (
|
|||
WARNING,
|
||||
unit_get,
|
||||
)
|
||||
from charmhelpers.core.decorators import (
|
||||
retry_on_exception,
|
||||
)
|
||||
from charmhelpers.core.strutils import (
|
||||
bool_from_string,
|
||||
)
|
||||
|
||||
DC_RESOURCE_NAME = 'DC'
|
||||
|
||||
|
||||
class HAIncompleteConfig(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class CRMResourceNotFound(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def is_elected_leader(resource):
|
||||
"""
|
||||
Returns True if the charm executing this is the elected cluster leader.
|
||||
|
@ -67,24 +97,53 @@ def is_clustered():
|
|||
return False
|
||||
|
||||
|
||||
def is_crm_leader(resource):
|
||||
def is_crm_dc():
|
||||
"""
|
||||
Determine leadership by querying the pacemaker Designated Controller
|
||||
"""
|
||||
cmd = ['crm', 'status']
|
||||
try:
|
||||
status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
|
||||
if not isinstance(status, six.text_type):
|
||||
status = six.text_type(status, "utf-8")
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
current_dc = ''
|
||||
for line in status.split('\n'):
|
||||
if line.startswith('Current DC'):
|
||||
# Current DC: juju-lytrusty-machine-2 (168108163) - partition with quorum
|
||||
current_dc = line.split(':')[1].split()[0]
|
||||
if current_dc == get_unit_hostname():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@retry_on_exception(5, base_delay=2, exc_type=CRMResourceNotFound)
|
||||
def is_crm_leader(resource, retry=False):
|
||||
"""
|
||||
Returns True if the charm calling this is the elected corosync leader,
|
||||
as returned by calling the external "crm" command.
|
||||
|
||||
We allow this operation to be retried to avoid the possibility of getting a
|
||||
false negative. See LP #1396246 for more info.
|
||||
"""
|
||||
cmd = [
|
||||
"crm", "resource",
|
||||
"show", resource
|
||||
]
|
||||
if resource == DC_RESOURCE_NAME:
|
||||
return is_crm_dc()
|
||||
cmd = ['crm', 'resource', 'show', resource]
|
||||
try:
|
||||
status = subprocess.check_output(cmd)
|
||||
status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
|
||||
if not isinstance(status, six.text_type):
|
||||
status = six.text_type(status, "utf-8")
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
else:
|
||||
if get_unit_hostname() in status:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
status = None
|
||||
|
||||
if status and get_unit_hostname() in status:
|
||||
return True
|
||||
|
||||
if status and "resource %s is NOT running" % (resource) in status:
|
||||
raise CRMResourceNotFound("CRM resource %s not found" % (resource))
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def is_leader(resource):
|
||||
|
@ -133,7 +192,8 @@ def https():
|
|||
.
|
||||
returns: boolean
|
||||
'''
|
||||
if config_get('use-https') == "yes":
|
||||
use_https = config_get('use-https')
|
||||
if use_https and bool_from_string(use_https):
|
||||
return True
|
||||
if config_get('ssl_cert') and config_get('ssl_key'):
|
||||
return True
|
||||
|
@ -150,54 +210,66 @@ def https():
|
|||
return False
|
||||
|
||||
|
||||
def determine_api_port(public_port):
|
||||
def determine_api_port(public_port, singlenode_mode=False):
|
||||
'''
|
||||
Determine correct API server listening port based on
|
||||
existence of HTTPS reverse proxy and/or haproxy.
|
||||
|
||||
public_port: int: standard public port for given service
|
||||
|
||||
singlenode_mode: boolean: Shuffle ports when only a single unit is present
|
||||
|
||||
returns: int: the correct listening port for the API service
|
||||
'''
|
||||
i = 0
|
||||
if len(peer_units()) > 0 or is_clustered():
|
||||
if singlenode_mode:
|
||||
i += 1
|
||||
elif len(peer_units()) > 0 or is_clustered():
|
||||
i += 1
|
||||
if https():
|
||||
i += 1
|
||||
return public_port - (i * 10)
|
||||
|
||||
|
||||
def determine_apache_port(public_port):
|
||||
def determine_apache_port(public_port, singlenode_mode=False):
|
||||
'''
|
||||
Description: Determine correct apache listening port based on public IP +
|
||||
state of the cluster.
|
||||
|
||||
public_port: int: standard public port for given service
|
||||
|
||||
singlenode_mode: boolean: Shuffle ports when only a single unit is present
|
||||
|
||||
returns: int: the correct listening port for the HAProxy service
|
||||
'''
|
||||
i = 0
|
||||
if len(peer_units()) > 0 or is_clustered():
|
||||
if singlenode_mode:
|
||||
i += 1
|
||||
elif len(peer_units()) > 0 or is_clustered():
|
||||
i += 1
|
||||
return public_port - (i * 10)
|
||||
|
||||
|
||||
def get_hacluster_config():
|
||||
def get_hacluster_config(exclude_keys=None):
|
||||
'''
|
||||
Obtains all relevant configuration from charm configuration required
|
||||
for initiating a relation to hacluster:
|
||||
|
||||
ha-bindiface, ha-mcastport, vip
|
||||
|
||||
param: exclude_keys: list of setting key(s) to be excluded.
|
||||
returns: dict: A dict containing settings keyed by setting name.
|
||||
raises: HAIncompleteConfig if settings are missing.
|
||||
'''
|
||||
settings = ['ha-bindiface', 'ha-mcastport', 'vip']
|
||||
conf = {}
|
||||
for setting in settings:
|
||||
if exclude_keys and setting in exclude_keys:
|
||||
continue
|
||||
|
||||
conf[setting] = config_get(setting)
|
||||
missing = []
|
||||
[missing.append(s) for s, v in conf.iteritems() if v is None]
|
||||
[missing.append(s) for s, v in six.iteritems(conf) if v is None]
|
||||
if missing:
|
||||
log('Insufficient config data to configure hacluster.', level=ERROR)
|
||||
raise HAIncompleteConfig
|
||||
|
|
|
@ -0,0 +1,15 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
|
@ -1,15 +1,32 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import glob
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import six
|
||||
import socket
|
||||
|
||||
from functools import partial
|
||||
|
||||
from charmhelpers.core.hookenv import unit_get
|
||||
from charmhelpers.fetch import apt_install
|
||||
from charmhelpers.core.hookenv import (
|
||||
ERROR,
|
||||
log
|
||||
log,
|
||||
WARNING,
|
||||
)
|
||||
|
||||
try:
|
||||
|
@ -33,31 +50,28 @@ def _validate_cidr(network):
|
|||
network)
|
||||
|
||||
|
||||
def no_ip_found_error_out(network):
|
||||
errmsg = ("No IP address found in network: %s" % network)
|
||||
raise ValueError(errmsg)
|
||||
|
||||
|
||||
def get_address_in_network(network, fallback=None, fatal=False):
|
||||
"""
|
||||
Get an IPv4 or IPv6 address within the network from the host.
|
||||
"""Get an IPv4 or IPv6 address within the network from the host.
|
||||
|
||||
:param network (str): CIDR presentation format. For example,
|
||||
'192.168.1.0/24'.
|
||||
:param fallback (str): If no address is found, return fallback.
|
||||
:param fatal (boolean): If no address is found, fallback is not
|
||||
set and fatal is True then exit(1).
|
||||
|
||||
"""
|
||||
|
||||
def not_found_error_out():
|
||||
log("No IP address found in network: %s" % network,
|
||||
level=ERROR)
|
||||
sys.exit(1)
|
||||
|
||||
if network is None:
|
||||
if fallback is not None:
|
||||
return fallback
|
||||
|
||||
if fatal:
|
||||
no_ip_found_error_out(network)
|
||||
else:
|
||||
if fatal:
|
||||
not_found_error_out()
|
||||
else:
|
||||
return None
|
||||
return None
|
||||
|
||||
_validate_cidr(network)
|
||||
network = netaddr.IPNetwork(network)
|
||||
|
@ -69,6 +83,7 @@ def get_address_in_network(network, fallback=None, fatal=False):
|
|||
cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
|
||||
if cidr in network:
|
||||
return str(cidr.ip)
|
||||
|
||||
if network.version == 6 and netifaces.AF_INET6 in addresses:
|
||||
for addr in addresses[netifaces.AF_INET6]:
|
||||
if not addr['addr'].startswith('fe80'):
|
||||
|
@ -81,20 +96,20 @@ def get_address_in_network(network, fallback=None, fatal=False):
|
|||
return fallback
|
||||
|
||||
if fatal:
|
||||
not_found_error_out()
|
||||
no_ip_found_error_out(network)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def is_ipv6(address):
|
||||
'''Determine whether provided address is IPv6 or not'''
|
||||
"""Determine whether provided address is IPv6 or not."""
|
||||
try:
|
||||
address = netaddr.IPAddress(address)
|
||||
except netaddr.AddrFormatError:
|
||||
# probably a hostname - so not an address at all!
|
||||
return False
|
||||
else:
|
||||
return address.version == 6
|
||||
|
||||
return address.version == 6
|
||||
|
||||
|
||||
def is_address_in_network(network, address):
|
||||
|
@ -112,11 +127,13 @@ def is_address_in_network(network, address):
|
|||
except (netaddr.core.AddrFormatError, ValueError):
|
||||
raise ValueError("Network (%s) is not in CIDR presentation format" %
|
||||
network)
|
||||
|
||||
try:
|
||||
address = netaddr.IPAddress(address)
|
||||
except (netaddr.core.AddrFormatError, ValueError):
|
||||
raise ValueError("Address (%s) is not in correct presentation format" %
|
||||
address)
|
||||
|
||||
if address in network:
|
||||
return True
|
||||
else:
|
||||
|
@ -146,6 +163,7 @@ def _get_for_address(address, key):
|
|||
return iface
|
||||
else:
|
||||
return addresses[netifaces.AF_INET][0][key]
|
||||
|
||||
if address.version == 6 and netifaces.AF_INET6 in addresses:
|
||||
for addr in addresses[netifaces.AF_INET6]:
|
||||
if not addr['addr'].startswith('fe80'):
|
||||
|
@ -159,40 +177,42 @@ def _get_for_address(address, key):
|
|||
return str(cidr).split('/')[1]
|
||||
else:
|
||||
return addr[key]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
get_iface_for_address = partial(_get_for_address, key='iface')
|
||||
|
||||
|
||||
get_netmask_for_address = partial(_get_for_address, key='netmask')
|
||||
|
||||
|
||||
def format_ipv6_addr(address):
|
||||
"""
|
||||
IPv6 needs to be wrapped with [] in url link to parse correctly.
|
||||
"""If address is IPv6, wrap it in '[]' otherwise return None.
|
||||
|
||||
This is required by most configuration files when specifying IPv6
|
||||
addresses.
|
||||
"""
|
||||
if is_ipv6(address):
|
||||
address = "[%s]" % address
|
||||
else:
|
||||
address = None
|
||||
return "[%s]" % address
|
||||
|
||||
return address
|
||||
return None
|
||||
|
||||
|
||||
def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False,
|
||||
fatal=True, exc_list=None):
|
||||
"""
|
||||
Return the assigned IP address for a given interface, if any, or [].
|
||||
"""
|
||||
"""Return the assigned IP address for a given interface, if any."""
|
||||
# Extract nic if passed /dev/ethX
|
||||
if '/' in iface:
|
||||
iface = iface.split('/')[-1]
|
||||
|
||||
if not exc_list:
|
||||
exc_list = []
|
||||
|
||||
try:
|
||||
inet_num = getattr(netifaces, inet_type)
|
||||
except AttributeError:
|
||||
raise Exception('Unknown inet type ' + str(inet_type))
|
||||
raise Exception("Unknown inet type '%s'" % str(inet_type))
|
||||
|
||||
interfaces = netifaces.interfaces()
|
||||
if inc_aliases:
|
||||
|
@ -200,15 +220,18 @@ def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False,
|
|||
for _iface in interfaces:
|
||||
if iface == _iface or _iface.split(':')[0] == iface:
|
||||
ifaces.append(_iface)
|
||||
|
||||
if fatal and not ifaces:
|
||||
raise Exception("Invalid interface '%s'" % iface)
|
||||
|
||||
ifaces.sort()
|
||||
else:
|
||||
if iface not in interfaces:
|
||||
if fatal:
|
||||
raise Exception("%s not found " % (iface))
|
||||
raise Exception("Interface '%s' not found " % (iface))
|
||||
else:
|
||||
return []
|
||||
|
||||
else:
|
||||
ifaces = [iface]
|
||||
|
||||
|
@ -219,10 +242,13 @@ def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False,
|
|||
for entry in net_info[inet_num]:
|
||||
if 'addr' in entry and entry['addr'] not in exc_list:
|
||||
addresses.append(entry['addr'])
|
||||
|
||||
if fatal and not addresses:
|
||||
raise Exception("Interface '%s' doesn't have any %s addresses." %
|
||||
(iface, inet_type))
|
||||
return addresses
|
||||
|
||||
return sorted(addresses)
|
||||
|
||||
|
||||
get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET')
|
||||
|
||||
|
@ -239,6 +265,7 @@ def get_iface_from_addr(addr):
|
|||
raw = re.match(ll_key, _addr)
|
||||
if raw:
|
||||
_addr = raw.group(1)
|
||||
|
||||
if _addr == addr:
|
||||
log("Address '%s' is configured on iface '%s'" %
|
||||
(addr, iface))
|
||||
|
@ -249,8 +276,9 @@ def get_iface_from_addr(addr):
|
|||
|
||||
|
||||
def sniff_iface(f):
|
||||
"""If no iface provided, inject net iface inferred from unit private
|
||||
address.
|
||||
"""Ensure decorated function is called with a value for iface.
|
||||
|
||||
If no iface provided, inject net iface inferred from unit private address.
|
||||
"""
|
||||
def iface_sniffer(*args, **kwargs):
|
||||
if not kwargs.get('iface', None):
|
||||
|
@ -293,7 +321,7 @@ def get_ipv6_addr(iface=None, inc_aliases=False, fatal=True, exc_list=None,
|
|||
if global_addrs:
|
||||
# Make sure any found global addresses are not temporary
|
||||
cmd = ['ip', 'addr', 'show', iface]
|
||||
out = subprocess.check_output(cmd)
|
||||
out = subprocess.check_output(cmd).decode('UTF-8')
|
||||
if dynamic_only:
|
||||
key = re.compile("inet6 (.+)/[0-9]+ scope global dynamic.*")
|
||||
else:
|
||||
|
@ -315,33 +343,108 @@ def get_ipv6_addr(iface=None, inc_aliases=False, fatal=True, exc_list=None,
|
|||
return addrs
|
||||
|
||||
if fatal:
|
||||
raise Exception("Interface '%s' doesn't have a scope global "
|
||||
raise Exception("Interface '%s' does not have a scope global "
|
||||
"non-temporary ipv6 address." % iface)
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def get_bridges(vnic_dir='/sys/devices/virtual/net'):
|
||||
"""
|
||||
Return a list of bridges on the system or []
|
||||
"""
|
||||
b_rgex = vnic_dir + '/*/bridge'
|
||||
return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_rgex)]
|
||||
"""Return a list of bridges on the system."""
|
||||
b_regex = "%s/*/bridge" % vnic_dir
|
||||
return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_regex)]
|
||||
|
||||
|
||||
def get_bridge_nics(bridge, vnic_dir='/sys/devices/virtual/net'):
|
||||
"""
|
||||
Return a list of nics comprising a given bridge on the system or []
|
||||
"""
|
||||
brif_rgex = "%s/%s/brif/*" % (vnic_dir, bridge)
|
||||
return [x.split('/')[-1] for x in glob.glob(brif_rgex)]
|
||||
"""Return a list of nics comprising a given bridge on the system."""
|
||||
brif_regex = "%s/%s/brif/*" % (vnic_dir, bridge)
|
||||
return [x.split('/')[-1] for x in glob.glob(brif_regex)]
|
||||
|
||||
|
||||
def is_bridge_member(nic):
|
||||
"""
|
||||
Check if a given nic is a member of a bridge
|
||||
"""
|
||||
"""Check if a given nic is a member of a bridge."""
|
||||
for bridge in get_bridges():
|
||||
if nic in get_bridge_nics(bridge):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def is_ip(address):
|
||||
"""
|
||||
Returns True if address is a valid IP address.
|
||||
"""
|
||||
try:
|
||||
# Test to see if already an IPv4 address
|
||||
socket.inet_aton(address)
|
||||
return True
|
||||
except socket.error:
|
||||
return False
|
||||
|
||||
|
||||
def ns_query(address):
|
||||
try:
|
||||
import dns.resolver
|
||||
except ImportError:
|
||||
apt_install('python-dnspython')
|
||||
import dns.resolver
|
||||
|
||||
if isinstance(address, dns.name.Name):
|
||||
rtype = 'PTR'
|
||||
elif isinstance(address, six.string_types):
|
||||
rtype = 'A'
|
||||
else:
|
||||
return None
|
||||
|
||||
answers = dns.resolver.query(address, rtype)
|
||||
if answers:
|
||||
return str(answers[0])
|
||||
return None
|
||||
|
||||
|
||||
def get_host_ip(hostname, fallback=None):
|
||||
"""
|
||||
Resolves the IP for a given hostname, or returns
|
||||
the input if it is already an IP.
|
||||
"""
|
||||
if is_ip(hostname):
|
||||
return hostname
|
||||
|
||||
ip_addr = ns_query(hostname)
|
||||
if not ip_addr:
|
||||
try:
|
||||
ip_addr = socket.gethostbyname(hostname)
|
||||
except:
|
||||
log("Failed to resolve hostname '%s'" % (hostname),
|
||||
level=WARNING)
|
||||
return fallback
|
||||
return ip_addr
|
||||
|
||||
|
||||
def get_hostname(address, fqdn=True):
|
||||
"""
|
||||
Resolves hostname for given IP, or returns the input
|
||||
if it is already a hostname.
|
||||
"""
|
||||
if is_ip(address):
|
||||
try:
|
||||
import dns.reversename
|
||||
except ImportError:
|
||||
apt_install("python-dnspython")
|
||||
import dns.reversename
|
||||
|
||||
rev = dns.reversename.from_address(address)
|
||||
result = ns_query(rev)
|
||||
if not result:
|
||||
return None
|
||||
else:
|
||||
result = address
|
||||
|
||||
if fqdn:
|
||||
# strip trailing .
|
||||
if result.endswith('.'):
|
||||
return result[:-1]
|
||||
else:
|
||||
return result
|
||||
else:
|
||||
return result.split('.')[0]
|
||||
|
|
|
@ -1,3 +1,19 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
''' Helpers for interacting with OpenvSwitch '''
|
||||
import subprocess
|
||||
import os
|
||||
|
|
|
@ -0,0 +1,276 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
This module contains helpers to add and remove ufw rules.
|
||||
|
||||
Examples:
|
||||
|
||||
- open SSH port for subnet 10.0.3.0/24:
|
||||
|
||||
>>> from charmhelpers.contrib.network import ufw
|
||||
>>> ufw.enable()
|
||||
>>> ufw.grant_access(src='10.0.3.0/24', dst='any', port='22', proto='tcp')
|
||||
|
||||
- open service by name as defined in /etc/services:
|
||||
|
||||
>>> from charmhelpers.contrib.network import ufw
|
||||
>>> ufw.enable()
|
||||
>>> ufw.service('ssh', 'open')
|
||||
|
||||
- close service by port number:
|
||||
|
||||
>>> from charmhelpers.contrib.network import ufw
|
||||
>>> ufw.enable()
|
||||
>>> ufw.service('4949', 'close') # munin
|
||||
"""
|
||||
import re
|
||||
import os
|
||||
import subprocess
|
||||
from charmhelpers.core import hookenv
|
||||
|
||||
__author__ = "Felipe Reyes <felipe.reyes@canonical.com>"
|
||||
|
||||
|
||||
class UFWError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class UFWIPv6Error(UFWError):
|
||||
pass
|
||||
|
||||
|
||||
def is_enabled():
|
||||
"""
|
||||
Check if `ufw` is enabled
|
||||
|
||||
:returns: True if ufw is enabled
|
||||
"""
|
||||
output = subprocess.check_output(['ufw', 'status'],
|
||||
universal_newlines=True,
|
||||
env={'LANG': 'en_US',
|
||||
'PATH': os.environ['PATH']})
|
||||
|
||||
m = re.findall(r'^Status: active\n', output, re.M)
|
||||
|
||||
return len(m) >= 1
|
||||
|
||||
|
||||
def is_ipv6_ok(soft_fail=False):
|
||||
"""
|
||||
Check if IPv6 support is present and ip6tables functional
|
||||
|
||||
:param soft_fail: If set to True and IPv6 support is broken, then reports
|
||||
that the host doesn't have IPv6 support, otherwise a
|
||||
UFWIPv6Error exception is raised.
|
||||
:returns: True if IPv6 is working, False otherwise
|
||||
"""
|
||||
|
||||
# do we have IPv6 in the machine?
|
||||
if os.path.isdir('/proc/sys/net/ipv6'):
|
||||
# is ip6tables kernel module loaded?
|
||||
lsmod = subprocess.check_output(['lsmod'], universal_newlines=True)
|
||||
matches = re.findall('^ip6_tables[ ]+', lsmod, re.M)
|
||||
if len(matches) == 0:
|
||||
# ip6tables support isn't complete, let's try to load it
|
||||
try:
|
||||
subprocess.check_output(['modprobe', 'ip6_tables'],
|
||||
universal_newlines=True)
|
||||
# great, we could load the module
|
||||
return True
|
||||
except subprocess.CalledProcessError as ex:
|
||||
hookenv.log("Couldn't load ip6_tables module: %s" % ex.output,
|
||||
level="WARN")
|
||||
# we are in a world where ip6tables isn't working
|
||||
if soft_fail:
|
||||
# so we inform that the machine doesn't have IPv6
|
||||
return False
|
||||
else:
|
||||
raise UFWIPv6Error("IPv6 firewall support broken")
|
||||
else:
|
||||
# the module is present :)
|
||||
return True
|
||||
|
||||
else:
|
||||
# the system doesn't have IPv6
|
||||
return False
|
||||
|
||||
|
||||
def disable_ipv6():
|
||||
"""
|
||||
Disable ufw IPv6 support in /etc/default/ufw
|
||||
"""
|
||||
exit_code = subprocess.call(['sed', '-i', 's/IPV6=.*/IPV6=no/g',
|
||||
'/etc/default/ufw'])
|
||||
if exit_code == 0:
|
||||
hookenv.log('IPv6 support in ufw disabled', level='INFO')
|
||||
else:
|
||||
hookenv.log("Couldn't disable IPv6 support in ufw", level="ERROR")
|
||||
raise UFWError("Couldn't disable IPv6 support in ufw")
|
||||
|
||||
|
||||
def enable(soft_fail=False):
|
||||
"""
|
||||
Enable ufw
|
||||
|
||||
:param soft_fail: If set to True silently disables IPv6 support in ufw,
|
||||
otherwise a UFWIPv6Error exception is raised when IP6
|
||||
support is broken.
|
||||
:returns: True if ufw is successfully enabled
|
||||
"""
|
||||
if is_enabled():
|
||||
return True
|
||||
|
||||
if not is_ipv6_ok(soft_fail):
|
||||
disable_ipv6()
|
||||
|
||||
output = subprocess.check_output(['ufw', 'enable'],
|
||||
universal_newlines=True,
|
||||
env={'LANG': 'en_US',
|
||||
'PATH': os.environ['PATH']})
|
||||
|
||||
m = re.findall('^Firewall is active and enabled on system startup\n',
|
||||
output, re.M)
|
||||
hookenv.log(output, level='DEBUG')
|
||||
|
||||
if len(m) == 0:
|
||||
hookenv.log("ufw couldn't be enabled", level='WARN')
|
||||
return False
|
||||
else:
|
||||
hookenv.log("ufw enabled", level='INFO')
|
||||
return True
|
||||
|
||||
|
||||
def disable():
|
||||
"""
|
||||
Disable ufw
|
||||
|
||||
:returns: True if ufw is successfully disabled
|
||||
"""
|
||||
if not is_enabled():
|
||||
return True
|
||||
|
||||
output = subprocess.check_output(['ufw', 'disable'],
|
||||
universal_newlines=True,
|
||||
env={'LANG': 'en_US',
|
||||
'PATH': os.environ['PATH']})
|
||||
|
||||
m = re.findall(r'^Firewall stopped and disabled on system startup\n',
|
||||
output, re.M)
|
||||
hookenv.log(output, level='DEBUG')
|
||||
|
||||
if len(m) == 0:
|
||||
hookenv.log("ufw couldn't be disabled", level='WARN')
|
||||
return False
|
||||
else:
|
||||
hookenv.log("ufw disabled", level='INFO')
|
||||
return True
|
||||
|
||||
|
||||
def modify_access(src, dst='any', port=None, proto=None, action='allow'):
|
||||
"""
|
||||
Grant access to an address or subnet
|
||||
|
||||
:param src: address (e.g. 192.168.1.234) or subnet
|
||||
(e.g. 192.168.1.0/24).
|
||||
:param dst: destiny of the connection, if the machine has multiple IPs and
|
||||
connections to only one of those have to accepted this is the
|
||||
field has to be set.
|
||||
:param port: destiny port
|
||||
:param proto: protocol (tcp or udp)
|
||||
:param action: `allow` or `delete`
|
||||
"""
|
||||
if not is_enabled():
|
||||
hookenv.log('ufw is disabled, skipping modify_access()', level='WARN')
|
||||
return
|
||||
|
||||
if action == 'delete':
|
||||
cmd = ['ufw', 'delete', 'allow']
|
||||
else:
|
||||
cmd = ['ufw', action]
|
||||
|
||||
if src is not None:
|
||||
cmd += ['from', src]
|
||||
|
||||
if dst is not None:
|
||||
cmd += ['to', dst]
|
||||
|
||||
if port is not None:
|
||||
cmd += ['port', str(port)]
|
||||
|
||||
if proto is not None:
|
||||
cmd += ['proto', proto]
|
||||
|
||||
hookenv.log('ufw {}: {}'.format(action, ' '.join(cmd)), level='DEBUG')
|
||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
|
||||
(stdout, stderr) = p.communicate()
|
||||
|
||||
hookenv.log(stdout, level='INFO')
|
||||
|
||||
if p.returncode != 0:
|
||||
hookenv.log(stderr, level='ERROR')
|
||||
hookenv.log('Error running: {}, exit code: {}'.format(' '.join(cmd),
|
||||
p.returncode),
|
||||
level='ERROR')
|
||||
|
||||
|
||||
def grant_access(src, dst='any', port=None, proto=None):
|
||||
"""
|
||||
Grant access to an address or subnet
|
||||
|
||||
:param src: address (e.g. 192.168.1.234) or subnet
|
||||
(e.g. 192.168.1.0/24).
|
||||
:param dst: destiny of the connection, if the machine has multiple IPs and
|
||||
connections to only one of those have to accepted this is the
|
||||
field has to be set.
|
||||
:param port: destiny port
|
||||
:param proto: protocol (tcp or udp)
|
||||
"""
|
||||
return modify_access(src, dst=dst, port=port, proto=proto, action='allow')
|
||||
|
||||
|
||||
def revoke_access(src, dst='any', port=None, proto=None):
|
||||
"""
|
||||
Revoke access to an address or subnet
|
||||
|
||||
:param src: address (e.g. 192.168.1.234) or subnet
|
||||
(e.g. 192.168.1.0/24).
|
||||
:param dst: destiny of the connection, if the machine has multiple IPs and
|
||||
connections to only one of those have to accepted this is the
|
||||
field has to be set.
|
||||
:param port: destiny port
|
||||
:param proto: protocol (tcp or udp)
|
||||
"""
|
||||
return modify_access(src, dst=dst, port=port, proto=proto, action='delete')
|
||||
|
||||
|
||||
def service(name, action):
|
||||
"""
|
||||
Open/close access to a service
|
||||
|
||||
:param name: could be a service name defined in `/etc/services` or a port
|
||||
number.
|
||||
:param action: `open` or `close`
|
||||
"""
|
||||
if action == 'open':
|
||||
subprocess.check_output(['ufw', 'allow', str(name)],
|
||||
universal_newlines=True)
|
||||
elif action == 'close':
|
||||
subprocess.check_output(['ufw', 'delete', 'allow', str(name)],
|
||||
universal_newlines=True)
|
||||
else:
|
||||
raise UFWError(("'{}' not supported, use 'allow' "
|
||||
"or 'delete'").format(action))
|
|
@ -0,0 +1,15 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
|
@ -1,3 +1,19 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
''' Helper for managing alternatives for file conflict resolution '''
|
||||
|
||||
import subprocess
|
||||
|
|
|
@ -0,0 +1,15 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
|
@ -1,3 +1,21 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import six
|
||||
from collections import OrderedDict
|
||||
from charmhelpers.contrib.amulet.deployment import (
|
||||
AmuletDeployment
|
||||
)
|
||||
|
@ -26,17 +44,24 @@ class OpenStackAmuletDeployment(AmuletDeployment):
|
|||
Determine if the local branch being tested is derived from its
|
||||
stable or next (dev) branch, and based on this, use the corresonding
|
||||
stable or next branches for the other_services."""
|
||||
base_charms = ['mysql', 'mongodb', 'rabbitmq-server']
|
||||
base_charms = ['mysql', 'mongodb']
|
||||
|
||||
if self.series in ['precise', 'trusty']:
|
||||
base_series = self.series
|
||||
else:
|
||||
base_series = self.current_next
|
||||
|
||||
if self.stable:
|
||||
for svc in other_services:
|
||||
temp = 'lp:charms/{}'
|
||||
svc['location'] = temp.format(svc['name'])
|
||||
temp = 'lp:charms/{}/{}'
|
||||
svc['location'] = temp.format(base_series,
|
||||
svc['name'])
|
||||
else:
|
||||
for svc in other_services:
|
||||
if svc['name'] in base_charms:
|
||||
temp = 'lp:charms/{}'
|
||||
svc['location'] = temp.format(svc['name'])
|
||||
temp = 'lp:charms/{}/{}'
|
||||
svc['location'] = temp.format(base_series,
|
||||
svc['name'])
|
||||
else:
|
||||
temp = 'lp:~openstack-charmers/charms/{}/{}/next'
|
||||
svc['location'] = temp.format(self.current_next,
|
||||
|
@ -54,22 +79,25 @@ class OpenStackAmuletDeployment(AmuletDeployment):
|
|||
services.append(this_service)
|
||||
use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
|
||||
'ceph-osd', 'ceph-radosgw']
|
||||
# Openstack subordinate charms do not expose an origin option as that
|
||||
# is controlled by the principle
|
||||
ignore = ['neutron-openvswitch']
|
||||
|
||||
if self.openstack:
|
||||
for svc in services:
|
||||
if svc['name'] not in use_source:
|
||||
if svc['name'] not in use_source + ignore:
|
||||
config = {'openstack-origin': self.openstack}
|
||||
self.d.configure(svc['name'], config)
|
||||
|
||||
if self.source:
|
||||
for svc in services:
|
||||
if svc['name'] in use_source:
|
||||
if svc['name'] in use_source and svc['name'] not in ignore:
|
||||
config = {'source': self.source}
|
||||
self.d.configure(svc['name'], config)
|
||||
|
||||
def _configure_services(self, configs):
|
||||
"""Configure all of the services."""
|
||||
for service, config in configs.iteritems():
|
||||
for service, config in six.iteritems(configs):
|
||||
self.d.configure(service, config)
|
||||
|
||||
def _get_openstack_release(self):
|
||||
|
@ -78,14 +106,41 @@ class OpenStackAmuletDeployment(AmuletDeployment):
|
|||
Return an integer representing the enum value of the openstack
|
||||
release.
|
||||
"""
|
||||
# Must be ordered by OpenStack release (not by Ubuntu release):
|
||||
(self.precise_essex, self.precise_folsom, self.precise_grizzly,
|
||||
self.precise_havana, self.precise_icehouse,
|
||||
self.trusty_icehouse) = range(6)
|
||||
self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
|
||||
self.trusty_kilo, self.vivid_kilo) = range(10)
|
||||
|
||||
releases = {
|
||||
('precise', None): self.precise_essex,
|
||||
('precise', 'cloud:precise-folsom'): self.precise_folsom,
|
||||
('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
|
||||
('precise', 'cloud:precise-havana'): self.precise_havana,
|
||||
('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
|
||||
('trusty', None): self.trusty_icehouse}
|
||||
('trusty', None): self.trusty_icehouse,
|
||||
('trusty', 'cloud:trusty-juno'): self.trusty_juno,
|
||||
('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
|
||||
('utopic', None): self.utopic_juno,
|
||||
('vivid', None): self.vivid_kilo}
|
||||
return releases[(self.series, self.openstack)]
|
||||
|
||||
def _get_openstack_release_string(self):
|
||||
"""Get openstack release string.
|
||||
|
||||
Return a string representing the openstack release.
|
||||
"""
|
||||
releases = OrderedDict([
|
||||
('precise', 'essex'),
|
||||
('quantal', 'folsom'),
|
||||
('raring', 'grizzly'),
|
||||
('saucy', 'havana'),
|
||||
('trusty', 'icehouse'),
|
||||
('utopic', 'juno'),
|
||||
('vivid', 'kilo'),
|
||||
])
|
||||
if self.openstack:
|
||||
os_origin = self.openstack.split(':')[1]
|
||||
return os_origin.split('%s-' % self.series)[1].split('/')[0]
|
||||
else:
|
||||
return releases[self.series]
|
||||
|
|
|
@ -1,3 +1,19 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
@ -7,6 +23,8 @@ import glanceclient.v1.client as glance_client
|
|||
import keystoneclient.v2_0 as keystone_client
|
||||
import novaclient.v1_1.client as nova_client
|
||||
|
||||
import six
|
||||
|
||||
from charmhelpers.contrib.amulet.utils import (
|
||||
AmuletUtils
|
||||
)
|
||||
|
@ -60,7 +78,7 @@ class OpenStackAmuletUtils(AmuletUtils):
|
|||
expected service catalog endpoints.
|
||||
"""
|
||||
self.log.debug('actual: {}'.format(repr(actual)))
|
||||
for k, v in expected.iteritems():
|
||||
for k, v in six.iteritems(expected):
|
||||
if k in actual:
|
||||
ret = self._validate_dict_data(expected[k][0], actual[k][0])
|
||||
if ret:
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,18 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# dummy __init__.py to fool syncer into thinking this is a syncable python
|
||||
# module
|
|
@ -0,0 +1,32 @@
|
|||
#!/bin/bash
|
||||
#--------------------------------------------
|
||||
# This file is managed by Juju
|
||||
#--------------------------------------------
|
||||
#
|
||||
# Copyright 2009,2012 Canonical Ltd.
|
||||
# Author: Tom Haddon
|
||||
|
||||
CRITICAL=0
|
||||
NOTACTIVE=''
|
||||
LOGFILE=/var/log/nagios/check_haproxy.log
|
||||
AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}')
|
||||
|
||||
for appserver in $(grep ' server' /etc/haproxy/haproxy.cfg | awk '{print $2'});
|
||||
do
|
||||
output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 --regex="class=\"(active|backup)(2|3).*${appserver}" -e ' 200 OK')
|
||||
if [ $? != 0 ]; then
|
||||
date >> $LOGFILE
|
||||
echo $output >> $LOGFILE
|
||||
/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -v | grep $appserver >> $LOGFILE 2>&1
|
||||
CRITICAL=1
|
||||
NOTACTIVE="${NOTACTIVE} $appserver"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $CRITICAL = 1 ]; then
|
||||
echo "CRITICAL:${NOTACTIVE}"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo "OK: All haproxy instances looking good"
|
||||
exit 0
|
|
@ -0,0 +1,30 @@
|
|||
#!/bin/bash
|
||||
#--------------------------------------------
|
||||
# This file is managed by Juju
|
||||
#--------------------------------------------
|
||||
#
|
||||
# Copyright 2009,2012 Canonical Ltd.
|
||||
# Author: Tom Haddon
|
||||
|
||||
# These should be config options at some stage
|
||||
CURRQthrsh=0
|
||||
MAXQthrsh=100
|
||||
|
||||
AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}')
|
||||
|
||||
HAPROXYSTATS=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' -v)
|
||||
|
||||
for BACKEND in $(echo $HAPROXYSTATS| xargs -n1 | grep BACKEND | awk -F , '{print $1}')
|
||||
do
|
||||
CURRQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 3)
|
||||
MAXQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 4)
|
||||
|
||||
if [[ $CURRQ -gt $CURRQthrsh || $MAXQ -gt $MAXQthrsh ]] ; then
|
||||
echo "CRITICAL: queue depth for $BACKEND - CURRENT:$CURRQ MAX:$MAXQ"
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
echo "OK: All haproxy queue depths looking good"
|
||||
exit 0
|
||||
|
|
@ -1,22 +1,38 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from charmhelpers.core.hookenv import (
|
||||
config,
|
||||
unit_get,
|
||||
)
|
||||
|
||||
from charmhelpers.contrib.network.ip import (
|
||||
get_address_in_network,
|
||||
is_address_in_network,
|
||||
is_ipv6,
|
||||
get_ipv6_addr,
|
||||
)
|
||||
|
||||
from charmhelpers.contrib.hahelpers.cluster import is_clustered
|
||||
|
||||
from functools import partial
|
||||
|
||||
PUBLIC = 'public'
|
||||
INTERNAL = 'int'
|
||||
ADMIN = 'admin'
|
||||
|
||||
_address_map = {
|
||||
ADDRESS_MAP = {
|
||||
PUBLIC: {
|
||||
'config': 'os-public-network',
|
||||
'fallback': 'public-address'
|
||||
|
@ -33,16 +49,14 @@ _address_map = {
|
|||
|
||||
|
||||
def canonical_url(configs, endpoint_type=PUBLIC):
|
||||
'''
|
||||
Returns the correct HTTP URL to this host given the state of HTTPS
|
||||
"""Returns the correct HTTP URL to this host given the state of HTTPS
|
||||
configuration, hacluster and charm configuration.
|
||||
|
||||
:configs OSTemplateRenderer: A config tempating object to inspect for
|
||||
a complete https context.
|
||||
:endpoint_type str: The endpoint type to resolve.
|
||||
|
||||
:returns str: Base URL for services on the current service unit.
|
||||
'''
|
||||
:param configs: OSTemplateRenderer config templating object to inspect
|
||||
for a complete https context.
|
||||
:param endpoint_type: str endpoint type to resolve.
|
||||
:param returns: str base URL for services on the current service unit.
|
||||
"""
|
||||
scheme = 'http'
|
||||
if 'https' in configs.complete_contexts():
|
||||
scheme = 'https'
|
||||
|
@ -53,27 +67,80 @@ def canonical_url(configs, endpoint_type=PUBLIC):
|
|||
|
||||
|
||||
def resolve_address(endpoint_type=PUBLIC):
|
||||
"""Return unit address depending on net config.
|
||||
|
||||
If unit is clustered with vip(s) and has net splits defined, return vip on
|
||||
correct network. If clustered with no nets defined, return primary vip.
|
||||
|
||||
If not clustered, return unit address ensuring address is on configured net
|
||||
split if one is configured.
|
||||
|
||||
:param endpoint_type: Network endpoing type
|
||||
"""
|
||||
resolved_address = None
|
||||
if is_clustered():
|
||||
if config(_address_map[endpoint_type]['config']) is None:
|
||||
# Assume vip is simple and pass back directly
|
||||
resolved_address = config('vip')
|
||||
vips = config('vip')
|
||||
if vips:
|
||||
vips = vips.split()
|
||||
|
||||
net_type = ADDRESS_MAP[endpoint_type]['config']
|
||||
net_addr = config(net_type)
|
||||
net_fallback = ADDRESS_MAP[endpoint_type]['fallback']
|
||||
clustered = is_clustered()
|
||||
if clustered:
|
||||
if not net_addr:
|
||||
# If no net-splits defined, we expect a single vip
|
||||
resolved_address = vips[0]
|
||||
else:
|
||||
for vip in config('vip').split():
|
||||
if is_address_in_network(
|
||||
config(_address_map[endpoint_type]['config']),
|
||||
vip):
|
||||
for vip in vips:
|
||||
if is_address_in_network(net_addr, vip):
|
||||
resolved_address = vip
|
||||
break
|
||||
else:
|
||||
if config('prefer-ipv6'):
|
||||
fallback_addr = get_ipv6_addr(exc_list=[config('vip')])[0]
|
||||
fallback_addr = get_ipv6_addr(exc_list=vips)[0]
|
||||
else:
|
||||
fallback_addr = unit_get(_address_map[endpoint_type]['fallback'])
|
||||
resolved_address = get_address_in_network(
|
||||
config(_address_map[endpoint_type]['config']), fallback_addr)
|
||||
fallback_addr = unit_get(net_fallback)
|
||||
|
||||
resolved_address = get_address_in_network(net_addr, fallback_addr)
|
||||
|
||||
if resolved_address is None:
|
||||
raise ValueError('Unable to resolve a suitable IP address'
|
||||
' based on charm state and configuration')
|
||||
else:
|
||||
return resolved_address
|
||||
raise ValueError("Unable to resolve a suitable IP address based on "
|
||||
"charm state and configuration. (net_type=%s, "
|
||||
"clustered=%s)" % (net_type, clustered))
|
||||
|
||||
return resolved_address
|
||||
|
||||
|
||||
def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC,
|
||||
override=None):
|
||||
"""Returns the correct endpoint URL to advertise to Keystone.
|
||||
|
||||
This method provides the correct endpoint URL which should be advertised to
|
||||
the keystone charm for endpoint creation. This method allows for the url to
|
||||
be overridden to force a keystone endpoint to have specific URL for any of
|
||||
the defined scopes (admin, internal, public).
|
||||
|
||||
:param configs: OSTemplateRenderer config templating object to inspect
|
||||
for a complete https context.
|
||||
:param url_template: str format string for creating the url template. Only
|
||||
two values will be passed - the scheme+hostname
|
||||
returned by the canonical_url and the port.
|
||||
:param endpoint_type: str endpoint type to resolve.
|
||||
:param override: str the name of the config option which overrides the
|
||||
endpoint URL defined by the charm itself. None will
|
||||
disable any overrides (default).
|
||||
"""
|
||||
if override:
|
||||
# Return any user-defined overrides for the keystone endpoint URL.
|
||||
user_value = config(override)
|
||||
if user_value:
|
||||
return user_value.strip()
|
||||
|
||||
return url_template % (canonical_url(configs, endpoint_type), port)
|
||||
|
||||
|
||||
public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC)
|
||||
|
||||
internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL)
|
||||
|
||||
admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN)
|
||||
|
|
|
@ -1,5 +1,22 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# Various utilies for dealing with Neutron and the renaming from Quantum.
|
||||
|
||||
import six
|
||||
from subprocess import check_output
|
||||
|
||||
from charmhelpers.core.hookenv import (
|
||||
|
@ -14,7 +31,7 @@ from charmhelpers.contrib.openstack.utils import os_release
|
|||
def headers_package():
|
||||
"""Ensures correct linux-headers for running kernel are installed,
|
||||
for building DKMS package"""
|
||||
kver = check_output(['uname', '-r']).strip()
|
||||
kver = check_output(['uname', '-r']).decode('UTF-8').strip()
|
||||
return 'linux-headers-%s' % kver
|
||||
|
||||
QUANTUM_CONF_DIR = '/etc/quantum'
|
||||
|
@ -22,7 +39,7 @@ QUANTUM_CONF_DIR = '/etc/quantum'
|
|||
|
||||
def kernel_version():
|
||||
""" Retrieve the current major kernel version as a tuple e.g. (3, 13) """
|
||||
kver = check_output(['uname', '-r']).strip()
|
||||
kver = check_output(['uname', '-r']).decode('UTF-8').strip()
|
||||
kver = kver.split('.')
|
||||
return (int(kver[0]), int(kver[1]))
|
||||
|
||||
|
@ -152,11 +169,30 @@ def neutron_plugins():
|
|||
database=config('neutron-database'),
|
||||
relation_prefix='neutron',
|
||||
ssl_dir=NEUTRON_CONF_DIR)],
|
||||
'services': ['calico-compute', 'bird', 'neutron-dhcp-agent'],
|
||||
'services': ['calico-felix',
|
||||
'bird',
|
||||
'neutron-dhcp-agent',
|
||||
'nova-api-metadata'],
|
||||
'packages': [[headers_package()] + determine_dkms_package(),
|
||||
['calico-compute', 'bird', 'neutron-dhcp-agent']],
|
||||
['calico-compute',
|
||||
'bird',
|
||||
'neutron-dhcp-agent',
|
||||
'nova-api-metadata']],
|
||||
'server_packages': ['neutron-server', 'calico-control'],
|
||||
'server_services': ['neutron-server']
|
||||
},
|
||||
'vsp': {
|
||||
'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini',
|
||||
'driver': 'neutron.plugins.nuage.plugin.NuagePlugin',
|
||||
'contexts': [
|
||||
context.SharedDBContext(user=config('neutron-database-user'),
|
||||
database=config('neutron-database'),
|
||||
relation_prefix='neutron',
|
||||
ssl_dir=NEUTRON_CONF_DIR)],
|
||||
'services': [],
|
||||
'packages': [],
|
||||
'server_packages': ['neutron-server', 'neutron-plugin-nuage'],
|
||||
'server_services': ['neutron-server']
|
||||
}
|
||||
}
|
||||
if release >= 'icehouse':
|
||||
|
@ -177,7 +213,8 @@ def neutron_plugin_attribute(plugin, attr, net_manager=None):
|
|||
elif manager == 'neutron':
|
||||
plugins = neutron_plugins()
|
||||
else:
|
||||
log('Error: Network manager does not support plugins.')
|
||||
log("Network manager '%s' does not support plugins." % (manager),
|
||||
level=ERROR)
|
||||
raise Exception
|
||||
|
||||
try:
|
||||
|
@ -214,3 +251,77 @@ def network_manager():
|
|||
else:
|
||||
# ensure accurate naming for all releases post-H
|
||||
return 'neutron'
|
||||
|
||||
|
||||
def parse_mappings(mappings):
|
||||
parsed = {}
|
||||
if mappings:
|
||||
mappings = mappings.split()
|
||||
for m in mappings:
|
||||
p = m.partition(':')
|
||||
key = p[0].strip()
|
||||
if p[1]:
|
||||
parsed[key] = p[2].strip()
|
||||
else:
|
||||
parsed[key] = ''
|
||||
|
||||
return parsed
|
||||
|
||||
|
||||
def parse_bridge_mappings(mappings):
|
||||
"""Parse bridge mappings.
|
||||
|
||||
Mappings must be a space-delimited list of provider:bridge mappings.
|
||||
|
||||
Returns dict of the form {provider:bridge}.
|
||||
"""
|
||||
return parse_mappings(mappings)
|
||||
|
||||
|
||||
def parse_data_port_mappings(mappings, default_bridge='br-data'):
|
||||
"""Parse data port mappings.
|
||||
|
||||
Mappings must be a space-delimited list of bridge:port mappings.
|
||||
|
||||
Returns dict of the form {bridge:port}.
|
||||
"""
|
||||
_mappings = parse_mappings(mappings)
|
||||
if not _mappings or list(_mappings.values()) == ['']:
|
||||
if not mappings:
|
||||
return {}
|
||||
|
||||
# For backwards-compatibility we need to support port-only provided in
|
||||
# config.
|
||||
_mappings = {default_bridge: mappings.split()[0]}
|
||||
|
||||
bridges = _mappings.keys()
|
||||
ports = _mappings.values()
|
||||
if len(set(bridges)) != len(bridges):
|
||||
raise Exception("It is not allowed to have more than one port "
|
||||
"configured on the same bridge")
|
||||
|
||||
if len(set(ports)) != len(ports):
|
||||
raise Exception("It is not allowed to have the same port configured "
|
||||
"on more than one bridge")
|
||||
|
||||
return _mappings
|
||||
|
||||
|
||||
def parse_vlan_range_mappings(mappings):
|
||||
"""Parse vlan range mappings.
|
||||
|
||||
Mappings must be a space-delimited list of provider:start:end mappings.
|
||||
|
||||
The start:end range is optional and may be omitted.
|
||||
|
||||
Returns dict of the form {provider: (start, end)}.
|
||||
"""
|
||||
_mappings = parse_mappings(mappings)
|
||||
if not _mappings:
|
||||
return {}
|
||||
|
||||
mappings = {}
|
||||
for p, r in six.iteritems(_mappings):
|
||||
mappings[p] = tuple(r.split(':'))
|
||||
|
||||
return mappings
|
||||
|
|
|
@ -1,2 +1,18 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# dummy __init__.py to fool syncer into thinking this is a syncable python
|
||||
# module
|
||||
|
|
|
@ -0,0 +1,17 @@
|
|||
description "{{ service_description }}"
|
||||
author "Juju {{ service_name }} Charm <juju@localhost>"
|
||||
|
||||
start on runlevel [2345]
|
||||
stop on runlevel [!2345]
|
||||
|
||||
respawn
|
||||
|
||||
exec start-stop-daemon --start --chuid {{ user_name }} \
|
||||
--chdir {{ start_dir }} --name {{ process_name }} \
|
||||
--exec {{ executable_name }} -- \
|
||||
{% for config_file in config_files -%}
|
||||
--config-file={{ config_file }} \
|
||||
{% endfor -%}
|
||||
{% if log_file -%}
|
||||
--log-file={{ log_file }}
|
||||
{% endif -%}
|
|
@ -35,18 +35,22 @@ listen stats {{ stat_port }}
|
|||
stats auth admin:password
|
||||
|
||||
{% if frontends -%}
|
||||
{% for service, ports in service_ports.iteritems() -%}
|
||||
{% for service, ports in service_ports.items() -%}
|
||||
frontend tcp-in_{{ service }}
|
||||
bind *:{{ ports[0] }}
|
||||
{% if ipv6 -%}
|
||||
bind :::{{ ports[0] }}
|
||||
{% endif -%}
|
||||
{% for frontend in frontends -%}
|
||||
acl net_{{ frontend }} dst {{ frontends[frontend]['network'] }}
|
||||
use_backend {{ service }}_{{ frontend }} if net_{{ frontend }}
|
||||
{% endfor %}
|
||||
{% endfor -%}
|
||||
default_backend {{ service }}_{{ default_backend }}
|
||||
|
||||
{% for frontend in frontends -%}
|
||||
backend {{ service }}_{{ frontend }}
|
||||
balance leastconn
|
||||
{% for unit, address in frontends[frontend]['backends'].iteritems() -%}
|
||||
{% for unit, address in frontends[frontend]['backends'].items() -%}
|
||||
server {{ unit }} {{ address }}:{{ ports[1] }} check
|
||||
{% endfor %}
|
||||
{% endfor -%}
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
{% if auth_host -%}
|
||||
[keystone_authtoken]
|
||||
identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/{{ auth_admin_prefix }}
|
||||
auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }}
|
||||
admin_tenant_name = {{ admin_tenant_name }}
|
||||
admin_user = {{ admin_user }}
|
||||
admin_password = {{ admin_password }}
|
||||
signing_dir = {{ signing_dir }}
|
||||
{% endif -%}
|
|
@ -0,0 +1,22 @@
|
|||
{% if rabbitmq_host or rabbitmq_hosts -%}
|
||||
[oslo_messaging_rabbit]
|
||||
rabbit_userid = {{ rabbitmq_user }}
|
||||
rabbit_virtual_host = {{ rabbitmq_virtual_host }}
|
||||
rabbit_password = {{ rabbitmq_password }}
|
||||
{% if rabbitmq_hosts -%}
|
||||
rabbit_hosts = {{ rabbitmq_hosts }}
|
||||
{% if rabbitmq_ha_queues -%}
|
||||
rabbit_ha_queues = True
|
||||
rabbit_durable_queues = False
|
||||
{% endif -%}
|
||||
{% else -%}
|
||||
rabbit_host = {{ rabbitmq_host }}
|
||||
{% endif -%}
|
||||
{% if rabbit_ssl_port -%}
|
||||
rabbit_use_ssl = True
|
||||
rabbit_port = {{ rabbit_ssl_port }}
|
||||
{% if rabbit_ssl_ca -%}
|
||||
kombu_ssl_ca_certs = {{ rabbit_ssl_ca }}
|
||||
{% endif -%}
|
||||
{% endif -%}
|
||||
{% endif -%}
|
|
@ -0,0 +1,14 @@
|
|||
{% if zmq_host -%}
|
||||
# ZeroMQ configuration (restart-nonce: {{ zmq_nonce }})
|
||||
rpc_backend = zmq
|
||||
rpc_zmq_host = {{ zmq_host }}
|
||||
{% if zmq_redis_address -%}
|
||||
rpc_zmq_matchmaker = redis
|
||||
matchmaker_heartbeat_freq = 15
|
||||
matchmaker_heartbeat_ttl = 30
|
||||
[matchmaker_redis]
|
||||
host = {{ zmq_redis_address }}
|
||||
{% else -%}
|
||||
rpc_zmq_matchmaker = ring
|
||||
{% endif -%}
|
||||
{% endif -%}
|
|
@ -1,13 +1,29 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
|
||||
from charmhelpers.fetch import apt_install
|
||||
import six
|
||||
|
||||
from charmhelpers.fetch import apt_install
|
||||
from charmhelpers.core.hookenv import (
|
||||
log,
|
||||
ERROR,
|
||||
INFO
|
||||
)
|
||||
|
||||
from charmhelpers.contrib.openstack.utils import OPENSTACK_CODENAMES
|
||||
|
||||
try:
|
||||
|
@ -43,7 +59,7 @@ def get_loader(templates_dir, os_release):
|
|||
order by OpenStack release.
|
||||
"""
|
||||
tmpl_dirs = [(rel, os.path.join(templates_dir, rel))
|
||||
for rel in OPENSTACK_CODENAMES.itervalues()]
|
||||
for rel in six.itervalues(OPENSTACK_CODENAMES)]
|
||||
|
||||
if not os.path.isdir(templates_dir):
|
||||
log('Templates directory not found @ %s.' % templates_dir,
|
||||
|
@ -258,7 +274,7 @@ class OSConfigRenderer(object):
|
|||
"""
|
||||
Write out all registered config files.
|
||||
"""
|
||||
[self.write(k) for k in self.templates.iterkeys()]
|
||||
[self.write(k) for k in six.iterkeys(self.templates)]
|
||||
|
||||
def set_release(self, openstack_release):
|
||||
"""
|
||||
|
@ -275,5 +291,5 @@ class OSConfigRenderer(object):
|
|||
'''
|
||||
interfaces = []
|
||||
[interfaces.extend(i.complete_contexts())
|
||||
for i in self.templates.itervalues()]
|
||||
for i in six.itervalues(self.templates)]
|
||||
return interfaces
|
||||
|
|
|
@ -1,5 +1,21 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# Common python helper functions used for OpenStack charms.
|
||||
from collections import OrderedDict
|
||||
from functools import wraps
|
||||
|
@ -7,14 +23,21 @@ from functools import wraps
|
|||
import subprocess
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
|
||||
import six
|
||||
import yaml
|
||||
|
||||
from charmhelpers.contrib.network import ip
|
||||
|
||||
from charmhelpers.core import (
|
||||
unitdata,
|
||||
)
|
||||
|
||||
from charmhelpers.core.hookenv import (
|
||||
config,
|
||||
log as juju_log,
|
||||
charm_dir,
|
||||
ERROR,
|
||||
INFO,
|
||||
relation_ids,
|
||||
relation_set
|
||||
|
@ -30,8 +53,13 @@ from charmhelpers.contrib.network.ip import (
|
|||
get_ipv6_addr
|
||||
)
|
||||
|
||||
from charmhelpers.contrib.python.packages import (
|
||||
pip_create_virtualenv,
|
||||
pip_install,
|
||||
)
|
||||
|
||||
from charmhelpers.core.host import lsb_release, mounts, umount
|
||||
from charmhelpers.fetch import apt_install, apt_cache
|
||||
from charmhelpers.fetch import apt_install, apt_cache, install_remote
|
||||
from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
|
||||
from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
|
||||
|
||||
|
@ -50,6 +78,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
|
|||
('saucy', 'havana'),
|
||||
('trusty', 'icehouse'),
|
||||
('utopic', 'juno'),
|
||||
('vivid', 'kilo'),
|
||||
])
|
||||
|
||||
|
||||
|
@ -61,6 +90,7 @@ OPENSTACK_CODENAMES = OrderedDict([
|
|||
('2013.2', 'havana'),
|
||||
('2014.1', 'icehouse'),
|
||||
('2014.2', 'juno'),
|
||||
('2015.1', 'kilo'),
|
||||
])
|
||||
|
||||
# The ugly duckling
|
||||
|
@ -81,6 +111,8 @@ SWIFT_CODENAMES = OrderedDict([
|
|||
('2.0.0', 'juno'),
|
||||
('2.1.0', 'juno'),
|
||||
('2.2.0', 'juno'),
|
||||
('2.2.1', 'kilo'),
|
||||
('2.2.2', 'kilo'),
|
||||
])
|
||||
|
||||
DEFAULT_LOOPBACK_SIZE = '5G'
|
||||
|
@ -113,7 +145,7 @@ def get_os_codename_install_source(src):
|
|||
|
||||
# Best guess match based on deb string provided
|
||||
if src.startswith('deb') or src.startswith('ppa'):
|
||||
for k, v in OPENSTACK_CODENAMES.iteritems():
|
||||
for k, v in six.iteritems(OPENSTACK_CODENAMES):
|
||||
if v in src:
|
||||
return v
|
||||
|
||||
|
@ -134,7 +166,7 @@ def get_os_codename_version(vers):
|
|||
|
||||
def get_os_version_codename(codename):
|
||||
'''Determine OpenStack version number from codename.'''
|
||||
for k, v in OPENSTACK_CODENAMES.iteritems():
|
||||
for k, v in six.iteritems(OPENSTACK_CODENAMES):
|
||||
if v == codename:
|
||||
return k
|
||||
e = 'Could not derive OpenStack version for '\
|
||||
|
@ -194,7 +226,7 @@ def get_os_version_package(pkg, fatal=True):
|
|||
else:
|
||||
vers_map = OPENSTACK_CODENAMES
|
||||
|
||||
for version, cname in vers_map.iteritems():
|
||||
for version, cname in six.iteritems(vers_map):
|
||||
if cname == codename:
|
||||
return version
|
||||
# e = "Could not determine OpenStack version for package: %s" % pkg
|
||||
|
@ -286,6 +318,9 @@ def configure_installation_source(rel):
|
|||
'juno': 'trusty-updates/juno',
|
||||
'juno/updates': 'trusty-updates/juno',
|
||||
'juno/proposed': 'trusty-proposed/juno',
|
||||
'kilo': 'trusty-updates/kilo',
|
||||
'kilo/updates': 'trusty-updates/kilo',
|
||||
'kilo/proposed': 'trusty-proposed/kilo',
|
||||
}
|
||||
|
||||
try:
|
||||
|
@ -303,6 +338,21 @@ def configure_installation_source(rel):
|
|||
error_out("Invalid openstack-release specified: %s" % rel)
|
||||
|
||||
|
||||
def config_value_changed(option):
|
||||
"""
|
||||
Determine if config value changed since last call to this function.
|
||||
"""
|
||||
hook_data = unitdata.HookData()
|
||||
with hook_data():
|
||||
db = unitdata.kv()
|
||||
current = config(option)
|
||||
saved = db.get(option)
|
||||
db.set(option, current)
|
||||
if saved is None:
|
||||
return False
|
||||
return current != saved
|
||||
|
||||
|
||||
def save_script_rc(script_path="scripts/scriptrc", **env_vars):
|
||||
"""
|
||||
Write an rc file in the charm-delivered directory containing
|
||||
|
@ -318,7 +368,7 @@ def save_script_rc(script_path="scripts/scriptrc", **env_vars):
|
|||
rc_script.write(
|
||||
"#!/bin/bash\n")
|
||||
[rc_script.write('export %s=%s\n' % (u, p))
|
||||
for u, p in env_vars.iteritems() if u != "script_path"]
|
||||
for u, p in six.iteritems(env_vars) if u != "script_path"]
|
||||
|
||||
|
||||
def openstack_upgrade_available(package):
|
||||
|
@ -351,8 +401,8 @@ def ensure_block_device(block_device):
|
|||
'''
|
||||
_none = ['None', 'none', None]
|
||||
if (block_device in _none):
|
||||
error_out('prepare_storage(): Missing required input: '
|
||||
'block_device=%s.' % block_device, level=ERROR)
|
||||
error_out('prepare_storage(): Missing required input: block_device=%s.'
|
||||
% block_device)
|
||||
|
||||
if block_device.startswith('/dev/'):
|
||||
bdev = block_device
|
||||
|
@ -368,8 +418,7 @@ def ensure_block_device(block_device):
|
|||
bdev = '/dev/%s' % block_device
|
||||
|
||||
if not is_block_device(bdev):
|
||||
error_out('Failed to locate valid block device at %s' % bdev,
|
||||
level=ERROR)
|
||||
error_out('Failed to locate valid block device at %s' % bdev)
|
||||
|
||||
return bdev
|
||||
|
||||
|
@ -396,77 +445,10 @@ def clean_storage(block_device):
|
|||
else:
|
||||
zap_disk(block_device)
|
||||
|
||||
|
||||
def is_ip(address):
|
||||
"""
|
||||
Returns True if address is a valid IP address.
|
||||
"""
|
||||
try:
|
||||
# Test to see if already an IPv4 address
|
||||
socket.inet_aton(address)
|
||||
return True
|
||||
except socket.error:
|
||||
return False
|
||||
|
||||
|
||||
def ns_query(address):
|
||||
try:
|
||||
import dns.resolver
|
||||
except ImportError:
|
||||
apt_install('python-dnspython')
|
||||
import dns.resolver
|
||||
|
||||
if isinstance(address, dns.name.Name):
|
||||
rtype = 'PTR'
|
||||
elif isinstance(address, basestring):
|
||||
rtype = 'A'
|
||||
else:
|
||||
return None
|
||||
|
||||
answers = dns.resolver.query(address, rtype)
|
||||
if answers:
|
||||
return str(answers[0])
|
||||
return None
|
||||
|
||||
|
||||
def get_host_ip(hostname):
|
||||
"""
|
||||
Resolves the IP for a given hostname, or returns
|
||||
the input if it is already an IP.
|
||||
"""
|
||||
if is_ip(hostname):
|
||||
return hostname
|
||||
|
||||
return ns_query(hostname)
|
||||
|
||||
|
||||
def get_hostname(address, fqdn=True):
|
||||
"""
|
||||
Resolves hostname for given IP, or returns the input
|
||||
if it is already a hostname.
|
||||
"""
|
||||
if is_ip(address):
|
||||
try:
|
||||
import dns.reversename
|
||||
except ImportError:
|
||||
apt_install('python-dnspython')
|
||||
import dns.reversename
|
||||
|
||||
rev = dns.reversename.from_address(address)
|
||||
result = ns_query(rev)
|
||||
if not result:
|
||||
return None
|
||||
else:
|
||||
result = address
|
||||
|
||||
if fqdn:
|
||||
# strip trailing .
|
||||
if result.endswith('.'):
|
||||
return result[:-1]
|
||||
else:
|
||||
return result
|
||||
else:
|
||||
return result.split('.')[0]
|
||||
is_ip = ip.is_ip
|
||||
ns_query = ip.ns_query
|
||||
get_host_ip = ip.get_host_ip
|
||||
get_hostname = ip.get_hostname
|
||||
|
||||
|
||||
def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'):
|
||||
|
@ -486,8 +468,7 @@ def sync_db_with_multi_ipv6_addresses(database, database_user,
|
|||
'hostname': json.dumps(hosts)}
|
||||
|
||||
if relation_prefix:
|
||||
keys = kwargs.keys()
|
||||
for key in keys:
|
||||
for key in list(kwargs.keys()):
|
||||
kwargs["%s_%s" % (relation_prefix, key)] = kwargs[key]
|
||||
del kwargs[key]
|
||||
|
||||
|
@ -508,3 +489,201 @@ def os_requires_version(ostack_release, pkg):
|
|||
f(*args)
|
||||
return wrapped_f
|
||||
return wrap
|
||||
|
||||
|
||||
def git_install_requested():
|
||||
"""
|
||||
Returns true if openstack-origin-git is specified.
|
||||
"""
|
||||
return config('openstack-origin-git') is not None
|
||||
|
||||
|
||||
requirements_dir = None
|
||||
|
||||
|
||||
def _git_yaml_load(projects_yaml):
|
||||
"""
|
||||
Load the specified yaml into a dictionary.
|
||||
"""
|
||||
if not projects_yaml:
|
||||
return None
|
||||
|
||||
return yaml.load(projects_yaml)
|
||||
|
||||
|
||||
def git_clone_and_install(projects_yaml, core_project, depth=1):
|
||||
"""
|
||||
Clone/install all specified OpenStack repositories.
|
||||
|
||||
The expected format of projects_yaml is:
|
||||
repositories:
|
||||
- {name: keystone,
|
||||
repository: 'git://git.openstack.org/openstack/keystone.git',
|
||||
branch: 'stable/icehouse'}
|
||||
- {name: requirements,
|
||||
repository: 'git://git.openstack.org/openstack/requirements.git',
|
||||
branch: 'stable/icehouse'}
|
||||
directory: /mnt/openstack-git
|
||||
http_proxy: squid-proxy-url
|
||||
https_proxy: squid-proxy-url
|
||||
|
||||
The directory, http_proxy, and https_proxy keys are optional.
|
||||
"""
|
||||
global requirements_dir
|
||||
parent_dir = '/mnt/openstack-git'
|
||||
http_proxy = None
|
||||
|
||||
projects = _git_yaml_load(projects_yaml)
|
||||
_git_validate_projects_yaml(projects, core_project)
|
||||
|
||||
old_environ = dict(os.environ)
|
||||
|
||||
if 'http_proxy' in projects.keys():
|
||||
http_proxy = projects['http_proxy']
|
||||
os.environ['http_proxy'] = projects['http_proxy']
|
||||
if 'https_proxy' in projects.keys():
|
||||
os.environ['https_proxy'] = projects['https_proxy']
|
||||
|
||||
if 'directory' in projects.keys():
|
||||
parent_dir = projects['directory']
|
||||
|
||||
pip_create_virtualenv(os.path.join(parent_dir, 'venv'))
|
||||
|
||||
for p in projects['repositories']:
|
||||
repo = p['repository']
|
||||
branch = p['branch']
|
||||
if p['name'] == 'requirements':
|
||||
repo_dir = _git_clone_and_install_single(repo, branch, depth,
|
||||
parent_dir, http_proxy,
|
||||
update_requirements=False)
|
||||
requirements_dir = repo_dir
|
||||
else:
|
||||
repo_dir = _git_clone_and_install_single(repo, branch, depth,
|
||||
parent_dir, http_proxy,
|
||||
update_requirements=True)
|
||||
|
||||
os.environ = old_environ
|
||||
|
||||
|
||||
def _git_validate_projects_yaml(projects, core_project):
|
||||
"""
|
||||
Validate the projects yaml.
|
||||
"""
|
||||
_git_ensure_key_exists('repositories', projects)
|
||||
|
||||
for project in projects['repositories']:
|
||||
_git_ensure_key_exists('name', project.keys())
|
||||
_git_ensure_key_exists('repository', project.keys())
|
||||
_git_ensure_key_exists('branch', project.keys())
|
||||
|
||||
if projects['repositories'][0]['name'] != 'requirements':
|
||||
error_out('{} git repo must be specified first'.format('requirements'))
|
||||
|
||||
if projects['repositories'][-1]['name'] != core_project:
|
||||
error_out('{} git repo must be specified last'.format(core_project))
|
||||
|
||||
|
||||
def _git_ensure_key_exists(key, keys):
|
||||
"""
|
||||
Ensure that key exists in keys.
|
||||
"""
|
||||
if key not in keys:
|
||||
error_out('openstack-origin-git key \'{}\' is missing'.format(key))
|
||||
|
||||
|
||||
def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
|
||||
update_requirements):
|
||||
"""
|
||||
Clone and install a single git repository.
|
||||
"""
|
||||
dest_dir = os.path.join(parent_dir, os.path.basename(repo))
|
||||
|
||||
if not os.path.exists(parent_dir):
|
||||
juju_log('Directory already exists at {}. '
|
||||
'No need to create directory.'.format(parent_dir))
|
||||
os.mkdir(parent_dir)
|
||||
|
||||
if not os.path.exists(dest_dir):
|
||||
juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
|
||||
repo_dir = install_remote(repo, dest=parent_dir, branch=branch,
|
||||
depth=depth)
|
||||
else:
|
||||
repo_dir = dest_dir
|
||||
|
||||
if update_requirements:
|
||||
if not requirements_dir:
|
||||
error_out('requirements repo must be cloned before '
|
||||
'updating from global requirements.')
|
||||
_git_update_requirements(repo_dir, requirements_dir)
|
||||
|
||||
juju_log('Installing git repo from dir: {}'.format(repo_dir))
|
||||
if http_proxy:
|
||||
pip_install(repo_dir, proxy=http_proxy,
|
||||
venv=os.path.join(parent_dir, 'venv'))
|
||||
else:
|
||||
pip_install(repo_dir,
|
||||
venv=os.path.join(parent_dir, 'venv'))
|
||||
|
||||
return repo_dir
|
||||
|
||||
|
||||
def _git_update_requirements(package_dir, reqs_dir):
|
||||
"""
|
||||
Update from global requirements.
|
||||
|
||||
Update an OpenStack git directory's requirements.txt and
|
||||
test-requirements.txt from global-requirements.txt.
|
||||
"""
|
||||
orig_dir = os.getcwd()
|
||||
os.chdir(reqs_dir)
|
||||
cmd = ['python', 'update.py', package_dir]
|
||||
try:
|
||||
subprocess.check_call(cmd)
|
||||
except subprocess.CalledProcessError:
|
||||
package = os.path.basename(package_dir)
|
||||
error_out("Error updating {} from global-requirements.txt".format(package))
|
||||
os.chdir(orig_dir)
|
||||
|
||||
|
||||
def git_pip_venv_dir(projects_yaml):
|
||||
"""
|
||||
Return the pip virtualenv path.
|
||||
"""
|
||||
parent_dir = '/mnt/openstack-git'
|
||||
|
||||
projects = _git_yaml_load(projects_yaml)
|
||||
|
||||
if 'directory' in projects.keys():
|
||||
parent_dir = projects['directory']
|
||||
|
||||
return os.path.join(parent_dir, 'venv')
|
||||
|
||||
|
||||
def git_src_dir(projects_yaml, project):
|
||||
"""
|
||||
Return the directory where the specified project's source is located.
|
||||
"""
|
||||
parent_dir = '/mnt/openstack-git'
|
||||
|
||||
projects = _git_yaml_load(projects_yaml)
|
||||
|
||||
if 'directory' in projects.keys():
|
||||
parent_dir = projects['directory']
|
||||
|
||||
for p in projects['repositories']:
|
||||
if p['name'] == project:
|
||||
return os.path.join(parent_dir, os.path.basename(p['repository']))
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def git_yaml_value(projects_yaml, key):
|
||||
"""
|
||||
Return the value in projects_yaml for the specified key.
|
||||
"""
|
||||
projects = _git_yaml_load(projects_yaml)
|
||||
|
||||
if key in projects.keys():
|
||||
return projects[key]
|
||||
|
||||
return None
|
||||
|
|
|
@ -0,0 +1,15 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
|
@ -0,0 +1,15 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
|
@ -1,3 +1,19 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
#
|
||||
# Copyright 2012 Canonical Ltd.
|
||||
#
|
||||
|
@ -16,19 +32,18 @@ import time
|
|||
from subprocess import (
|
||||
check_call,
|
||||
check_output,
|
||||
CalledProcessError
|
||||
CalledProcessError,
|
||||
)
|
||||
|
||||
from charmhelpers.core.hookenv import (
|
||||
relation_get,
|
||||
relation_ids,
|
||||
related_units,
|
||||
log,
|
||||
DEBUG,
|
||||
INFO,
|
||||
WARNING,
|
||||
ERROR
|
||||
ERROR,
|
||||
)
|
||||
|
||||
from charmhelpers.core.host import (
|
||||
mount,
|
||||
mounts,
|
||||
|
@ -37,7 +52,6 @@ from charmhelpers.core.host import (
|
|||
service_running,
|
||||
umount,
|
||||
)
|
||||
|
||||
from charmhelpers.fetch import (
|
||||
apt_install,
|
||||
)
|
||||
|
@ -56,99 +70,85 @@ CEPH_CONF = """[global]
|
|||
|
||||
|
||||
def install():
|
||||
''' Basic Ceph client installation '''
|
||||
"""Basic Ceph client installation."""
|
||||
ceph_dir = "/etc/ceph"
|
||||
if not os.path.exists(ceph_dir):
|
||||
os.mkdir(ceph_dir)
|
||||
|
||||
apt_install('ceph-common', fatal=True)
|
||||
|
||||
|
||||
def rbd_exists(service, pool, rbd_img):
|
||||
''' Check to see if a RADOS block device exists '''
|
||||
"""Check to see if a RADOS block device exists."""
|
||||
try:
|
||||
out = check_output(['rbd', 'list', '--id', service,
|
||||
'--pool', pool])
|
||||
out = check_output(['rbd', 'list', '--id',
|
||||
service, '--pool', pool]).decode('UTF-8')
|
||||
except CalledProcessError:
|
||||
return False
|
||||
else:
|
||||
return rbd_img in out
|
||||
|
||||
return rbd_img in out
|
||||
|
||||
|
||||
def create_rbd_image(service, pool, image, sizemb):
|
||||
''' Create a new RADOS block device '''
|
||||
cmd = [
|
||||
'rbd',
|
||||
'create',
|
||||
image,
|
||||
'--size',
|
||||
str(sizemb),
|
||||
'--id',
|
||||
service,
|
||||
'--pool',
|
||||
pool
|
||||
]
|
||||
"""Create a new RADOS block device."""
|
||||
cmd = ['rbd', 'create', image, '--size', str(sizemb), '--id', service,
|
||||
'--pool', pool]
|
||||
check_call(cmd)
|
||||
|
||||
|
||||
def pool_exists(service, name):
|
||||
''' Check to see if a RADOS pool already exists '''
|
||||
"""Check to see if a RADOS pool already exists."""
|
||||
try:
|
||||
out = check_output(['rados', '--id', service, 'lspools'])
|
||||
out = check_output(['rados', '--id', service,
|
||||
'lspools']).decode('UTF-8')
|
||||
except CalledProcessError:
|
||||
return False
|
||||
else:
|
||||
return name in out
|
||||
|
||||
return name in out
|
||||
|
||||
|
||||
def get_osds(service):
|
||||
'''
|
||||
Return a list of all Ceph Object Storage Daemons
|
||||
currently in the cluster
|
||||
'''
|
||||
"""Return a list of all Ceph Object Storage Daemons currently in the
|
||||
cluster.
|
||||
"""
|
||||
version = ceph_version()
|
||||
if version and version >= '0.56':
|
||||
return json.loads(check_output(['ceph', '--id', service,
|
||||
'osd', 'ls', '--format=json']))
|
||||
else:
|
||||
return None
|
||||
'osd', 'ls',
|
||||
'--format=json']).decode('UTF-8'))
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def create_pool(service, name, replicas=3):
|
||||
''' Create a new RADOS pool '''
|
||||
"""Create a new RADOS pool."""
|
||||
if pool_exists(service, name):
|
||||
log("Ceph pool {} already exists, skipping creation".format(name),
|
||||
level=WARNING)
|
||||
return
|
||||
|
||||
# Calculate the number of placement groups based
|
||||
# on upstream recommended best practices.
|
||||
osds = get_osds(service)
|
||||
if osds:
|
||||
pgnum = (len(osds) * 100 / replicas)
|
||||
pgnum = (len(osds) * 100 // replicas)
|
||||
else:
|
||||
# NOTE(james-page): Default to 200 for older ceph versions
|
||||
# which don't support OSD query from cli
|
||||
pgnum = 200
|
||||
cmd = [
|
||||
'ceph', '--id', service,
|
||||
'osd', 'pool', 'create',
|
||||
name, str(pgnum)
|
||||
]
|
||||
|
||||
cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pgnum)]
|
||||
check_call(cmd)
|
||||
cmd = [
|
||||
'ceph', '--id', service,
|
||||
'osd', 'pool', 'set', name,
|
||||
'size', str(replicas)
|
||||
]
|
||||
|
||||
cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', name, 'size',
|
||||
str(replicas)]
|
||||
check_call(cmd)
|
||||
|
||||
|
||||
def delete_pool(service, name):
|
||||
''' Delete a RADOS pool from ceph '''
|
||||
cmd = [
|
||||
'ceph', '--id', service,
|
||||
'osd', 'pool', 'delete',
|
||||
name, '--yes-i-really-really-mean-it'
|
||||
]
|
||||
"""Delete a RADOS pool from ceph."""
|
||||
cmd = ['ceph', '--id', service, 'osd', 'pool', 'delete', name,
|
||||
'--yes-i-really-really-mean-it']
|
||||
check_call(cmd)
|
||||
|
||||
|
||||
|
@ -161,44 +161,54 @@ def _keyring_path(service):
|
|||
|
||||
|
||||
def create_keyring(service, key):
|
||||
''' Create a new Ceph keyring containing key'''
|
||||
"""Create a new Ceph keyring containing key."""
|
||||
keyring = _keyring_path(service)
|
||||
if os.path.exists(keyring):
|
||||
log('ceph: Keyring exists at %s.' % keyring, level=WARNING)
|
||||
log('Ceph keyring exists at %s.' % keyring, level=WARNING)
|
||||
return
|
||||
cmd = [
|
||||
'ceph-authtool',
|
||||
keyring,
|
||||
'--create-keyring',
|
||||
'--name=client.{}'.format(service),
|
||||
'--add-key={}'.format(key)
|
||||
]
|
||||
|
||||
cmd = ['ceph-authtool', keyring, '--create-keyring',
|
||||
'--name=client.{}'.format(service), '--add-key={}'.format(key)]
|
||||
check_call(cmd)
|
||||
log('ceph: Created new ring at %s.' % keyring, level=INFO)
|
||||
log('Created new ceph keyring at %s.' % keyring, level=DEBUG)
|
||||
|
||||
|
||||
def delete_keyring(service):
|
||||
"""Delete an existing Ceph keyring."""
|
||||
keyring = _keyring_path(service)
|
||||
if not os.path.exists(keyring):
|
||||
log('Keyring does not exist at %s' % keyring, level=WARNING)
|
||||
return
|
||||
|
||||
os.remove(keyring)
|
||||
log('Deleted ring at %s.' % keyring, level=INFO)
|
||||
|
||||
|
||||
def create_key_file(service, key):
|
||||
''' Create a file containing key '''
|
||||
"""Create a file containing key."""
|
||||
keyfile = _keyfile_path(service)
|
||||
if os.path.exists(keyfile):
|
||||
log('ceph: Keyfile exists at %s.' % keyfile, level=WARNING)
|
||||
log('Keyfile exists at %s.' % keyfile, level=WARNING)
|
||||
return
|
||||
|
||||
with open(keyfile, 'w') as fd:
|
||||
fd.write(key)
|
||||
log('ceph: Created new keyfile at %s.' % keyfile, level=INFO)
|
||||
|
||||
log('Created new keyfile at %s.' % keyfile, level=INFO)
|
||||
|
||||
|
||||
def get_ceph_nodes():
|
||||
''' Query named relation 'ceph' to detemine current nodes '''
|
||||
"""Query named relation 'ceph' to determine current nodes."""
|
||||
hosts = []
|
||||
for r_id in relation_ids('ceph'):
|
||||
for unit in related_units(r_id):
|
||||
hosts.append(relation_get('private-address', unit=unit, rid=r_id))
|
||||
|
||||
return hosts
|
||||
|
||||
|
||||
def configure(service, key, auth, use_syslog):
|
||||
''' Perform basic configuration of Ceph '''
|
||||
"""Perform basic configuration of Ceph."""
|
||||
create_keyring(service, key)
|
||||
create_key_file(service, key)
|
||||
hosts = get_ceph_nodes()
|
||||
|
@ -211,17 +221,17 @@ def configure(service, key, auth, use_syslog):
|
|||
|
||||
|
||||
def image_mapped(name):
|
||||
''' Determine whether a RADOS block device is mapped locally '''
|
||||
"""Determine whether a RADOS block device is mapped locally."""
|
||||
try:
|
||||
out = check_output(['rbd', 'showmapped'])
|
||||
out = check_output(['rbd', 'showmapped']).decode('UTF-8')
|
||||
except CalledProcessError:
|
||||
return False
|
||||
else:
|
||||
return name in out
|
||||
|
||||
return name in out
|
||||
|
||||
|
||||
def map_block_storage(service, pool, image):
|
||||
''' Map a RADOS block device for local use '''
|
||||
"""Map a RADOS block device for local use."""
|
||||
cmd = [
|
||||
'rbd',
|
||||
'map',
|
||||
|
@ -235,31 +245,32 @@ def map_block_storage(service, pool, image):
|
|||
|
||||
|
||||
def filesystem_mounted(fs):
|
||||
''' Determine whether a filesytems is already mounted '''
|
||||
"""Determine whether a filesytems is already mounted."""
|
||||
return fs in [f for f, m in mounts()]
|
||||
|
||||
|
||||
def make_filesystem(blk_device, fstype='ext4', timeout=10):
|
||||
''' Make a new filesystem on the specified block device '''
|
||||
"""Make a new filesystem on the specified block device."""
|
||||
count = 0
|
||||
e_noent = os.errno.ENOENT
|
||||
while not os.path.exists(blk_device):
|
||||
if count >= timeout:
|
||||
log('ceph: gave up waiting on block device %s' % blk_device,
|
||||
log('Gave up waiting on block device %s' % blk_device,
|
||||
level=ERROR)
|
||||
raise IOError(e_noent, os.strerror(e_noent), blk_device)
|
||||
log('ceph: waiting for block device %s to appear' % blk_device,
|
||||
level=INFO)
|
||||
|
||||
log('Waiting for block device %s to appear' % blk_device,
|
||||
level=DEBUG)
|
||||
count += 1
|
||||
time.sleep(1)
|
||||
else:
|
||||
log('ceph: Formatting block device %s as filesystem %s.' %
|
||||
log('Formatting block device %s as filesystem %s.' %
|
||||
(blk_device, fstype), level=INFO)
|
||||
check_call(['mkfs', '-t', fstype, blk_device])
|
||||
|
||||
|
||||
def place_data_on_block_device(blk_device, data_src_dst):
|
||||
''' Migrate data in data_src_dst to blk_device and then remount '''
|
||||
"""Migrate data in data_src_dst to blk_device and then remount."""
|
||||
# mount block device into /mnt
|
||||
mount(blk_device, '/mnt')
|
||||
# copy data to /mnt
|
||||
|
@ -279,8 +290,8 @@ def place_data_on_block_device(blk_device, data_src_dst):
|
|||
|
||||
# TODO: re-use
|
||||
def modprobe(module):
|
||||
''' Load a kernel module and configure for auto-load on reboot '''
|
||||
log('ceph: Loading kernel module', level=INFO)
|
||||
"""Load a kernel module and configure for auto-load on reboot."""
|
||||
log('Loading kernel module', level=INFO)
|
||||
cmd = ['modprobe', module]
|
||||
check_call(cmd)
|
||||
with open('/etc/modules', 'r+') as modules:
|
||||
|
@ -289,7 +300,7 @@ def modprobe(module):
|
|||
|
||||
|
||||
def copy_files(src, dst, symlinks=False, ignore=None):
|
||||
''' Copy files from src to dst '''
|
||||
"""Copy files from src to dst."""
|
||||
for item in os.listdir(src):
|
||||
s = os.path.join(src, item)
|
||||
d = os.path.join(dst, item)
|
||||
|
@ -302,8 +313,7 @@ def copy_files(src, dst, symlinks=False, ignore=None):
|
|||
def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
|
||||
blk_device, fstype, system_services=[],
|
||||
replicas=3):
|
||||
"""
|
||||
NOTE: This function must only be called from a single service unit for
|
||||
"""NOTE: This function must only be called from a single service unit for
|
||||
the same rbd_img otherwise data loss will occur.
|
||||
|
||||
Ensures given pool and RBD image exists, is mapped to a block device,
|
||||
|
@ -317,15 +327,16 @@ def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
|
|||
"""
|
||||
# Ensure pool, RBD image, RBD mappings are in place.
|
||||
if not pool_exists(service, pool):
|
||||
log('ceph: Creating new pool {}.'.format(pool))
|
||||
log('Creating new pool {}.'.format(pool), level=INFO)
|
||||
create_pool(service, pool, replicas=replicas)
|
||||
|
||||
if not rbd_exists(service, pool, rbd_img):
|
||||
log('ceph: Creating RBD image ({}).'.format(rbd_img))
|
||||
log('Creating RBD image ({}).'.format(rbd_img), level=INFO)
|
||||
create_rbd_image(service, pool, rbd_img, sizemb)
|
||||
|
||||
if not image_mapped(rbd_img):
|
||||
log('ceph: Mapping RBD Image {} as a Block Device.'.format(rbd_img))
|
||||
log('Mapping RBD Image {} as a Block Device.'.format(rbd_img),
|
||||
level=INFO)
|
||||
map_block_storage(service, pool, rbd_img)
|
||||
|
||||
# make file system
|
||||
|
@ -340,45 +351,47 @@ def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
|
|||
|
||||
for svc in system_services:
|
||||
if service_running(svc):
|
||||
log('ceph: Stopping services {} prior to migrating data.'
|
||||
.format(svc))
|
||||
log('Stopping services {} prior to migrating data.'
|
||||
.format(svc), level=DEBUG)
|
||||
service_stop(svc)
|
||||
|
||||
place_data_on_block_device(blk_device, mount_point)
|
||||
|
||||
for svc in system_services:
|
||||
log('ceph: Starting service {} after migrating data.'
|
||||
.format(svc))
|
||||
log('Starting service {} after migrating data.'
|
||||
.format(svc), level=DEBUG)
|
||||
service_start(svc)
|
||||
|
||||
|
||||
def ensure_ceph_keyring(service, user=None, group=None):
|
||||
'''
|
||||
Ensures a ceph keyring is created for a named service
|
||||
and optionally ensures user and group ownership.
|
||||
"""Ensures a ceph keyring is created for a named service and optionally
|
||||
ensures user and group ownership.
|
||||
|
||||
Returns False if no ceph key is available in relation state.
|
||||
'''
|
||||
"""
|
||||
key = None
|
||||
for rid in relation_ids('ceph'):
|
||||
for unit in related_units(rid):
|
||||
key = relation_get('key', rid=rid, unit=unit)
|
||||
if key:
|
||||
break
|
||||
|
||||
if not key:
|
||||
return False
|
||||
|
||||
create_keyring(service=service, key=key)
|
||||
keyring = _keyring_path(service)
|
||||
if user and group:
|
||||
check_call(['chown', '%s.%s' % (user, group), keyring])
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def ceph_version():
|
||||
''' Retrieve the local version of ceph '''
|
||||
"""Retrieve the local version of ceph."""
|
||||
if os.path.exists('/usr/bin/ceph'):
|
||||
cmd = ['ceph', '-v']
|
||||
output = check_output(cmd)
|
||||
output = check_output(cmd).decode('US-ASCII')
|
||||
output = output.split()
|
||||
if len(output) > 3:
|
||||
return output[2]
|
||||
|
@ -386,3 +399,46 @@ def ceph_version():
|
|||
return None
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
class CephBrokerRq(object):
|
||||
"""Ceph broker request.
|
||||
|
||||
Multiple operations can be added to a request and sent to the Ceph broker
|
||||
to be executed.
|
||||
|
||||
Request is json-encoded for sending over the wire.
|
||||
|
||||
The API is versioned and defaults to version 1.
|
||||
"""
|
||||
def __init__(self, api_version=1):
|
||||
self.api_version = api_version
|
||||
self.ops = []
|
||||
|
||||
def add_op_create_pool(self, name, replica_count=3):
|
||||
self.ops.append({'op': 'create-pool', 'name': name,
|
||||
'replicas': replica_count})
|
||||
|
||||
@property
|
||||
def request(self):
|
||||
return json.dumps({'api-version': self.api_version, 'ops': self.ops})
|
||||
|
||||
|
||||
class CephBrokerRsp(object):
|
||||
"""Ceph broker response.
|
||||
|
||||
Response is json-decoded and contents provided as methods/properties.
|
||||
|
||||
The API is versioned and defaults to version 1.
|
||||
"""
|
||||
def __init__(self, encoded_rsp):
|
||||
self.api_version = None
|
||||
self.rsp = json.loads(encoded_rsp)
|
||||
|
||||
@property
|
||||
def exit_code(self):
|
||||
return self.rsp.get('exit-code')
|
||||
|
||||
@property
|
||||
def exit_msg(self):
|
||||
return self.rsp.get('stderr')
|
||||
|
|
|
@ -1,12 +1,28 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
from subprocess import (
|
||||
check_call,
|
||||
check_output,
|
||||
)
|
||||
|
||||
import six
|
||||
|
||||
|
||||
##################################################
|
||||
# loopback device helpers.
|
||||
|
@ -37,7 +53,7 @@ def create_loopback(file_path):
|
|||
'''
|
||||
file_path = os.path.abspath(file_path)
|
||||
check_call(['losetup', '--find', file_path])
|
||||
for d, f in loopback_devices().iteritems():
|
||||
for d, f in six.iteritems(loopback_devices()):
|
||||
if f == file_path:
|
||||
return d
|
||||
|
||||
|
@ -51,7 +67,7 @@ def ensure_loopback_device(path, size):
|
|||
|
||||
:returns: str: Full path to the ensured loopback device (eg, /dev/loop0)
|
||||
'''
|
||||
for d, f in loopback_devices().iteritems():
|
||||
for d, f in six.iteritems(loopback_devices()):
|
||||
if f == path:
|
||||
return d
|
||||
|
||||
|
|
|
@ -1,3 +1,19 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from subprocess import (
|
||||
CalledProcessError,
|
||||
check_call,
|
||||
|
@ -61,6 +77,7 @@ def list_lvm_volume_group(block_device):
|
|||
vg = None
|
||||
pvd = check_output(['pvdisplay', block_device]).splitlines()
|
||||
for l in pvd:
|
||||
l = l.decode('UTF-8')
|
||||
if l.strip().startswith('VG Name'):
|
||||
vg = ' '.join(l.strip().split()[2:])
|
||||
return vg
|
||||
|
|
|
@ -1,3 +1,19 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
import re
|
||||
from stat import S_ISBLK
|
||||
|
@ -30,7 +46,8 @@ def zap_disk(block_device):
|
|||
# sometimes sgdisk exits non-zero; this is OK, dd will clean up
|
||||
call(['sgdisk', '--zap-all', '--mbrtogpt',
|
||||
'--clear', block_device])
|
||||
dev_end = check_output(['blockdev', '--getsz', block_device])
|
||||
dev_end = check_output(['blockdev', '--getsz',
|
||||
block_device]).decode('UTF-8')
|
||||
gpt_end = int(dev_end.split()[0]) - 100
|
||||
check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device),
|
||||
'bs=1M', 'count=1'])
|
||||
|
@ -47,7 +64,7 @@ def is_device_mounted(device):
|
|||
it doesn't.
|
||||
'''
|
||||
is_partition = bool(re.search(r".*[0-9]+\b", device))
|
||||
out = check_output(['mount'])
|
||||
out = check_output(['mount']).decode('UTF-8')
|
||||
if is_partition:
|
||||
return bool(re.search(device + r"\b", out))
|
||||
return bool(re.search(device + r"[0-9]+\b", out))
|
||||
|
|
|
@ -0,0 +1,15 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
|
@ -0,0 +1,57 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
#
|
||||
# Copyright 2014 Canonical Ltd.
|
||||
#
|
||||
# Authors:
|
||||
# Edward Hope-Morley <opentastic@gmail.com>
|
||||
#
|
||||
|
||||
import time
|
||||
|
||||
from charmhelpers.core.hookenv import (
|
||||
log,
|
||||
INFO,
|
||||
)
|
||||
|
||||
|
||||
def retry_on_exception(num_retries, base_delay=0, exc_type=Exception):
|
||||
"""If the decorated function raises exception exc_type, allow num_retries
|
||||
retry attempts before raise the exception.
|
||||
"""
|
||||
def _retry_on_exception_inner_1(f):
|
||||
def _retry_on_exception_inner_2(*args, **kwargs):
|
||||
retries = num_retries
|
||||
multiplier = 1
|
||||
while True:
|
||||
try:
|
||||
return f(*args, **kwargs)
|
||||
except exc_type:
|
||||
if not retries:
|
||||
raise
|
||||
|
||||
delay = base_delay * multiplier
|
||||
multiplier += 1
|
||||
log("Retrying '%s' %d more times (delay=%s)" %
|
||||
(f.__name__, retries, delay), level=INFO)
|
||||
retries -= 1
|
||||
if delay:
|
||||
time.sleep(delay)
|
||||
|
||||
return _retry_on_exception_inner_2
|
||||
|
||||
return _retry_on_exception_inner_1
|
|
@ -1,12 +1,29 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
|
||||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import io
|
||||
import os
|
||||
|
||||
__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
|
||||
|
||||
class Fstab(file):
|
||||
|
||||
class Fstab(io.FileIO):
|
||||
"""This class extends file in order to implement a file reader/writer
|
||||
for file `/etc/fstab`
|
||||
"""
|
||||
|
@ -24,8 +41,8 @@ class Fstab(file):
|
|||
options = "defaults"
|
||||
|
||||
self.options = options
|
||||
self.d = d
|
||||
self.p = p
|
||||
self.d = int(d)
|
||||
self.p = int(p)
|
||||
|
||||
def __eq__(self, o):
|
||||
return str(self) == str(o)
|
||||
|
@ -45,7 +62,7 @@ class Fstab(file):
|
|||
self._path = path
|
||||
else:
|
||||
self._path = self.DEFAULT_PATH
|
||||
file.__init__(self, self._path, 'r+')
|
||||
super(Fstab, self).__init__(self._path, 'rb+')
|
||||
|
||||
def _hydrate_entry(self, line):
|
||||
# NOTE: use split with no arguments to split on any
|
||||
|
@ -58,8 +75,9 @@ class Fstab(file):
|
|||
def entries(self):
|
||||
self.seek(0)
|
||||
for line in self.readlines():
|
||||
line = line.decode('us-ascii')
|
||||
try:
|
||||
if not line.startswith("#"):
|
||||
if line.strip() and not line.strip().startswith("#"):
|
||||
yield self._hydrate_entry(line)
|
||||
except ValueError:
|
||||
pass
|
||||
|
@ -75,18 +93,18 @@ class Fstab(file):
|
|||
if self.get_entry_by_attr('device', entry.device):
|
||||
return False
|
||||
|
||||
self.write(str(entry) + '\n')
|
||||
self.write((str(entry) + '\n').encode('us-ascii'))
|
||||
self.truncate()
|
||||
return entry
|
||||
|
||||
def remove_entry(self, entry):
|
||||
self.seek(0)
|
||||
|
||||
lines = self.readlines()
|
||||
lines = [l.decode('us-ascii') for l in self.readlines()]
|
||||
|
||||
found = False
|
||||
for index, line in enumerate(lines):
|
||||
if not line.startswith("#"):
|
||||
if line.strip() and not line.strip().startswith("#"):
|
||||
if self._hydrate_entry(line) == entry:
|
||||
found = True
|
||||
break
|
||||
|
@ -97,7 +115,7 @@ class Fstab(file):
|
|||
lines.remove(line)
|
||||
|
||||
self.seek(0)
|
||||
self.write(''.join(lines))
|
||||
self.write(''.join(lines).encode('us-ascii'))
|
||||
self.truncate()
|
||||
return True
|
||||
|
||||
|
|
|
@ -1,17 +1,42 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
"Interactions with the Juju environment"
|
||||
# Copyright 2013 Canonical Ltd.
|
||||
#
|
||||
# Authors:
|
||||
# Charm Helpers Developers <juju@lists.ubuntu.com>
|
||||
|
||||
from __future__ import print_function
|
||||
from functools import wraps
|
||||
import os
|
||||
import json
|
||||
import yaml
|
||||
import subprocess
|
||||
import sys
|
||||
import UserDict
|
||||
import errno
|
||||
import tempfile
|
||||
from subprocess import CalledProcessError
|
||||
|
||||
import six
|
||||
if not six.PY3:
|
||||
from UserDict import UserDict
|
||||
else:
|
||||
from collections import UserDict
|
||||
|
||||
CRITICAL = "CRITICAL"
|
||||
ERROR = "ERROR"
|
||||
WARNING = "WARNING"
|
||||
|
@ -35,15 +60,17 @@ def cached(func):
|
|||
|
||||
will cache the result of unit_get + 'test' for future calls.
|
||||
"""
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
global cache
|
||||
key = str((func, args, kwargs))
|
||||
try:
|
||||
return cache[key]
|
||||
except KeyError:
|
||||
res = func(*args, **kwargs)
|
||||
cache[key] = res
|
||||
return res
|
||||
pass # Drop out of the exception handler scope.
|
||||
res = func(*args, **kwargs)
|
||||
cache[key] = res
|
||||
return res
|
||||
return wrapper
|
||||
|
||||
|
||||
|
@ -63,16 +90,29 @@ def log(message, level=None):
|
|||
command = ['juju-log']
|
||||
if level:
|
||||
command += ['-l', level]
|
||||
if not isinstance(message, six.string_types):
|
||||
message = repr(message)
|
||||
command += [message]
|
||||
subprocess.call(command)
|
||||
# Missing juju-log should not cause failures in unit tests
|
||||
# Send log output to stderr
|
||||
try:
|
||||
subprocess.call(command)
|
||||
except OSError as e:
|
||||
if e.errno == errno.ENOENT:
|
||||
if level:
|
||||
message = "{}: {}".format(level, message)
|
||||
message = "juju-log: {}".format(message)
|
||||
print(message, file=sys.stderr)
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
class Serializable(UserDict.IterableUserDict):
|
||||
class Serializable(UserDict):
|
||||
"""Wrapper, an object that can be serialized to yaml or json"""
|
||||
|
||||
def __init__(self, obj):
|
||||
# wrap the object
|
||||
UserDict.IterableUserDict.__init__(self)
|
||||
UserDict.__init__(self)
|
||||
self.data = obj
|
||||
|
||||
def __getattr__(self, attr):
|
||||
|
@ -142,7 +182,7 @@ def local_unit():
|
|||
|
||||
def remote_unit():
|
||||
"""The remote unit for the current relation hook"""
|
||||
return os.environ['JUJU_REMOTE_UNIT']
|
||||
return os.environ.get('JUJU_REMOTE_UNIT', None)
|
||||
|
||||
|
||||
def service_name():
|
||||
|
@ -214,11 +254,17 @@ class Config(dict):
|
|||
except KeyError:
|
||||
return (self._prev_dict or {})[key]
|
||||
|
||||
def get(self, key, default=None):
|
||||
try:
|
||||
return self[key]
|
||||
except KeyError:
|
||||
return default
|
||||
|
||||
def keys(self):
|
||||
prev_keys = []
|
||||
if self._prev_dict is not None:
|
||||
prev_keys = self._prev_dict.keys()
|
||||
return list(set(prev_keys + dict.keys(self)))
|
||||
return list(set(prev_keys + list(dict.keys(self))))
|
||||
|
||||
def load_previous(self, path=None):
|
||||
"""Load previous copy of config from disk.
|
||||
|
@ -269,7 +315,7 @@ class Config(dict):
|
|||
|
||||
"""
|
||||
if self._prev_dict:
|
||||
for k, v in self._prev_dict.iteritems():
|
||||
for k, v in six.iteritems(self._prev_dict):
|
||||
if k not in self:
|
||||
self[k] = v
|
||||
with open(self.path, 'w') as f:
|
||||
|
@ -284,7 +330,8 @@ def config(scope=None):
|
|||
config_cmd_line.append(scope)
|
||||
config_cmd_line.append('--format=json')
|
||||
try:
|
||||
config_data = json.loads(subprocess.check_output(config_cmd_line))
|
||||
config_data = json.loads(
|
||||
subprocess.check_output(config_cmd_line).decode('UTF-8'))
|
||||
if scope is not None:
|
||||
return config_data
|
||||
return Config(config_data)
|
||||
|
@ -303,10 +350,10 @@ def relation_get(attribute=None, unit=None, rid=None):
|
|||
if unit:
|
||||
_args.append(unit)
|
||||
try:
|
||||
return json.loads(subprocess.check_output(_args))
|
||||
return json.loads(subprocess.check_output(_args).decode('UTF-8'))
|
||||
except ValueError:
|
||||
return None
|
||||
except CalledProcessError, e:
|
||||
except CalledProcessError as e:
|
||||
if e.returncode == 2:
|
||||
return None
|
||||
raise
|
||||
|
@ -316,18 +363,49 @@ def relation_set(relation_id=None, relation_settings=None, **kwargs):
|
|||
"""Set relation information for the current unit"""
|
||||
relation_settings = relation_settings if relation_settings else {}
|
||||
relation_cmd_line = ['relation-set']
|
||||
accepts_file = "--file" in subprocess.check_output(
|
||||
relation_cmd_line + ["--help"], universal_newlines=True)
|
||||
if relation_id is not None:
|
||||
relation_cmd_line.extend(('-r', relation_id))
|
||||
for k, v in (relation_settings.items() + kwargs.items()):
|
||||
if v is None:
|
||||
relation_cmd_line.append('{}='.format(k))
|
||||
else:
|
||||
relation_cmd_line.append('{}={}'.format(k, v))
|
||||
subprocess.check_call(relation_cmd_line)
|
||||
settings = relation_settings.copy()
|
||||
settings.update(kwargs)
|
||||
for key, value in settings.items():
|
||||
# Force value to be a string: it always should, but some call
|
||||
# sites pass in things like dicts or numbers.
|
||||
if value is not None:
|
||||
settings[key] = "{}".format(value)
|
||||
if accepts_file:
|
||||
# --file was introduced in Juju 1.23.2. Use it by default if
|
||||
# available, since otherwise we'll break if the relation data is
|
||||
# too big. Ideally we should tell relation-set to read the data from
|
||||
# stdin, but that feature is broken in 1.23.2: Bug #1454678.
|
||||
with tempfile.NamedTemporaryFile(delete=False) as settings_file:
|
||||
settings_file.write(yaml.safe_dump(settings).encode("utf-8"))
|
||||
subprocess.check_call(
|
||||
relation_cmd_line + ["--file", settings_file.name])
|
||||
os.remove(settings_file.name)
|
||||
else:
|
||||
for key, value in settings.items():
|
||||
if value is None:
|
||||
relation_cmd_line.append('{}='.format(key))
|
||||
else:
|
||||
relation_cmd_line.append('{}={}'.format(key, value))
|
||||
subprocess.check_call(relation_cmd_line)
|
||||
# Flush cache of any relation-gets for local unit
|
||||
flush(local_unit())
|
||||
|
||||
|
||||
def relation_clear(r_id=None):
|
||||
''' Clears any relation data already set on relation r_id '''
|
||||
settings = relation_get(rid=r_id,
|
||||
unit=local_unit())
|
||||
for setting in settings:
|
||||
if setting not in ['public-address', 'private-address']:
|
||||
settings[setting] = None
|
||||
relation_set(relation_id=r_id,
|
||||
**settings)
|
||||
|
||||
|
||||
@cached
|
||||
def relation_ids(reltype=None):
|
||||
"""A list of relation_ids"""
|
||||
|
@ -335,7 +413,8 @@ def relation_ids(reltype=None):
|
|||
relid_cmd_line = ['relation-ids', '--format=json']
|
||||
if reltype is not None:
|
||||
relid_cmd_line.append(reltype)
|
||||
return json.loads(subprocess.check_output(relid_cmd_line)) or []
|
||||
return json.loads(
|
||||
subprocess.check_output(relid_cmd_line).decode('UTF-8')) or []
|
||||
return []
|
||||
|
||||
|
||||
|
@ -346,7 +425,8 @@ def related_units(relid=None):
|
|||
units_cmd_line = ['relation-list', '--format=json']
|
||||
if relid is not None:
|
||||
units_cmd_line.extend(('-r', relid))
|
||||
return json.loads(subprocess.check_output(units_cmd_line)) or []
|
||||
return json.loads(
|
||||
subprocess.check_output(units_cmd_line).decode('UTF-8')) or []
|
||||
|
||||
|
||||
@cached
|
||||
|
@ -385,21 +465,31 @@ def relations_of_type(reltype=None):
|
|||
return relation_data
|
||||
|
||||
|
||||
@cached
|
||||
def metadata():
|
||||
"""Get the current charm metadata.yaml contents as a python object"""
|
||||
with open(os.path.join(charm_dir(), 'metadata.yaml')) as md:
|
||||
return yaml.safe_load(md)
|
||||
|
||||
|
||||
@cached
|
||||
def relation_types():
|
||||
"""Get a list of relation types supported by this charm"""
|
||||
charmdir = os.environ.get('CHARM_DIR', '')
|
||||
mdf = open(os.path.join(charmdir, 'metadata.yaml'))
|
||||
md = yaml.safe_load(mdf)
|
||||
rel_types = []
|
||||
md = metadata()
|
||||
for key in ('provides', 'requires', 'peers'):
|
||||
section = md.get(key)
|
||||
if section:
|
||||
rel_types.extend(section.keys())
|
||||
mdf.close()
|
||||
return rel_types
|
||||
|
||||
|
||||
@cached
|
||||
def charm_name():
|
||||
"""Get the name of the current charm as is specified on metadata.yaml"""
|
||||
return metadata().get('name')
|
||||
|
||||
|
||||
@cached
|
||||
def relations():
|
||||
"""Get a nested dictionary of relation data for all related units"""
|
||||
|
@ -455,11 +545,16 @@ def unit_get(attribute):
|
|||
"""Get the unit ID for the remote unit"""
|
||||
_args = ['unit-get', '--format=json', attribute]
|
||||
try:
|
||||
return json.loads(subprocess.check_output(_args))
|
||||
return json.loads(subprocess.check_output(_args).decode('UTF-8'))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def unit_public_ip():
|
||||
"""Get this unit's public IP address"""
|
||||
return unit_get('public-address')
|
||||
|
||||
|
||||
def unit_private_ip():
|
||||
"""Get this unit's private IP address"""
|
||||
return unit_get('private-address')
|
||||
|
@ -530,3 +625,120 @@ class Hooks(object):
|
|||
def charm_dir():
|
||||
"""Return the root directory of the current charm"""
|
||||
return os.environ.get('CHARM_DIR')
|
||||
|
||||
|
||||
@cached
|
||||
def action_get(key=None):
|
||||
"""Gets the value of an action parameter, or all key/value param pairs"""
|
||||
cmd = ['action-get']
|
||||
if key is not None:
|
||||
cmd.append(key)
|
||||
cmd.append('--format=json')
|
||||
action_data = json.loads(subprocess.check_output(cmd).decode('UTF-8'))
|
||||
return action_data
|
||||
|
||||
|
||||
def action_set(values):
|
||||
"""Sets the values to be returned after the action finishes"""
|
||||
cmd = ['action-set']
|
||||
for k, v in list(values.items()):
|
||||
cmd.append('{}={}'.format(k, v))
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
|
||||
def action_fail(message):
|
||||
"""Sets the action status to failed and sets the error message.
|
||||
|
||||
The results set by action_set are preserved."""
|
||||
subprocess.check_call(['action-fail', message])
|
||||
|
||||
|
||||
def status_set(workload_state, message):
|
||||
"""Set the workload state with a message
|
||||
|
||||
Use status-set to set the workload state with a message which is visible
|
||||
to the user via juju status. If the status-set command is not found then
|
||||
assume this is juju < 1.23 and juju-log the message unstead.
|
||||
|
||||
workload_state -- valid juju workload state.
|
||||
message -- status update message
|
||||
"""
|
||||
valid_states = ['maintenance', 'blocked', 'waiting', 'active']
|
||||
if workload_state not in valid_states:
|
||||
raise ValueError(
|
||||
'{!r} is not a valid workload state'.format(workload_state)
|
||||
)
|
||||
cmd = ['status-set', workload_state, message]
|
||||
try:
|
||||
ret = subprocess.call(cmd)
|
||||
if ret == 0:
|
||||
return
|
||||
except OSError as e:
|
||||
if e.errno != errno.ENOENT:
|
||||
raise
|
||||
log_message = 'status-set failed: {} {}'.format(workload_state,
|
||||
message)
|
||||
log(log_message, level='INFO')
|
||||
|
||||
|
||||
def status_get():
|
||||
"""Retrieve the previously set juju workload state
|
||||
|
||||
If the status-set command is not found then assume this is juju < 1.23 and
|
||||
return 'unknown'
|
||||
"""
|
||||
cmd = ['status-get']
|
||||
try:
|
||||
raw_status = subprocess.check_output(cmd, universal_newlines=True)
|
||||
status = raw_status.rstrip()
|
||||
return status
|
||||
except OSError as e:
|
||||
if e.errno == errno.ENOENT:
|
||||
return 'unknown'
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
def translate_exc(from_exc, to_exc):
|
||||
def inner_translate_exc1(f):
|
||||
def inner_translate_exc2(*args, **kwargs):
|
||||
try:
|
||||
return f(*args, **kwargs)
|
||||
except from_exc:
|
||||
raise to_exc
|
||||
|
||||
return inner_translate_exc2
|
||||
|
||||
return inner_translate_exc1
|
||||
|
||||
|
||||
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
|
||||
def is_leader():
|
||||
"""Does the current unit hold the juju leadership
|
||||
|
||||
Uses juju to determine whether the current unit is the leader of its peers
|
||||
"""
|
||||
cmd = ['is-leader', '--format=json']
|
||||
return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
|
||||
|
||||
|
||||
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
|
||||
def leader_get(attribute=None):
|
||||
"""Juju leader get value(s)"""
|
||||
cmd = ['leader-get', '--format=json'] + [attribute or '-']
|
||||
return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
|
||||
|
||||
|
||||
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
|
||||
def leader_set(settings=None, **kwargs):
|
||||
"""Juju leader set value(s)"""
|
||||
log("Juju leader-set '%s'" % (settings), level=DEBUG)
|
||||
cmd = ['leader-set']
|
||||
settings = settings or {}
|
||||
settings.update(kwargs)
|
||||
for k, v in settings.iteritems():
|
||||
if v is None:
|
||||
cmd.append('{}='.format(k))
|
||||
else:
|
||||
cmd.append('{}={}'.format(k, v))
|
||||
subprocess.check_call(cmd)
|
||||
|
|
|
@ -1,3 +1,19 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
"""Tools for working with the host system"""
|
||||
# Copyright 2012 Canonical Ltd.
|
||||
#
|
||||
|
@ -14,11 +30,12 @@ import string
|
|||
import subprocess
|
||||
import hashlib
|
||||
from contextlib import contextmanager
|
||||
|
||||
from collections import OrderedDict
|
||||
|
||||
from hookenv import log
|
||||
from fstab import Fstab
|
||||
import six
|
||||
|
||||
from .hookenv import log
|
||||
from .fstab import Fstab
|
||||
|
||||
|
||||
def service_start(service_name):
|
||||
|
@ -54,7 +71,9 @@ def service(action, service_name):
|
|||
def service_running(service):
|
||||
"""Determine whether a system service is running"""
|
||||
try:
|
||||
output = subprocess.check_output(['service', service, 'status'], stderr=subprocess.STDOUT)
|
||||
output = subprocess.check_output(
|
||||
['service', service, 'status'],
|
||||
stderr=subprocess.STDOUT).decode('UTF-8')
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
else:
|
||||
|
@ -67,9 +86,11 @@ def service_running(service):
|
|||
def service_available(service_name):
|
||||
"""Determine whether a system service is available"""
|
||||
try:
|
||||
subprocess.check_output(['service', service_name, 'status'], stderr=subprocess.STDOUT)
|
||||
subprocess.check_output(
|
||||
['service', service_name, 'status'],
|
||||
stderr=subprocess.STDOUT).decode('UTF-8')
|
||||
except subprocess.CalledProcessError as e:
|
||||
return 'unrecognized service' not in e.output
|
||||
return b'unrecognized service' not in e.output
|
||||
else:
|
||||
return True
|
||||
|
||||
|
@ -96,6 +117,26 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False):
|
|||
return user_info
|
||||
|
||||
|
||||
def add_group(group_name, system_group=False):
|
||||
"""Add a group to the system"""
|
||||
try:
|
||||
group_info = grp.getgrnam(group_name)
|
||||
log('group {0} already exists!'.format(group_name))
|
||||
except KeyError:
|
||||
log('creating group {0}'.format(group_name))
|
||||
cmd = ['addgroup']
|
||||
if system_group:
|
||||
cmd.append('--system')
|
||||
else:
|
||||
cmd.extend([
|
||||
'--group',
|
||||
])
|
||||
cmd.append(group_name)
|
||||
subprocess.check_call(cmd)
|
||||
group_info = grp.getgrnam(group_name)
|
||||
return group_info
|
||||
|
||||
|
||||
def add_user_to_group(username, group):
|
||||
"""Add a user to a group"""
|
||||
cmd = [
|
||||
|
@ -115,7 +156,7 @@ def rsync(from_path, to_path, flags='-r', options=None):
|
|||
cmd.append(from_path)
|
||||
cmd.append(to_path)
|
||||
log(" ".join(cmd))
|
||||
return subprocess.check_output(cmd).strip()
|
||||
return subprocess.check_output(cmd).decode('UTF-8').strip()
|
||||
|
||||
|
||||
def symlink(source, destination):
|
||||
|
@ -130,28 +171,31 @@ def symlink(source, destination):
|
|||
subprocess.check_call(cmd)
|
||||
|
||||
|
||||
def mkdir(path, owner='root', group='root', perms=0555, force=False):
|
||||
def mkdir(path, owner='root', group='root', perms=0o555, force=False):
|
||||
"""Create a directory"""
|
||||
log("Making dir {} {}:{} {:o}".format(path, owner, group,
|
||||
perms))
|
||||
uid = pwd.getpwnam(owner).pw_uid
|
||||
gid = grp.getgrnam(group).gr_gid
|
||||
realpath = os.path.abspath(path)
|
||||
if os.path.exists(realpath):
|
||||
if force and not os.path.isdir(realpath):
|
||||
path_exists = os.path.exists(realpath)
|
||||
if path_exists and force:
|
||||
if not os.path.isdir(realpath):
|
||||
log("Removing non-directory file {} prior to mkdir()".format(path))
|
||||
os.unlink(realpath)
|
||||
else:
|
||||
os.makedirs(realpath, perms)
|
||||
elif not path_exists:
|
||||
os.makedirs(realpath, perms)
|
||||
os.chown(realpath, uid, gid)
|
||||
os.chmod(realpath, perms)
|
||||
|
||||
|
||||
def write_file(path, content, owner='root', group='root', perms=0444):
|
||||
"""Create or overwrite a file with the contents of a string"""
|
||||
def write_file(path, content, owner='root', group='root', perms=0o444):
|
||||
"""Create or overwrite a file with the contents of a byte string."""
|
||||
log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
|
||||
uid = pwd.getpwnam(owner).pw_uid
|
||||
gid = grp.getgrnam(group).gr_gid
|
||||
with open(path, 'w') as target:
|
||||
with open(path, 'wb') as target:
|
||||
os.fchown(target.fileno(), uid, gid)
|
||||
os.fchmod(target.fileno(), perms)
|
||||
target.write(content)
|
||||
|
@ -177,7 +221,7 @@ def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"):
|
|||
cmd_args.extend([device, mountpoint])
|
||||
try:
|
||||
subprocess.check_output(cmd_args)
|
||||
except subprocess.CalledProcessError, e:
|
||||
except subprocess.CalledProcessError as e:
|
||||
log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output))
|
||||
return False
|
||||
|
||||
|
@ -191,7 +235,7 @@ def umount(mountpoint, persist=False):
|
|||
cmd_args = ['umount', mountpoint]
|
||||
try:
|
||||
subprocess.check_output(cmd_args)
|
||||
except subprocess.CalledProcessError, e:
|
||||
except subprocess.CalledProcessError as e:
|
||||
log('Error unmounting {}\n{}'.format(mountpoint, e.output))
|
||||
return False
|
||||
|
||||
|
@ -218,8 +262,8 @@ def file_hash(path, hash_type='md5'):
|
|||
"""
|
||||
if os.path.exists(path):
|
||||
h = getattr(hashlib, hash_type)()
|
||||
with open(path, 'r') as source:
|
||||
h.update(source.read()) # IGNORE:E1101 - it does have update
|
||||
with open(path, 'rb') as source:
|
||||
h.update(source.read())
|
||||
return h.hexdigest()
|
||||
else:
|
||||
return None
|
||||
|
@ -261,11 +305,11 @@ def restart_on_change(restart_map, stopstart=False):
|
|||
ceph_client_changed function.
|
||||
"""
|
||||
def wrap(f):
|
||||
def wrapped_f(*args):
|
||||
def wrapped_f(*args, **kwargs):
|
||||
checksums = {}
|
||||
for path in restart_map:
|
||||
checksums[path] = file_hash(path)
|
||||
f(*args)
|
||||
f(*args, **kwargs)
|
||||
restarts = []
|
||||
for path in restart_map:
|
||||
if checksums[path] != file_hash(path):
|
||||
|
@ -295,29 +339,33 @@ def lsb_release():
|
|||
def pwgen(length=None):
|
||||
"""Generate a random pasword."""
|
||||
if length is None:
|
||||
# A random length is ok to use a weak PRNG
|
||||
length = random.choice(range(35, 45))
|
||||
alphanumeric_chars = [
|
||||
l for l in (string.letters + string.digits)
|
||||
l for l in (string.ascii_letters + string.digits)
|
||||
if l not in 'l0QD1vAEIOUaeiou']
|
||||
# Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the
|
||||
# actual password
|
||||
random_generator = random.SystemRandom()
|
||||
random_chars = [
|
||||
random.choice(alphanumeric_chars) for _ in range(length)]
|
||||
random_generator.choice(alphanumeric_chars) for _ in range(length)]
|
||||
return(''.join(random_chars))
|
||||
|
||||
|
||||
def list_nics(nic_type):
|
||||
'''Return a list of nics of given type(s)'''
|
||||
if isinstance(nic_type, basestring):
|
||||
if isinstance(nic_type, six.string_types):
|
||||
int_types = [nic_type]
|
||||
else:
|
||||
int_types = nic_type
|
||||
interfaces = []
|
||||
for int_type in int_types:
|
||||
cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
|
||||
ip_output = subprocess.check_output(cmd).split('\n')
|
||||
ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
|
||||
ip_output = (line for line in ip_output if line)
|
||||
for line in ip_output:
|
||||
if line.split()[1].startswith(int_type):
|
||||
matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line)
|
||||
matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line)
|
||||
if matched:
|
||||
interface = matched.groups()[0]
|
||||
else:
|
||||
|
@ -335,7 +383,7 @@ def set_nic_mtu(nic, mtu):
|
|||
|
||||
def get_nic_mtu(nic):
|
||||
cmd = ['ip', 'addr', 'show', nic]
|
||||
ip_output = subprocess.check_output(cmd).split('\n')
|
||||
ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
|
||||
mtu = ""
|
||||
for line in ip_output:
|
||||
words = line.split()
|
||||
|
@ -346,7 +394,7 @@ def get_nic_mtu(nic):
|
|||
|
||||
def get_nic_hwaddr(nic):
|
||||
cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
|
||||
ip_output = subprocess.check_output(cmd)
|
||||
ip_output = subprocess.check_output(cmd).decode('UTF-8')
|
||||
hwaddr = ""
|
||||
words = ip_output.split()
|
||||
if 'link/ether' in words:
|
||||
|
@ -361,10 +409,13 @@ def cmp_pkgrevno(package, revno, pkgcache=None):
|
|||
* 0 => Installed revno is the same as supplied arg
|
||||
* -1 => Installed revno is less than supplied arg
|
||||
|
||||
This function imports apt_cache function from charmhelpers.fetch if
|
||||
the pkgcache argument is None. Be sure to add charmhelpers.fetch if
|
||||
you call this function, or pass an apt_pkg.Cache() instance.
|
||||
'''
|
||||
import apt_pkg
|
||||
from charmhelpers.fetch import apt_cache
|
||||
if not pkgcache:
|
||||
from charmhelpers.fetch import apt_cache
|
||||
pkgcache = apt_cache()
|
||||
pkg = pkgcache[package]
|
||||
return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)
|
||||
|
@ -379,13 +430,21 @@ def chdir(d):
|
|||
os.chdir(cur)
|
||||
|
||||
|
||||
def chownr(path, owner, group):
|
||||
def chownr(path, owner, group, follow_links=True):
|
||||
uid = pwd.getpwnam(owner).pw_uid
|
||||
gid = grp.getgrnam(group).gr_gid
|
||||
if follow_links:
|
||||
chown = os.chown
|
||||
else:
|
||||
chown = os.lchown
|
||||
|
||||
for root, dirs, files in os.walk(path):
|
||||
for name in dirs + files:
|
||||
full = os.path.join(root, name)
|
||||
broken_symlink = os.path.lexists(full) and not os.path.exists(full)
|
||||
if not broken_symlink:
|
||||
os.chown(full, uid, gid)
|
||||
chown(full, uid, gid)
|
||||
|
||||
|
||||
def lchownr(path, owner, group):
|
||||
chownr(path, owner, group, follow_links=False)
|
||||
|
|
|
@ -1,2 +1,18 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from .base import * # NOQA
|
||||
from .helpers import * # NOQA
|
||||
|
|
|
@ -1,7 +1,23 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
from collections import Iterable
|
||||
from inspect import getargspec
|
||||
from collections import Iterable, OrderedDict
|
||||
|
||||
from charmhelpers.core import host
|
||||
from charmhelpers.core import hookenv
|
||||
|
@ -103,7 +119,7 @@ class ServiceManager(object):
|
|||
"""
|
||||
self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json')
|
||||
self._ready = None
|
||||
self.services = {}
|
||||
self.services = OrderedDict()
|
||||
for service in services or []:
|
||||
service_name = service['service']
|
||||
self.services[service_name] = service
|
||||
|
@ -116,8 +132,8 @@ class ServiceManager(object):
|
|||
if hook_name == 'stop':
|
||||
self.stop_services()
|
||||
else:
|
||||
self.provide_data()
|
||||
self.reconfigure_services()
|
||||
self.provide_data()
|
||||
cfg = hookenv.config()
|
||||
if cfg.implicit_save:
|
||||
cfg.save()
|
||||
|
@ -129,15 +145,36 @@ class ServiceManager(object):
|
|||
A provider must have a `name` attribute, which indicates which relation
|
||||
to set data on, and a `provide_data()` method, which returns a dict of
|
||||
data to set.
|
||||
|
||||
The `provide_data()` method can optionally accept two parameters:
|
||||
|
||||
* ``remote_service`` The name of the remote service that the data will
|
||||
be provided to. The `provide_data()` method will be called once
|
||||
for each connected service (not unit). This allows the method to
|
||||
tailor its data to the given service.
|
||||
* ``service_ready`` Whether or not the service definition had all of
|
||||
its requirements met, and thus the ``data_ready`` callbacks run.
|
||||
|
||||
Note that the ``provided_data`` methods are now called **after** the
|
||||
``data_ready`` callbacks are run. This gives the ``data_ready`` callbacks
|
||||
a chance to generate any data necessary for the providing to the remote
|
||||
services.
|
||||
"""
|
||||
hook_name = hookenv.hook_name()
|
||||
for service in self.services.values():
|
||||
for service_name, service in self.services.items():
|
||||
service_ready = self.is_ready(service_name)
|
||||
for provider in service.get('provided_data', []):
|
||||
if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name):
|
||||
data = provider.provide_data()
|
||||
_ready = provider._is_ready(data) if hasattr(provider, '_is_ready') else data
|
||||
if _ready:
|
||||
hookenv.relation_set(None, data)
|
||||
for relid in hookenv.relation_ids(provider.name):
|
||||
units = hookenv.related_units(relid)
|
||||
if not units:
|
||||
continue
|
||||
remote_service = units[0].split('/')[0]
|
||||
argspec = getargspec(provider.provide_data)
|
||||
if len(argspec.args) > 1:
|
||||
data = provider.provide_data(remote_service, service_ready)
|
||||
else:
|
||||
data = provider.provide_data()
|
||||
if data:
|
||||
hookenv.relation_set(relid, data)
|
||||
|
||||
def reconfigure_services(self, *service_names):
|
||||
"""
|
||||
|
|
|
@ -1,3 +1,19 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
import yaml
|
||||
from charmhelpers.core import hookenv
|
||||
|
@ -29,12 +45,14 @@ class RelationContext(dict):
|
|||
"""
|
||||
name = None
|
||||
interface = None
|
||||
required_keys = []
|
||||
|
||||
def __init__(self, name=None, additional_required_keys=None):
|
||||
if not hasattr(self, 'required_keys'):
|
||||
self.required_keys = []
|
||||
|
||||
if name is not None:
|
||||
self.name = name
|
||||
if additional_required_keys is not None:
|
||||
if additional_required_keys:
|
||||
self.required_keys.extend(additional_required_keys)
|
||||
self.get_data()
|
||||
|
||||
|
@ -118,7 +136,10 @@ class MysqlRelation(RelationContext):
|
|||
"""
|
||||
name = 'db'
|
||||
interface = 'mysql'
|
||||
required_keys = ['host', 'user', 'password', 'database']
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.required_keys = ['host', 'user', 'password', 'database']
|
||||
RelationContext.__init__(self, *args, **kwargs)
|
||||
|
||||
|
||||
class HttpRelation(RelationContext):
|
||||
|
@ -130,7 +151,10 @@ class HttpRelation(RelationContext):
|
|||
"""
|
||||
name = 'website'
|
||||
interface = 'http'
|
||||
required_keys = ['host', 'port']
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.required_keys = ['host', 'port']
|
||||
RelationContext.__init__(self, *args, **kwargs)
|
||||
|
||||
def provide_data(self):
|
||||
return {
|
||||
|
@ -196,7 +220,7 @@ class StoredContext(dict):
|
|||
if not os.path.isabs(file_name):
|
||||
file_name = os.path.join(hookenv.charm_dir(), file_name)
|
||||
with open(file_name, 'w') as file_stream:
|
||||
os.fchmod(file_stream.fileno(), 0600)
|
||||
os.fchmod(file_stream.fileno(), 0o600)
|
||||
yaml.dump(config_data, file_stream)
|
||||
|
||||
def read_context(self, file_name):
|
||||
|
@ -211,15 +235,19 @@ class StoredContext(dict):
|
|||
|
||||
class TemplateCallback(ManagerCallback):
|
||||
"""
|
||||
Callback class that will render a Jinja2 template, for use as a ready action.
|
||||
Callback class that will render a Jinja2 template, for use as a ready
|
||||
action.
|
||||
|
||||
:param str source: The template source file, relative to
|
||||
`$CHARM_DIR/templates`
|
||||
|
||||
:param str source: The template source file, relative to `$CHARM_DIR/templates`
|
||||
:param str target: The target to write the rendered template to
|
||||
:param str owner: The owner of the rendered file
|
||||
:param str group: The group of the rendered file
|
||||
:param int perms: The permissions of the rendered file
|
||||
"""
|
||||
def __init__(self, source, target, owner='root', group='root', perms=0444):
|
||||
def __init__(self, source, target,
|
||||
owner='root', group='root', perms=0o444):
|
||||
self.source = source
|
||||
self.target = target
|
||||
self.owner = owner
|
||||
|
|
|
@ -0,0 +1,42 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import six
|
||||
|
||||
|
||||
def bool_from_string(value):
|
||||
"""Interpret string value as boolean.
|
||||
|
||||
Returns True if value translates to True otherwise False.
|
||||
"""
|
||||
if isinstance(value, six.string_types):
|
||||
value = six.text_type(value)
|
||||
else:
|
||||
msg = "Unable to interpret non-string value '%s' as boolean" % (value)
|
||||
raise ValueError(msg)
|
||||
|
||||
value = value.strip().lower()
|
||||
|
||||
if value in ['y', 'yes', 'true', 't', 'on']:
|
||||
return True
|
||||
elif value in ['n', 'no', 'false', 'f', 'off']:
|
||||
return False
|
||||
|
||||
msg = "Unable to interpret string value '%s' as boolean" % (value)
|
||||
raise ValueError(msg)
|
|
@ -1,7 +1,21 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
|
||||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import yaml
|
||||
|
||||
|
@ -10,25 +24,33 @@ from subprocess import check_call
|
|||
from charmhelpers.core.hookenv import (
|
||||
log,
|
||||
DEBUG,
|
||||
ERROR,
|
||||
)
|
||||
|
||||
__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
|
||||
|
||||
|
||||
def create(sysctl_dict, sysctl_file):
|
||||
"""Creates a sysctl.conf file from a YAML associative array
|
||||
|
||||
:param sysctl_dict: a dict of sysctl options eg { 'kernel.max_pid': 1337 }
|
||||
:type sysctl_dict: dict
|
||||
:param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }"
|
||||
:type sysctl_dict: str
|
||||
:param sysctl_file: path to the sysctl file to be saved
|
||||
:type sysctl_file: str or unicode
|
||||
:returns: None
|
||||
"""
|
||||
sysctl_dict = yaml.load(sysctl_dict)
|
||||
try:
|
||||
sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
|
||||
except yaml.YAMLError:
|
||||
log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
|
||||
level=ERROR)
|
||||
return
|
||||
|
||||
with open(sysctl_file, "w") as fd:
|
||||
for key, value in sysctl_dict.items():
|
||||
for key, value in sysctl_dict_parsed.items():
|
||||
fd.write("{}={}\n".format(key, value))
|
||||
|
||||
log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict),
|
||||
log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed),
|
||||
level=DEBUG)
|
||||
|
||||
check_call(["sysctl", "-p", sysctl_file])
|
||||
|
|
|
@ -1,10 +1,27 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
|
||||
from charmhelpers.core import host
|
||||
from charmhelpers.core import hookenv
|
||||
|
||||
|
||||
def render(source, target, context, owner='root', group='root', perms=0444, templates_dir=None):
|
||||
def render(source, target, context, owner='root', group='root',
|
||||
perms=0o444, templates_dir=None, encoding='UTF-8'):
|
||||
"""
|
||||
Render a template.
|
||||
|
||||
|
@ -47,5 +64,5 @@ def render(source, target, context, owner='root', group='root', perms=0444, temp
|
|||
level=hookenv.ERROR)
|
||||
raise e
|
||||
content = template.render(context)
|
||||
host.mkdir(os.path.dirname(target))
|
||||
host.write_file(target, content, owner, group, perms)
|
||||
host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
|
||||
host.write_file(target, content.encode(encoding), owner, group, perms)
|
||||
|
|
|
@ -0,0 +1,477 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
#
|
||||
# Authors:
|
||||
# Kapil Thangavelu <kapil.foss@gmail.com>
|
||||
#
|
||||
"""
|
||||
Intro
|
||||
-----
|
||||
|
||||
A simple way to store state in units. This provides a key value
|
||||
storage with support for versioned, transactional operation,
|
||||
and can calculate deltas from previous values to simplify unit logic
|
||||
when processing changes.
|
||||
|
||||
|
||||
Hook Integration
|
||||
----------------
|
||||
|
||||
There are several extant frameworks for hook execution, including
|
||||
|
||||
- charmhelpers.core.hookenv.Hooks
|
||||
- charmhelpers.core.services.ServiceManager
|
||||
|
||||
The storage classes are framework agnostic, one simple integration is
|
||||
via the HookData contextmanager. It will record the current hook
|
||||
execution environment (including relation data, config data, etc.),
|
||||
setup a transaction and allow easy access to the changes from
|
||||
previously seen values. One consequence of the integration is the
|
||||
reservation of particular keys ('rels', 'unit', 'env', 'config',
|
||||
'charm_revisions') for their respective values.
|
||||
|
||||
Here's a fully worked integration example using hookenv.Hooks::
|
||||
|
||||
from charmhelper.core import hookenv, unitdata
|
||||
|
||||
hook_data = unitdata.HookData()
|
||||
db = unitdata.kv()
|
||||
hooks = hookenv.Hooks()
|
||||
|
||||
@hooks.hook
|
||||
def config_changed():
|
||||
# Print all changes to configuration from previously seen
|
||||
# values.
|
||||
for changed, (prev, cur) in hook_data.conf.items():
|
||||
print('config changed', changed,
|
||||
'previous value', prev,
|
||||
'current value', cur)
|
||||
|
||||
# Get some unit specific bookeeping
|
||||
if not db.get('pkg_key'):
|
||||
key = urllib.urlopen('https://example.com/pkg_key').read()
|
||||
db.set('pkg_key', key)
|
||||
|
||||
# Directly access all charm config as a mapping.
|
||||
conf = db.getrange('config', True)
|
||||
|
||||
# Directly access all relation data as a mapping
|
||||
rels = db.getrange('rels', True)
|
||||
|
||||
if __name__ == '__main__':
|
||||
with hook_data():
|
||||
hook.execute()
|
||||
|
||||
|
||||
A more basic integration is via the hook_scope context manager which simply
|
||||
manages transaction scope (and records hook name, and timestamp)::
|
||||
|
||||
>>> from unitdata import kv
|
||||
>>> db = kv()
|
||||
>>> with db.hook_scope('install'):
|
||||
... # do work, in transactional scope.
|
||||
... db.set('x', 1)
|
||||
>>> db.get('x')
|
||||
1
|
||||
|
||||
|
||||
Usage
|
||||
-----
|
||||
|
||||
Values are automatically json de/serialized to preserve basic typing
|
||||
and complex data struct capabilities (dicts, lists, ints, booleans, etc).
|
||||
|
||||
Individual values can be manipulated via get/set::
|
||||
|
||||
>>> kv.set('y', True)
|
||||
>>> kv.get('y')
|
||||
True
|
||||
|
||||
# We can set complex values (dicts, lists) as a single key.
|
||||
>>> kv.set('config', {'a': 1, 'b': True'})
|
||||
|
||||
# Also supports returning dictionaries as a record which
|
||||
# provides attribute access.
|
||||
>>> config = kv.get('config', record=True)
|
||||
>>> config.b
|
||||
True
|
||||
|
||||
|
||||
Groups of keys can be manipulated with update/getrange::
|
||||
|
||||
>>> kv.update({'z': 1, 'y': 2}, prefix="gui.")
|
||||
>>> kv.getrange('gui.', strip=True)
|
||||
{'z': 1, 'y': 2}
|
||||
|
||||
When updating values, its very helpful to understand which values
|
||||
have actually changed and how have they changed. The storage
|
||||
provides a delta method to provide for this::
|
||||
|
||||
>>> data = {'debug': True, 'option': 2}
|
||||
>>> delta = kv.delta(data, 'config.')
|
||||
>>> delta.debug.previous
|
||||
None
|
||||
>>> delta.debug.current
|
||||
True
|
||||
>>> delta
|
||||
{'debug': (None, True), 'option': (None, 2)}
|
||||
|
||||
Note the delta method does not persist the actual change, it needs to
|
||||
be explicitly saved via 'update' method::
|
||||
|
||||
>>> kv.update(data, 'config.')
|
||||
|
||||
Values modified in the context of a hook scope retain historical values
|
||||
associated to the hookname.
|
||||
|
||||
>>> with db.hook_scope('config-changed'):
|
||||
... db.set('x', 42)
|
||||
>>> db.gethistory('x')
|
||||
[(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'),
|
||||
(2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')]
|
||||
|
||||
"""
|
||||
|
||||
import collections
|
||||
import contextlib
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import pprint
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
__author__ = 'Kapil Thangavelu <kapil.foss@gmail.com>'
|
||||
|
||||
|
||||
class Storage(object):
|
||||
"""Simple key value database for local unit state within charms.
|
||||
|
||||
Modifications are automatically committed at hook exit. That's
|
||||
currently regardless of exit code.
|
||||
|
||||
To support dicts, lists, integer, floats, and booleans values
|
||||
are automatically json encoded/decoded.
|
||||
"""
|
||||
def __init__(self, path=None):
|
||||
self.db_path = path
|
||||
if path is None:
|
||||
self.db_path = os.path.join(
|
||||
os.environ.get('CHARM_DIR', ''), '.unit-state.db')
|
||||
self.conn = sqlite3.connect('%s' % self.db_path)
|
||||
self.cursor = self.conn.cursor()
|
||||
self.revision = None
|
||||
self._closed = False
|
||||
self._init()
|
||||
|
||||
def close(self):
|
||||
if self._closed:
|
||||
return
|
||||
self.flush(False)
|
||||
self.cursor.close()
|
||||
self.conn.close()
|
||||
self._closed = True
|
||||
|
||||
def _scoped_query(self, stmt, params=None):
|
||||
if params is None:
|
||||
params = []
|
||||
return stmt, params
|
||||
|
||||
def get(self, key, default=None, record=False):
|
||||
self.cursor.execute(
|
||||
*self._scoped_query(
|
||||
'select data from kv where key=?', [key]))
|
||||
result = self.cursor.fetchone()
|
||||
if not result:
|
||||
return default
|
||||
if record:
|
||||
return Record(json.loads(result[0]))
|
||||
return json.loads(result[0])
|
||||
|
||||
def getrange(self, key_prefix, strip=False):
|
||||
stmt = "select key, data from kv where key like '%s%%'" % key_prefix
|
||||
self.cursor.execute(*self._scoped_query(stmt))
|
||||
result = self.cursor.fetchall()
|
||||
|
||||
if not result:
|
||||
return None
|
||||
if not strip:
|
||||
key_prefix = ''
|
||||
return dict([
|
||||
(k[len(key_prefix):], json.loads(v)) for k, v in result])
|
||||
|
||||
def update(self, mapping, prefix=""):
|
||||
for k, v in mapping.items():
|
||||
self.set("%s%s" % (prefix, k), v)
|
||||
|
||||
def unset(self, key):
|
||||
self.cursor.execute('delete from kv where key=?', [key])
|
||||
if self.revision and self.cursor.rowcount:
|
||||
self.cursor.execute(
|
||||
'insert into kv_revisions values (?, ?, ?)',
|
||||
[key, self.revision, json.dumps('DELETED')])
|
||||
|
||||
def set(self, key, value):
|
||||
serialized = json.dumps(value)
|
||||
|
||||
self.cursor.execute(
|
||||
'select data from kv where key=?', [key])
|
||||
exists = self.cursor.fetchone()
|
||||
|
||||
# Skip mutations to the same value
|
||||
if exists:
|
||||
if exists[0] == serialized:
|
||||
return value
|
||||
|
||||
if not exists:
|
||||
self.cursor.execute(
|
||||
'insert into kv (key, data) values (?, ?)',
|
||||
(key, serialized))
|
||||
else:
|
||||
self.cursor.execute('''
|
||||
update kv
|
||||
set data = ?
|
||||
where key = ?''', [serialized, key])
|
||||
|
||||
# Save
|
||||
if not self.revision:
|
||||
return value
|
||||
|
||||
self.cursor.execute(
|
||||
'select 1 from kv_revisions where key=? and revision=?',
|
||||
[key, self.revision])
|
||||
exists = self.cursor.fetchone()
|
||||
|
||||
if not exists:
|
||||
self.cursor.execute(
|
||||
'''insert into kv_revisions (
|
||||
revision, key, data) values (?, ?, ?)''',
|
||||
(self.revision, key, serialized))
|
||||
else:
|
||||
self.cursor.execute(
|
||||
'''
|
||||
update kv_revisions
|
||||
set data = ?
|
||||
where key = ?
|
||||
and revision = ?''',
|
||||
[serialized, key, self.revision])
|
||||
|
||||
return value
|
||||
|
||||
def delta(self, mapping, prefix):
|
||||
"""
|
||||
return a delta containing values that have changed.
|
||||
"""
|
||||
previous = self.getrange(prefix, strip=True)
|
||||
if not previous:
|
||||
pk = set()
|
||||
else:
|
||||
pk = set(previous.keys())
|
||||
ck = set(mapping.keys())
|
||||
delta = DeltaSet()
|
||||
|
||||
# added
|
||||
for k in ck.difference(pk):
|
||||
delta[k] = Delta(None, mapping[k])
|
||||
|
||||
# removed
|
||||
for k in pk.difference(ck):
|
||||
delta[k] = Delta(previous[k], None)
|
||||
|
||||
# changed
|
||||
for k in pk.intersection(ck):
|
||||
c = mapping[k]
|
||||
p = previous[k]
|
||||
if c != p:
|
||||
delta[k] = Delta(p, c)
|
||||
|
||||
return delta
|
||||
|
||||
@contextlib.contextmanager
|
||||
def hook_scope(self, name=""):
|
||||
"""Scope all future interactions to the current hook execution
|
||||
revision."""
|
||||
assert not self.revision
|
||||
self.cursor.execute(
|
||||
'insert into hooks (hook, date) values (?, ?)',
|
||||
(name or sys.argv[0],
|
||||
datetime.datetime.utcnow().isoformat()))
|
||||
self.revision = self.cursor.lastrowid
|
||||
try:
|
||||
yield self.revision
|
||||
self.revision = None
|
||||
except:
|
||||
self.flush(False)
|
||||
self.revision = None
|
||||
raise
|
||||
else:
|
||||
self.flush()
|
||||
|
||||
def flush(self, save=True):
|
||||
if save:
|
||||
self.conn.commit()
|
||||
elif self._closed:
|
||||
return
|
||||
else:
|
||||
self.conn.rollback()
|
||||
|
||||
def _init(self):
|
||||
self.cursor.execute('''
|
||||
create table if not exists kv (
|
||||
key text,
|
||||
data text,
|
||||
primary key (key)
|
||||
)''')
|
||||
self.cursor.execute('''
|
||||
create table if not exists kv_revisions (
|
||||
key text,
|
||||
revision integer,
|
||||
data text,
|
||||
primary key (key, revision)
|
||||
)''')
|
||||
self.cursor.execute('''
|
||||
create table if not exists hooks (
|
||||
version integer primary key autoincrement,
|
||||
hook text,
|
||||
date text
|
||||
)''')
|
||||
self.conn.commit()
|
||||
|
||||
def gethistory(self, key, deserialize=False):
|
||||
self.cursor.execute(
|
||||
'''
|
||||
select kv.revision, kv.key, kv.data, h.hook, h.date
|
||||
from kv_revisions kv,
|
||||
hooks h
|
||||
where kv.key=?
|
||||
and kv.revision = h.version
|
||||
''', [key])
|
||||
if deserialize is False:
|
||||
return self.cursor.fetchall()
|
||||
return map(_parse_history, self.cursor.fetchall())
|
||||
|
||||
def debug(self, fh=sys.stderr):
|
||||
self.cursor.execute('select * from kv')
|
||||
pprint.pprint(self.cursor.fetchall(), stream=fh)
|
||||
self.cursor.execute('select * from kv_revisions')
|
||||
pprint.pprint(self.cursor.fetchall(), stream=fh)
|
||||
|
||||
|
||||
def _parse_history(d):
|
||||
return (d[0], d[1], json.loads(d[2]), d[3],
|
||||
datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f"))
|
||||
|
||||
|
||||
class HookData(object):
|
||||
"""Simple integration for existing hook exec frameworks.
|
||||
|
||||
Records all unit information, and stores deltas for processing
|
||||
by the hook.
|
||||
|
||||
Sample::
|
||||
|
||||
from charmhelper.core import hookenv, unitdata
|
||||
|
||||
changes = unitdata.HookData()
|
||||
db = unitdata.kv()
|
||||
hooks = hookenv.Hooks()
|
||||
|
||||
@hooks.hook
|
||||
def config_changed():
|
||||
# View all changes to configuration
|
||||
for changed, (prev, cur) in changes.conf.items():
|
||||
print('config changed', changed,
|
||||
'previous value', prev,
|
||||
'current value', cur)
|
||||
|
||||
# Get some unit specific bookeeping
|
||||
if not db.get('pkg_key'):
|
||||
key = urllib.urlopen('https://example.com/pkg_key').read()
|
||||
db.set('pkg_key', key)
|
||||
|
||||
if __name__ == '__main__':
|
||||
with changes():
|
||||
hook.execute()
|
||||
|
||||
"""
|
||||
def __init__(self):
|
||||
self.kv = kv()
|
||||
self.conf = None
|
||||
self.rels = None
|
||||
|
||||
@contextlib.contextmanager
|
||||
def __call__(self):
|
||||
from charmhelpers.core import hookenv
|
||||
hook_name = hookenv.hook_name()
|
||||
|
||||
with self.kv.hook_scope(hook_name):
|
||||
self._record_charm_version(hookenv.charm_dir())
|
||||
delta_config, delta_relation = self._record_hook(hookenv)
|
||||
yield self.kv, delta_config, delta_relation
|
||||
|
||||
def _record_charm_version(self, charm_dir):
|
||||
# Record revisions.. charm revisions are meaningless
|
||||
# to charm authors as they don't control the revision.
|
||||
# so logic dependnent on revision is not particularly
|
||||
# useful, however it is useful for debugging analysis.
|
||||
charm_rev = open(
|
||||
os.path.join(charm_dir, 'revision')).read().strip()
|
||||
charm_rev = charm_rev or '0'
|
||||
revs = self.kv.get('charm_revisions', [])
|
||||
if charm_rev not in revs:
|
||||
revs.append(charm_rev.strip() or '0')
|
||||
self.kv.set('charm_revisions', revs)
|
||||
|
||||
def _record_hook(self, hookenv):
|
||||
data = hookenv.execution_environment()
|
||||
self.conf = conf_delta = self.kv.delta(data['conf'], 'config')
|
||||
self.rels = rels_delta = self.kv.delta(data['rels'], 'rels')
|
||||
self.kv.set('env', dict(data['env']))
|
||||
self.kv.set('unit', data['unit'])
|
||||
self.kv.set('relid', data.get('relid'))
|
||||
return conf_delta, rels_delta
|
||||
|
||||
|
||||
class Record(dict):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def __getattr__(self, k):
|
||||
if k in self:
|
||||
return self[k]
|
||||
raise AttributeError(k)
|
||||
|
||||
|
||||
class DeltaSet(Record):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
Delta = collections.namedtuple('Delta', ['previous', 'current'])
|
||||
|
||||
|
||||
_KV = None
|
||||
|
||||
|
||||
def kv():
|
||||
global _KV
|
||||
if _KV is None:
|
||||
_KV = Storage()
|
||||
return _KV
|
|
@ -1,3 +1,19 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import importlib
|
||||
from tempfile import NamedTemporaryFile
|
||||
import time
|
||||
|
@ -5,10 +21,6 @@ from yaml import safe_load
|
|||
from charmhelpers.core.host import (
|
||||
lsb_release
|
||||
)
|
||||
from urlparse import (
|
||||
urlparse,
|
||||
urlunparse,
|
||||
)
|
||||
import subprocess
|
||||
from charmhelpers.core.hookenv import (
|
||||
config,
|
||||
|
@ -16,6 +28,12 @@ from charmhelpers.core.hookenv import (
|
|||
)
|
||||
import os
|
||||
|
||||
import six
|
||||
if six.PY3:
|
||||
from urllib.parse import urlparse, urlunparse
|
||||
else:
|
||||
from urlparse import urlparse, urlunparse
|
||||
|
||||
|
||||
CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
|
||||
deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
|
||||
|
@ -62,9 +80,16 @@ CLOUD_ARCHIVE_POCKETS = {
|
|||
'trusty-juno/updates': 'trusty-updates/juno',
|
||||
'trusty-updates/juno': 'trusty-updates/juno',
|
||||
'juno/proposed': 'trusty-proposed/juno',
|
||||
'juno/proposed': 'trusty-proposed/juno',
|
||||
'trusty-juno/proposed': 'trusty-proposed/juno',
|
||||
'trusty-proposed/juno': 'trusty-proposed/juno',
|
||||
# Kilo
|
||||
'kilo': 'trusty-updates/kilo',
|
||||
'trusty-kilo': 'trusty-updates/kilo',
|
||||
'trusty-kilo/updates': 'trusty-updates/kilo',
|
||||
'trusty-updates/kilo': 'trusty-updates/kilo',
|
||||
'kilo/proposed': 'trusty-proposed/kilo',
|
||||
'trusty-kilo/proposed': 'trusty-proposed/kilo',
|
||||
'trusty-proposed/kilo': 'trusty-proposed/kilo',
|
||||
}
|
||||
|
||||
# The order of this list is very important. Handlers should be listed in from
|
||||
|
@ -133,7 +158,7 @@ def filter_installed_packages(packages):
|
|||
|
||||
def apt_cache(in_memory=True):
|
||||
"""Build and return an apt cache"""
|
||||
import apt_pkg
|
||||
from apt import apt_pkg
|
||||
apt_pkg.init()
|
||||
if in_memory:
|
||||
apt_pkg.config.set("Dir::Cache::pkgcache", "")
|
||||
|
@ -149,7 +174,7 @@ def apt_install(packages, options=None, fatal=False):
|
|||
cmd = ['apt-get', '--assume-yes']
|
||||
cmd.extend(options)
|
||||
cmd.append('install')
|
||||
if isinstance(packages, basestring):
|
||||
if isinstance(packages, six.string_types):
|
||||
cmd.append(packages)
|
||||
else:
|
||||
cmd.extend(packages)
|
||||
|
@ -182,7 +207,7 @@ def apt_update(fatal=False):
|
|||
def apt_purge(packages, fatal=False):
|
||||
"""Purge one or more packages"""
|
||||
cmd = ['apt-get', '--assume-yes', 'purge']
|
||||
if isinstance(packages, basestring):
|
||||
if isinstance(packages, six.string_types):
|
||||
cmd.append(packages)
|
||||
else:
|
||||
cmd.extend(packages)
|
||||
|
@ -193,7 +218,7 @@ def apt_purge(packages, fatal=False):
|
|||
def apt_hold(packages, fatal=False):
|
||||
"""Hold one or more packages"""
|
||||
cmd = ['apt-mark', 'hold']
|
||||
if isinstance(packages, basestring):
|
||||
if isinstance(packages, six.string_types):
|
||||
cmd.append(packages)
|
||||
else:
|
||||
cmd.extend(packages)
|
||||
|
@ -260,7 +285,7 @@ def add_source(source, key=None):
|
|||
|
||||
if key:
|
||||
if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
|
||||
with NamedTemporaryFile() as key_file:
|
||||
with NamedTemporaryFile('w+') as key_file:
|
||||
key_file.write(key)
|
||||
key_file.flush()
|
||||
key_file.seek(0)
|
||||
|
@ -297,14 +322,14 @@ def configure_sources(update=False,
|
|||
sources = safe_load((config(sources_var) or '').strip()) or []
|
||||
keys = safe_load((config(keys_var) or '').strip()) or None
|
||||
|
||||
if isinstance(sources, basestring):
|
||||
if isinstance(sources, six.string_types):
|
||||
sources = [sources]
|
||||
|
||||
if keys is None:
|
||||
for source in sources:
|
||||
add_source(source, None)
|
||||
else:
|
||||
if isinstance(keys, basestring):
|
||||
if isinstance(keys, six.string_types):
|
||||
keys = [keys]
|
||||
|
||||
if len(sources) != len(keys):
|
||||
|
@ -401,7 +426,7 @@ def _run_apt_command(cmd, fatal=False):
|
|||
while result is None or result == APT_NO_LOCK:
|
||||
try:
|
||||
result = subprocess.check_call(cmd, env=env)
|
||||
except subprocess.CalledProcessError, e:
|
||||
except subprocess.CalledProcessError as e:
|
||||
retry_count = retry_count + 1
|
||||
if retry_count > APT_NO_LOCK_RETRY_COUNT:
|
||||
raise
|
||||
|
|
|
@ -1,8 +1,22 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
import urllib2
|
||||
from urllib import urlretrieve
|
||||
import urlparse
|
||||
import hashlib
|
||||
import re
|
||||
|
||||
from charmhelpers.fetch import (
|
||||
BaseFetchHandler,
|
||||
|
@ -14,6 +28,41 @@ from charmhelpers.payload.archive import (
|
|||
)
|
||||
from charmhelpers.core.host import mkdir, check_hash
|
||||
|
||||
import six
|
||||
if six.PY3:
|
||||
from urllib.request import (
|
||||
build_opener, install_opener, urlopen, urlretrieve,
|
||||
HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
|
||||
)
|
||||
from urllib.parse import urlparse, urlunparse, parse_qs
|
||||
from urllib.error import URLError
|
||||
else:
|
||||
from urllib import urlretrieve
|
||||
from urllib2 import (
|
||||
build_opener, install_opener, urlopen,
|
||||
HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
|
||||
URLError
|
||||
)
|
||||
from urlparse import urlparse, urlunparse, parse_qs
|
||||
|
||||
|
||||
def splituser(host):
|
||||
'''urllib.splituser(), but six's support of this seems broken'''
|
||||
_userprog = re.compile('^(.*)@(.*)$')
|
||||
match = _userprog.match(host)
|
||||
if match:
|
||||
return match.group(1, 2)
|
||||
return None, host
|
||||
|
||||
|
||||
def splitpasswd(user):
|
||||
'''urllib.splitpasswd(), but six's support of this is missing'''
|
||||
_passwdprog = re.compile('^([^:]*):(.*)$', re.S)
|
||||
match = _passwdprog.match(user)
|
||||
if match:
|
||||
return match.group(1, 2)
|
||||
return user, None
|
||||
|
||||
|
||||
class ArchiveUrlFetchHandler(BaseFetchHandler):
|
||||
"""
|
||||
|
@ -42,20 +91,20 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
|
|||
"""
|
||||
# propogate all exceptions
|
||||
# URLError, OSError, etc
|
||||
proto, netloc, path, params, query, fragment = urlparse.urlparse(source)
|
||||
proto, netloc, path, params, query, fragment = urlparse(source)
|
||||
if proto in ('http', 'https'):
|
||||
auth, barehost = urllib2.splituser(netloc)
|
||||
auth, barehost = splituser(netloc)
|
||||
if auth is not None:
|
||||
source = urlparse.urlunparse((proto, barehost, path, params, query, fragment))
|
||||
username, password = urllib2.splitpasswd(auth)
|
||||
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
|
||||
source = urlunparse((proto, barehost, path, params, query, fragment))
|
||||
username, password = splitpasswd(auth)
|
||||
passman = HTTPPasswordMgrWithDefaultRealm()
|
||||
# Realm is set to None in add_password to force the username and password
|
||||
# to be used whatever the realm
|
||||
passman.add_password(None, source, username, password)
|
||||
authhandler = urllib2.HTTPBasicAuthHandler(passman)
|
||||
opener = urllib2.build_opener(authhandler)
|
||||
urllib2.install_opener(opener)
|
||||
response = urllib2.urlopen(source)
|
||||
authhandler = HTTPBasicAuthHandler(passman)
|
||||
opener = build_opener(authhandler)
|
||||
install_opener(opener)
|
||||
response = urlopen(source)
|
||||
try:
|
||||
with open(dest, 'w') as dest_file:
|
||||
dest_file.write(response.read())
|
||||
|
@ -91,17 +140,21 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
|
|||
url_parts = self.parse_url(source)
|
||||
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
|
||||
if not os.path.exists(dest_dir):
|
||||
mkdir(dest_dir, perms=0755)
|
||||
mkdir(dest_dir, perms=0o755)
|
||||
dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path))
|
||||
try:
|
||||
self.download(source, dld_file)
|
||||
except urllib2.URLError as e:
|
||||
except URLError as e:
|
||||
raise UnhandledSource(e.reason)
|
||||
except OSError as e:
|
||||
raise UnhandledSource(e.strerror)
|
||||
options = urlparse.parse_qs(url_parts.fragment)
|
||||
options = parse_qs(url_parts.fragment)
|
||||
for key, value in options.items():
|
||||
if key in hashlib.algorithms:
|
||||
if not six.PY3:
|
||||
algorithms = hashlib.algorithms
|
||||
else:
|
||||
algorithms = hashlib.algorithms_available
|
||||
if key in algorithms:
|
||||
check_hash(dld_file, value, key)
|
||||
if checksum:
|
||||
check_hash(dld_file, checksum, hash_type)
|
||||
|
|
|
@ -1,3 +1,19 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
from charmhelpers.fetch import (
|
||||
BaseFetchHandler,
|
||||
|
@ -5,12 +21,18 @@ from charmhelpers.fetch import (
|
|||
)
|
||||
from charmhelpers.core.host import mkdir
|
||||
|
||||
import six
|
||||
if six.PY3:
|
||||
raise ImportError('bzrlib does not support Python3')
|
||||
|
||||
try:
|
||||
from bzrlib.branch import Branch
|
||||
from bzrlib import bzrdir, workingtree, errors
|
||||
except ImportError:
|
||||
from charmhelpers.fetch import apt_install
|
||||
apt_install("python-bzrlib")
|
||||
from bzrlib.branch import Branch
|
||||
from bzrlib import bzrdir, workingtree, errors
|
||||
|
||||
|
||||
class BzrUrlFetchHandler(BaseFetchHandler):
|
||||
|
@ -30,9 +52,15 @@ class BzrUrlFetchHandler(BaseFetchHandler):
|
|||
if url_parts.scheme == "lp":
|
||||
from bzrlib.plugin import load_plugins
|
||||
load_plugins()
|
||||
try:
|
||||
local_branch = bzrdir.BzrDir.create_branch_convenience(dest)
|
||||
except errors.AlreadyControlDirError:
|
||||
local_branch = Branch.open(dest)
|
||||
try:
|
||||
remote_branch = Branch.open(source)
|
||||
remote_branch.bzrdir.sprout(dest).open_branch()
|
||||
remote_branch.push(local_branch)
|
||||
tree = workingtree.WorkingTree.open(dest)
|
||||
tree.update()
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
|
@ -42,7 +70,7 @@ class BzrUrlFetchHandler(BaseFetchHandler):
|
|||
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
|
||||
branch_name)
|
||||
if not os.path.exists(dest_dir):
|
||||
mkdir(dest_dir, perms=0755)
|
||||
mkdir(dest_dir, perms=0o755)
|
||||
try:
|
||||
self.branch(source, dest_dir)
|
||||
except OSError as e:
|
||||
|
|
|
@ -1,3 +1,19 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
from charmhelpers.fetch import (
|
||||
BaseFetchHandler,
|
||||
|
@ -5,6 +21,10 @@ from charmhelpers.fetch import (
|
|||
)
|
||||
from charmhelpers.core.host import mkdir
|
||||
|
||||
import six
|
||||
if six.PY3:
|
||||
raise ImportError('GitPython does not support Python 3')
|
||||
|
||||
try:
|
||||
from git import Repo
|
||||
except ImportError:
|
||||
|
@ -12,33 +32,42 @@ except ImportError:
|
|||
apt_install("python-git")
|
||||
from git import Repo
|
||||
|
||||
from git.exc import GitCommandError # noqa E402
|
||||
|
||||
|
||||
class GitUrlFetchHandler(BaseFetchHandler):
|
||||
"""Handler for git branches via generic and github URLs"""
|
||||
def can_handle(self, source):
|
||||
url_parts = self.parse_url(source)
|
||||
#TODO (mattyw) no support for ssh git@ yet
|
||||
# TODO (mattyw) no support for ssh git@ yet
|
||||
if url_parts.scheme not in ('http', 'https', 'git'):
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
def clone(self, source, dest, branch):
|
||||
def clone(self, source, dest, branch, depth=None):
|
||||
if not self.can_handle(source):
|
||||
raise UnhandledSource("Cannot handle {}".format(source))
|
||||
|
||||
repo = Repo.clone_from(source, dest)
|
||||
repo.git.checkout(branch)
|
||||
if depth:
|
||||
Repo.clone_from(source, dest, branch=branch, depth=depth)
|
||||
else:
|
||||
Repo.clone_from(source, dest, branch=branch)
|
||||
|
||||
def install(self, source, branch="master"):
|
||||
def install(self, source, branch="master", dest=None, depth=None):
|
||||
url_parts = self.parse_url(source)
|
||||
branch_name = url_parts.path.strip("/").split("/")[-1]
|
||||
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
|
||||
branch_name)
|
||||
if dest:
|
||||
dest_dir = os.path.join(dest, branch_name)
|
||||
else:
|
||||
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
|
||||
branch_name)
|
||||
if not os.path.exists(dest_dir):
|
||||
mkdir(dest_dir, perms=0755)
|
||||
mkdir(dest_dir, perms=0o755)
|
||||
try:
|
||||
self.clone(source, dest_dir, branch)
|
||||
self.clone(source, dest_dir, branch, depth)
|
||||
except GitCommandError as e:
|
||||
raise UnhandledSource(e.message)
|
||||
except OSError as e:
|
||||
raise UnhandledSource(e.strerror)
|
||||
return dest_dir
|
||||
|
|
|
@ -1 +1,17 @@
|
|||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
"Tools for working with files injected into a charm just before deployment."
|
||||
|
|
|
@ -1,5 +1,21 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
#
|
||||
# charm-helpers is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# charm-helpers is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
|
|
Loading…
Reference in New Issue