From a6ac97ae7d33015864eb1c010e8068051c363eba Mon Sep 17 00:00:00 2001 From: Hui Xiang Date: Sat, 20 Sep 2014 01:05:07 +0800 Subject: [PATCH] Use get_ipv6_addr()[0] directly --- hooks/charmhelpers/contrib/network/ip.py | 110 ++++++++++++++++++++--- hooks/charmhelpers/core/hookenv.py | 55 ++++++++---- hooks/charmhelpers/core/host.py | 8 +- hooks/charmhelpers/core/services/base.py | 26 ++++-- hooks/charmhelpers/fetch/__init__.py | 63 ++++++++++--- hooks/charmhelpers/fetch/archiveurl.py | 40 +++++++++ hooks/hooks.py | 8 +- hooks/utils.py | 2 +- 8 files changed, 252 insertions(+), 60 deletions(-) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 7edbcc4..b859a09 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -1,10 +1,11 @@ +import glob import sys from functools import partial from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( - ERROR, log, config, + ERROR, log, ) try: @@ -156,19 +157,102 @@ get_iface_for_address = partial(_get_for_address, key='iface') get_netmask_for_address = partial(_get_for_address, key='netmask') -def get_ipv6_addr(iface="eth0"): +def format_ipv6_addr(address): + """ + IPv6 needs to be wrapped with [] in url link to parse correctly. + """ + if is_ipv6(address): + address = "[%s]" % address + else: + log("Not an valid ipv6 address: %s" % address, + level=ERROR) + address = None + return address + + +def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, fatal=True, exc_list=None): + """ + Return the assigned IP address for a given interface, if any, or []. + """ + # Extract nic if passed /dev/ethX + if '/' in iface: + iface = iface.split('/')[-1] + if not exc_list: + exc_list = [] try: - iface_addrs = netifaces.ifaddresses(iface) - if netifaces.AF_INET6 not in iface_addrs: - raise Exception("Interface '%s' doesn't have an ipv6 address." % iface) + inet_num = getattr(netifaces, inet_type) + except AttributeError: + raise Exception('Unknown inet type ' + str(inet_type)) - addresses = netifaces.ifaddresses(iface)[netifaces.AF_INET6] - ipv6_addr = [a['addr'] for a in addresses if not a['addr'].startswith('fe80') - and config('vip') != a['addr']] - if not ipv6_addr: - raise Exception("Interface '%s' doesn't have global ipv6 address." % iface) + interfaces = netifaces.interfaces() + if inc_aliases: + ifaces = [] + for _iface in interfaces: + if iface == _iface or _iface.split(':')[0] == iface: + ifaces.append(_iface) + if fatal and not ifaces: + raise Exception("Invalid interface '%s'" % iface) + ifaces.sort() + else: + if iface not in interfaces: + if fatal: + raise Exception("%s not found " % (iface)) + else: + return [] + else: + ifaces = [iface] - return ipv6_addr[0] + addresses = [] + for netiface in ifaces: + net_info = netifaces.ifaddresses(netiface) + if inet_num in net_info: + for entry in net_info[inet_num]: + if 'addr' in entry and entry['addr'] not in exc_list: + addresses.append(entry['addr']) + if fatal and not addresses: + raise Exception("Interface '%s' doesn't have any %s addresses." % (iface, inet_type)) + return addresses - except ValueError: - raise ValueError("Invalid interface '%s'" % iface) +get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET') + + +def get_ipv6_addr(iface='eth0', inc_aliases=False, fatal=True, exc_list=None): + """ + Return the assigned IPv6 address for a given interface, if any, or []. + """ + addresses = get_iface_addr(iface=iface, inet_type='AF_INET6', + inc_aliases=inc_aliases, fatal=fatal, + exc_list=exc_list) + remotly_addressable = [] + for address in addresses: + if not address.startswith('fe80'): + remotly_addressable.append(address) + if fatal and not remotly_addressable: + raise Exception("Interface '%s' doesn't have global ipv6 address." % iface) + return remotly_addressable + + +def get_bridges(vnic_dir='/sys/devices/virtual/net'): + """ + Return a list of bridges on the system or [] + """ + b_rgex = vnic_dir + '/*/bridge' + return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_rgex)] + + +def get_bridge_nics(bridge, vnic_dir='/sys/devices/virtual/net'): + """ + Return a list of nics comprising a given bridge on the system or [] + """ + brif_rgex = "%s/%s/brif/*" % (vnic_dir, bridge) + return [x.split('/')[-1] for x in glob.glob(brif_rgex)] + + +def is_bridge_member(nic): + """ + Check if a given nic is a member of a bridge + """ + for bridge in get_bridges(): + if nic in get_bridge_nics(bridge): + return True + return False diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index c953043..324987e 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -156,12 +156,15 @@ def hook_name(): class Config(dict): - """A Juju charm config dictionary that can write itself to - disk (as json) and track which values have changed since - the previous hook invocation. + """A dictionary representation of the charm's config.yaml, with some + extra features: - Do not instantiate this object directly - instead call - ``hookenv.config()`` + - See which values in the dictionary have changed since the previous hook. + - For values that have changed, see what the previous value was. + - Store arbitrary data for use in a later hook. + + NOTE: Do not instantiate this object directly - instead call + ``hookenv.config()``, which will return an instance of :class:`Config`. Example usage:: @@ -170,8 +173,8 @@ class Config(dict): >>> config = hookenv.config() >>> config['foo'] 'bar' + >>> # store a new key/value for later use >>> config['mykey'] = 'myval' - >>> config.save() >>> # user runs `juju set mycharm foo=baz` @@ -188,22 +191,34 @@ class Config(dict): >>> # keys/values that we add are preserved across hooks >>> config['mykey'] 'myval' - >>> # don't forget to save at the end of hook! - >>> config.save() """ CONFIG_FILE_NAME = '.juju-persistent-config' def __init__(self, *args, **kw): super(Config, self).__init__(*args, **kw) + self.implicit_save = True self._prev_dict = None self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME) if os.path.exists(self.path): self.load_previous() + def __getitem__(self, key): + """For regular dict lookups, check the current juju config first, + then the previous (saved) copy. This ensures that user-saved values + will be returned by a dict lookup. + + """ + try: + return dict.__getitem__(self, key) + except KeyError: + return (self._prev_dict or {})[key] + def load_previous(self, path=None): - """Load previous copy of config from disk so that current values - can be compared to previous values. + """Load previous copy of config from disk. + + In normal usage you don't need to call this method directly - it + is called automatically at object initialization. :param path: @@ -218,8 +233,8 @@ class Config(dict): self._prev_dict = json.load(f) def changed(self, key): - """Return true if the value for this key has changed since - the last save. + """Return True if the current value for this key is different from + the previous value. """ if self._prev_dict is None: @@ -228,7 +243,7 @@ class Config(dict): def previous(self, key): """Return previous value for this key, or None if there - is no "previous" value. + is no previous value. """ if self._prev_dict: @@ -238,7 +253,13 @@ class Config(dict): def save(self): """Save this config to disk. - Preserves items in _prev_dict that do not exist in self. + If the charm is using the :mod:`Services Framework ` + or :meth:'@hook ' decorator, this + is called automatically at the end of successful hook execution. + Otherwise, it should be called directly by user code. + + To disable automatic saves, set ``implicit_save=False`` on this + instance. """ if self._prev_dict: @@ -285,8 +306,9 @@ def relation_get(attribute=None, unit=None, rid=None): raise -def relation_set(relation_id=None, relation_settings={}, **kwargs): +def relation_set(relation_id=None, relation_settings=None, **kwargs): """Set relation information for the current unit""" + relation_settings = relation_settings if relation_settings else {} relation_cmd_line = ['relation-set'] if relation_id is not None: relation_cmd_line.extend(('-r', relation_id)) @@ -477,6 +499,9 @@ class Hooks(object): hook_name = os.path.basename(args[0]) if hook_name in self._hooks: self._hooks[hook_name]() + cfg = config() + if cfg.implicit_save: + cfg.save() else: raise UnregisteredHookError(hook_name) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index ca7780d..b85b028 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -332,13 +332,9 @@ def cmp_pkgrevno(package, revno, pkgcache=None): ''' import apt_pkg + from charmhelpers.fetch import apt_cache if not pkgcache: - apt_pkg.init() - # Force Apt to build its cache in memory. That way we avoid race - # conditions with other applications building the cache in the same - # place. - apt_pkg.config.set("Dir::Cache::pkgcache", "") - pkgcache = apt_pkg.Cache() + pkgcache = apt_cache() pkg = pkgcache[package] return apt_pkg.version_compare(pkg.current_ver.ver_str, revno) diff --git a/hooks/charmhelpers/core/services/base.py b/hooks/charmhelpers/core/services/base.py index f08e6d7..87ecb13 100644 --- a/hooks/charmhelpers/core/services/base.py +++ b/hooks/charmhelpers/core/services/base.py @@ -17,20 +17,13 @@ class ServiceManager(object): """ Register a list of services, given their definitions. - Traditional charm authoring is focused on implementing hooks. That is, - the charm author is thinking in terms of "What hook am I handling; what - does this hook need to do?" However, in most cases, the real question - should be "Do I have the information I need to configure and start this - piece of software and, if so, what are the steps for doing so?" The - ServiceManager framework tries to bring the focus to the data and the - setup tasks, in the most declarative way possible. - Service definitions are dicts in the following formats (all keys except 'service' are optional):: { "service": , "required_data": , + "provided_data": , "data_ready": , "data_lost": , "start": , @@ -44,6 +37,10 @@ class ServiceManager(object): of 'data_ready' and 'start' callbacks executed. See `is_ready()` for more information. + The 'provided_data' list should contain relation data providers, most likely + a subclass of :class:`charmhelpers.core.services.helpers.RelationContext`, + that will indicate a set of data to set on a given relation. + The 'data_ready' value should be either a single callback, or a list of callbacks, to be called when all items in 'required_data' pass `is_ready()`. Each callback will be called with the service name as the only parameter. @@ -121,14 +118,25 @@ class ServiceManager(object): else: self.provide_data() self.reconfigure_services() + cfg = hookenv.config() + if cfg.implicit_save: + cfg.save() def provide_data(self): + """ + Set the relation data for each provider in the ``provided_data`` list. + + A provider must have a `name` attribute, which indicates which relation + to set data on, and a `provide_data()` method, which returns a dict of + data to set. + """ hook_name = hookenv.hook_name() for service in self.services.values(): for provider in service.get('provided_data', []): if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name): data = provider.provide_data() - if provider._is_ready(data): + _ready = provider._is_ready(data) if hasattr(provider, '_is_ready') else data + if _ready: hookenv.relation_set(None, data) def reconfigure_services(self, *service_names): diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 5be512c..8e9d380 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -1,4 +1,5 @@ import importlib +from tempfile import NamedTemporaryFile import time from yaml import safe_load from charmhelpers.core.host import ( @@ -116,14 +117,7 @@ class BaseFetchHandler(object): def filter_installed_packages(packages): """Returns a list of packages that require installation""" - import apt_pkg - apt_pkg.init() - - # Tell apt to build an in-memory cache to prevent race conditions (if - # another process is already building the cache). - apt_pkg.config.set("Dir::Cache::pkgcache", "") - - cache = apt_pkg.Cache() + cache = apt_cache() _pkgs = [] for package in packages: try: @@ -136,6 +130,16 @@ def filter_installed_packages(packages): return _pkgs +def apt_cache(in_memory=True): + """Build and return an apt cache""" + import apt_pkg + apt_pkg.init() + if in_memory: + apt_pkg.config.set("Dir::Cache::pkgcache", "") + apt_pkg.config.set("Dir::Cache::srcpkgcache", "") + return apt_pkg.Cache() + + def apt_install(packages, options=None, fatal=False): """Install one or more packages""" if options is None: @@ -201,6 +205,27 @@ def apt_hold(packages, fatal=False): def add_source(source, key=None): + """Add a package source to this system. + + @param source: a URL or sources.list entry, as supported by + add-apt-repository(1). Examples: + ppa:charmers/example + deb https://stub:key@private.example.com/ubuntu trusty main + + In addition: + 'proposed:' may be used to enable the standard 'proposed' + pocket for the release. + 'cloud:' may be used to activate official cloud archive pockets, + such as 'cloud:icehouse' + + @param key: A key to be added to the system's APT keyring and used + to verify the signatures on packages. Ideally, this should be an + ASCII format GPG public key including the block headers. A GPG key + id may also be used, but be aware that only insecure protocols are + available to retrieve the actual public key from a public keyserver + placing your Juju environment at risk. ppa and cloud archive keys + are securely added automtically, so sould not be provided. + """ if source is None: log('Source is not present. Skipping') return @@ -225,10 +250,23 @@ def add_source(source, key=None): release = lsb_release()['DISTRIB_CODENAME'] with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt: apt.write(PROPOSED_POCKET.format(release)) + else: + raise SourceConfigError("Unknown source: {!r}".format(source)) + if key: - subprocess.check_call(['apt-key', 'adv', '--keyserver', - 'hkp://keyserver.ubuntu.com:80', '--recv', - key]) + if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key: + with NamedTemporaryFile() as key_file: + key_file.write(key) + key_file.flush() + key_file.seek(0) + subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file) + else: + # Note that hkp: is in no way a secure protocol. Using a + # GPG key id is pointless from a security POV unless you + # absolutely trust your network and DNS. + subprocess.check_call(['apt-key', 'adv', '--keyserver', + 'hkp://keyserver.ubuntu.com:80', '--recv', + key]) def configure_sources(update=False, @@ -238,7 +276,8 @@ def configure_sources(update=False, Configure multiple sources from charm configuration. The lists are encoded as yaml fragments in the configuration. - The frament needs to be included as a string. + The frament needs to be included as a string. Sources and their + corresponding keys are of the types supported by add_source(). Example config: install_sources: | diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index 87e7071..1b11fa0 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -1,6 +1,8 @@ import os import urllib2 +from urllib import urlretrieve import urlparse +import hashlib from charmhelpers.fetch import ( BaseFetchHandler, @@ -12,7 +14,17 @@ from charmhelpers.payload.archive import ( ) from charmhelpers.core.host import mkdir +""" +This class is a plugin for charmhelpers.fetch.install_remote. +It grabs, validates and installs remote archives fetched over "http", "https", "ftp" or "file" protocols. The contents of the archive are installed in $CHARM_DIR/fetched/. + +Example usage: +install_remote("https://example.com/some/archive.tar.gz") +# Installs the contents of archive.tar.gz in $CHARM_DIR/fetched/. + +See charmhelpers.fetch.archiveurl.get_archivehandler for supported archive types. +""" class ArchiveUrlFetchHandler(BaseFetchHandler): """Handler for archives via generic URLs""" def can_handle(self, source): @@ -61,3 +73,31 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): except OSError as e: raise UnhandledSource(e.strerror) return extract(dld_file) + + # Mandatory file validation via Sha1 or MD5 hashing. + def download_and_validate(self, url, hashsum, validate="sha1"): + if validate == 'sha1' and len(hashsum) != 40: + raise ValueError("HashSum must be = 40 characters when using sha1" + " validation") + if validate == 'md5' and len(hashsum) != 32: + raise ValueError("HashSum must be = 32 characters when using md5" + " validation") + tempfile, headers = urlretrieve(url) + self.validate_file(tempfile, hashsum, validate) + return tempfile + + # Predicate method that returns status of hash matching expected hash. + def validate_file(self, source, hashsum, vmethod='sha1'): + if vmethod != 'sha1' and vmethod != 'md5': + raise ValueError("Validation Method not supported") + + if vmethod == 'md5': + m = hashlib.md5() + if vmethod == 'sha1': + m = hashlib.sha1() + with open(source) as f: + for line in f: + m.update(line) + if hashsum != m.hexdigest(): + msg = "Hash Mismatch on {} expected {} got {}" + raise ValueError(msg.format(source, hashsum, m.hexdigest())) diff --git a/hooks/hooks.py b/hooks/hooks.py index 865cf4b..209fa65 100755 --- a/hooks/hooks.py +++ b/hooks/hooks.py @@ -76,7 +76,7 @@ def install(): def emit_cephconf(): if config('prefer-ipv6'): - host_ip = '%s' % get_ipv6_addr() + host_ip = '%s' % get_ipv6_addr()[0] cephcontext = { 'auth_supported': config('auth-supported'), @@ -197,7 +197,7 @@ def mon_relation(): emit_cephconf() if config('prefer-ipv6'): - host = get_ipv6_addr() + host = get_ipv6_addr()[0] else: host = unit_get('private-address') @@ -283,7 +283,7 @@ def radosgw_relation(relid=None): log('mon cluster not in quorum - deferring key provision') if config('prefer-ipv6'): - host = get_ipv6_addr() + host = get_ipv6_addr()[0] else: host = unit_get('private-address') @@ -297,7 +297,7 @@ def radosgw_relation(relid=None): @hooks.hook('client-relation-joined') def client_relation(relid=None): if config('prefer-ipv6'): - host = get_ipv6_addr() + host = get_ipv6_addr()[0] else: host = unit_get('private-address') diff --git a/hooks/utils.py b/hooks/utils.py index a937380..f5e7e85 100644 --- a/hooks/utils.py +++ b/hooks/utils.py @@ -92,7 +92,7 @@ def get_public_addr(): if addr and is_ipv6(addr): return addr else: - return get_ipv6_addr() + return get_ipv6_addr()[0] else: return ip.get_address_in_network(addr, fallback=get_host_ip())