Sync with alternatives charmhelpers, installed ceph.conf as alternative for charm co-existence

charm-helpers-sync.yaml

@@ -1,7 +1,8 @@
-branch: lp:charm-helpers
+branch: lp:~openstack-charmers/charm-helpers/os-alternatives
 destination: hooks/charmhelpers
 include:
     - core
     - fetch
     - contrib.storage.linux:
         - utils
+    - contrib.openstack.alternatives

hooks/charmhelpers/contrib/openstack/alternatives.py

@@ -0,0 +1,17 @@
+''' Helper for managing alternatives for file conflict resolution '''
+
+import subprocess
+import shutil
+import os
+
+
+def install_alternative(name, target, source, priority=50):
+    ''' Install alternative configuration '''
+    if (os.path.exists(target) and not os.path.islink(target)):
+        # Move existing file/directory away before installing
+        shutil.move(target, '{}.bak'.format(target))
+    cmd = [
+        'update-alternatives', '--force', '--install',
+        target, name, source, str(priority)
+    ]
+    subprocess.check_call(cmd)

hooks/charmhelpers/core/hookenv.py

@@ -143,6 +143,11 @@ def remote_unit():
     return os.environ['JUJU_REMOTE_UNIT']
 
 
+def service_name():
+    "The name service group this unit belongs to"
+    return local_unit().split('/')[0]
+
+
 @cached
 def config(scope=None):
     "Juju charm configuration"
@@ -192,7 +197,7 @@ def relation_ids(reltype=None):
     relid_cmd_line = ['relation-ids', '--format=json']
     if reltype is not None:
         relid_cmd_line.append(reltype)
-        return json.loads(subprocess.check_output(relid_cmd_line))
+        return json.loads(subprocess.check_output(relid_cmd_line)) or []
     return []
 
 
@@ -203,7 +208,7 @@ def related_units(relid=None):
     units_cmd_line = ['relation-list', '--format=json']
     if relid is not None:
         units_cmd_line.extend(('-r', relid))
-    return json.loads(subprocess.check_output(units_cmd_line))
+    return json.loads(subprocess.check_output(units_cmd_line)) or []
 
 
 @cached
@@ -330,5 +335,6 @@ class Hooks(object):
             return decorated
         return wrapper
 
+
 def charm_dir():
     return os.environ.get('CHARM_DIR')

hooks/charmhelpers/core/host.py

@@ -5,33 +5,36 @@
 #  Nick Moffitt <nick.moffitt@canonical.com>
 #  Matthew Wedgwood <matthew.wedgwood@canonical.com>
 
-import apt_pkg
 import os
 import pwd
 import grp
+import random
+import string
 import subprocess
 import hashlib
 
 from collections import OrderedDict
 
-from hookenv import log, execution_environment
+from hookenv import log
 
 
 def service_start(service_name):
-    service('start', service_name)
+    return service('start', service_name)
 
 
 def service_stop(service_name):
-    service('stop', service_name)
+    return service('stop', service_name)
 
 
 def service_restart(service_name):
-    service('restart', service_name)
+    return service('restart', service_name)
 
 
 def service_reload(service_name, restart_on_failure=False):
-    if not service('reload', service_name) and restart_on_failure:
+    service_result = service('reload', service_name)
-        service('restart', service_name)
+    if not service_result and restart_on_failure:
+        service_result = service('restart', service_name)
+    return service_result
 
 
 def service(action, service_name):
@@ -39,6 +42,18 @@ def service(action, service_name):
     return subprocess.call(cmd) == 0
 
 
+def service_running(service):
+    try:
+        output = subprocess.check_output(['service', service, 'status'])
+    except subprocess.CalledProcessError:
+        return False
+    else:
+        if ("start/running" in output or "is running" in output):
+            return True
+        else:
+            return False
+
+
 def adduser(username, password=None, shell='/bin/bash', system_user=False):
     """Add a user"""
     try:
@@ -74,36 +89,33 @@ def add_user_to_group(username, group):
 
 def rsync(from_path, to_path, flags='-r', options=None):
     """Replicate the contents of a path"""
-    context = execution_environment()
     options = options or ['--delete', '--executability']
     cmd = ['/usr/bin/rsync', flags]
     cmd.extend(options)
-    cmd.append(from_path.format(**context))
+    cmd.append(from_path)
-    cmd.append(to_path.format(**context))
+    cmd.append(to_path)
     log(" ".join(cmd))
     return subprocess.check_output(cmd).strip()
 
 
 def symlink(source, destination):
     """Create a symbolic link"""
-    context = execution_environment()
     log("Symlinking {} as {}".format(source, destination))
     cmd = [
         'ln',
         '-sf',
-        source.format(**context),
+        source,
-        destination.format(**context)
+        destination,
     ]
     subprocess.check_call(cmd)
 
 
 def mkdir(path, owner='root', group='root', perms=0555, force=False):
     """Create a directory"""
-    context = execution_environment()
     log("Making dir {} {}:{} {:o}".format(path, owner, group,
                                           perms))
-    uid = pwd.getpwnam(owner.format(**context)).pw_uid
+    uid = pwd.getpwnam(owner).pw_uid
-    gid = grp.getgrnam(group.format(**context)).gr_gid
+    gid = grp.getgrnam(group).gr_gid
     realpath = os.path.abspath(path)
     if os.path.exists(realpath):
         if force and not os.path.isdir(realpath):
@@ -114,71 +126,15 @@ def mkdir(path, owner='root', group='root', perms=0555, force=False):
     os.chown(realpath, uid, gid)
 
 
-def write_file(path, fmtstr, owner='root', group='root', perms=0444, **kwargs):
+def write_file(path, content, owner='root', group='root', perms=0444):
     """Create or overwrite a file with the contents of a string"""
-    context = execution_environment()
+    log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
-    context.update(kwargs)
+    uid = pwd.getpwnam(owner).pw_uid
-    log("Writing file {} {}:{} {:o}".format(path, owner, group,
+    gid = grp.getgrnam(group).gr_gid
-        perms))
+    with open(path, 'w') as target:
-    uid = pwd.getpwnam(owner.format(**context)).pw_uid
-    gid = grp.getgrnam(group.format(**context)).gr_gid
-    with open(path.format(**context), 'w') as target:
         os.fchown(target.fileno(), uid, gid)
         os.fchmod(target.fileno(), perms)
-        target.write(fmtstr.format(**context))
+        target.write(content)
-
-
-def render_template_file(source, destination, **kwargs):
-    """Create or overwrite a file using a template"""
-    log("Rendering template {} for {}".format(source,
-        destination))
-    context = execution_environment()
-    with open(source.format(**context), 'r') as template:
-        write_file(destination.format(**context), template.read(),
-                   **kwargs)
-
-
-def filter_installed_packages(packages):
-    """Returns a list of packages that require installation"""
-    apt_pkg.init()
-    cache = apt_pkg.Cache()
-    _pkgs = []
-    for package in packages:
-        try:
-            p = cache[package]
-            p.current_ver or _pkgs.append(package)
-        except KeyError:
-            log('Package {} has no installation candidate.'.format(package),
-                level='WARNING')
-            _pkgs.append(package)
-    return _pkgs
-
-
-def apt_install(packages, options=None, fatal=False):
-    """Install one or more packages"""
-    options = options or []
-    cmd = ['apt-get', '-y']
-    cmd.extend(options)
-    cmd.append('install')
-    if isinstance(packages, basestring):
-        cmd.append(packages)
-    else:
-        cmd.extend(packages)
-    log("Installing {} with options: {}".format(packages,
-                                                options))
-    if fatal:
-        subprocess.check_call(cmd)
-    else:
-        subprocess.call(cmd)
-
-
-def apt_update(fatal=False):
-    """Update local apt cache"""
-    cmd = ['apt-get', 'update']
-    if fatal:
-        subprocess.check_call(cmd)
-    else:
-        subprocess.call(cmd)
 
 
 def mount(device, mountpoint, options=None, persist=False):
@@ -271,3 +227,15 @@ def lsb_release():
             k, v = l.split('=')
             d[k.strip()] = v.strip()
     return d
+
+
+def pwgen(length=None):
+    '''Generate a random pasword.'''
+    if length is None:
+        length = random.choice(range(35, 45))
+    alphanumeric_chars = [
+        l for l in (string.letters + string.digits)
+        if l not in 'l0QD1vAEIOUaeiou']
+    random_chars = [
+        random.choice(alphanumeric_chars) for _ in range(length)]
+    return(''.join(random_chars))

hooks/charmhelpers/fetch/__init__.py

@@ -1,9 +1,6 @@
 import importlib
 from yaml import safe_load
 from charmhelpers.core.host import (
-    apt_install,
-    apt_update,
-    filter_installed_packages,
     lsb_release
 )
 from urlparse import (
@@ -15,6 +12,7 @@ from charmhelpers.core.hookenv import (
     config,
     log,
 )
+import apt_pkg
 
 CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
 deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
@@ -24,10 +22,67 @@ deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restri
 """
 
 
+def filter_installed_packages(packages):
+    """Returns a list of packages that require installation"""
+    apt_pkg.init()
+    cache = apt_pkg.Cache()
+    _pkgs = []
+    for package in packages:
+        try:
+            p = cache[package]
+            p.current_ver or _pkgs.append(package)
+        except KeyError:
+            log('Package {} has no installation candidate.'.format(package),
+                level='WARNING')
+            _pkgs.append(package)
+    return _pkgs
+
+
+def apt_install(packages, options=None, fatal=False):
+    """Install one or more packages"""
+    options = options or []
+    cmd = ['apt-get', '-y']
+    cmd.extend(options)
+    cmd.append('install')
+    if isinstance(packages, basestring):
+        cmd.append(packages)
+    else:
+        cmd.extend(packages)
+    log("Installing {} with options: {}".format(packages,
+                                                options))
+    if fatal:
+        subprocess.check_call(cmd)
+    else:
+        subprocess.call(cmd)
+
+
+def apt_update(fatal=False):
+    """Update local apt cache"""
+    cmd = ['apt-get', 'update']
+    if fatal:
+        subprocess.check_call(cmd)
+    else:
+        subprocess.call(cmd)
+
+
+def apt_purge(packages, fatal=False):
+    """Purge one or more packages"""
+    cmd = ['apt-get', '-y', 'purge']
+    if isinstance(packages, basestring):
+        cmd.append(packages)
+    else:
+        cmd.extend(packages)
+    log("Purging {}".format(packages))
+    if fatal:
+        subprocess.check_call(cmd)
+    else:
+        subprocess.call(cmd)
+
+
 def add_source(source, key=None):
     if ((source.startswith('ppa:') or
          source.startswith('http:'))):
-        subprocess.check_call(['add-apt-repository', source])
+        subprocess.check_call(['add-apt-repository', '--yes', source])
     elif source.startswith('cloud:'):
         apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
                     fatal=True)
@@ -79,6 +134,7 @@ def configure_sources(update=False,
 # least- to most-specific URL matching.
 FETCH_HANDLERS = (
     'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
+    'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
 )
 
 
@@ -98,6 +154,7 @@ def install_remote(source):
     # We ONLY check for True here because can_handle may return a string
     # explaining why it can't handle a given source.
     handlers = [h for h in plugins() if h.can_handle(source) is True]
+    installed_to = None
     for handler in handlers:
         try:
             installed_to = handler.install(source)

hooks/charmhelpers/fetch/archiveurl.py

@@ -8,6 +8,7 @@ from charmhelpers.payload.archive import (
     get_archive_handler,
     extract,
 )
+from charmhelpers.core.host import mkdir
 
 
 class ArchiveUrlFetchHandler(BaseFetchHandler):
@@ -24,20 +25,24 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
         # propogate all exceptions
         # URLError, OSError, etc
         response = urllib2.urlopen(source)
-        with open(dest, 'w') as dest_file:
+        try:
-            dest_file.write(response.read())
+            with open(dest, 'w') as dest_file:
+                dest_file.write(response.read())
+        except Exception as e:
+            if os.path.isfile(dest):
+                os.unlink(dest)
+            raise e
 
     def install(self, source):
         url_parts = self.parse_url(source)
         dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
+        if not os.path.exists(dest_dir):
+            mkdir(dest_dir, perms=0755)
         dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path))
         try:
             self.download(source, dld_file)
         except urllib2.URLError as e:
-            return UnhandledSource(e.reason)
+            raise UnhandledSource(e.reason)
         except OSError as e:
-            return UnhandledSource(e.strerror)
+            raise UnhandledSource(e.strerror)
-        finally:
-            if os.path.isfile(dld_file):
-                os.unlink(dld_file)
         return extract(dld_file)

hooks/charmhelpers/fetch/bzrurl.py

@@ -0,0 +1,49 @@
+import os
+from charmhelpers.fetch import (
+    BaseFetchHandler,
+    UnhandledSource
+)
+from charmhelpers.core.host import mkdir
+
+try:
+    from bzrlib.branch import Branch
+except ImportError:
+    from charmhelpers.fetch import apt_install
+    apt_install("python-bzrlib")
+    from bzrlib.branch import Branch
+
+class BzrUrlFetchHandler(BaseFetchHandler):
+    """Handler for bazaar branches via generic and lp URLs"""
+    def can_handle(self, source):
+        url_parts = self.parse_url(source)
+        if url_parts.scheme not in ('bzr+ssh', 'lp'):
+            return False
+        else:
+            return True
+
+    def branch(self, source, dest):
+        url_parts = self.parse_url(source)
+        # If we use lp:branchname scheme we need to load plugins
+        if not self.can_handle(source):
+            raise UnhandledSource("Cannot handle {}".format(source))
+        if url_parts.scheme == "lp":
+            from bzrlib.plugin import load_plugins
+            load_plugins()
+        try:
+            remote_branch = Branch.open(source)
+            remote_branch.bzrdir.sprout(dest).open_branch()
+        except Exception as e:
+            raise e
+
+    def install(self, source):
+        url_parts = self.parse_url(source)
+        branch_name = url_parts.path.strip("/").split("/")[-1]
+        dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", branch_name)
+        if not os.path.exists(dest_dir):
+            mkdir(dest_dir, perms=0755)
+        try:
+            self.branch(source, dest_dir)
+        except OSError as e:
+            raise UnhandledSource(e.strerror)
+        return dest_dir
+

hooks/hooks.py

@@ -22,16 +22,22 @@ from charmhelpers.core.hookenv import (
     relation_get,
     relation_set,
     remote_unit,
-    Hooks, UnregisteredHookError
+    Hooks, UnregisteredHookError,
+    service_name
 )
+
 from charmhelpers.core.host import (
+    service_restart,
+    umount,
+    mkdir
+)
+from charmhelpers.fetch import (
     apt_install,
     apt_update,
     filter_installed_packages,
-    service_restart,
+    add_source
-    umount
 )
-from charmhelpers.fetch import add_source
+from charmhelpers.contrib.openstack.alternatives import install_alternative
 
 from utils import (
     render_template,
@@ -65,9 +71,14 @@ def emit_cephconf():
         'fsid': config('fsid'),
         'version': ceph.get_ceph_version()
     }
-
+    # Install ceph.conf as an alternative to support
-    with open('/etc/ceph/ceph.conf', 'w') as cephconf:
+    # co-existence with other charms that write this file
+    charm_ceph_conf = "/var/lib/charm/{}/ceph.conf".format(service_name())
+    mkdir(os.path.dirname(charm_ceph_conf))
+    with open(charm_ceph_conf, 'w') as cephconf:
         cephconf.write(render_template('ceph.conf', cephcontext))
+    install_alternative('ceph.conf', '/etc/ceph/ceph.conf',
+                        charm_ceph_conf, 100)
 
 JOURNAL_ZAPPED = '/var/lib/ceph/journal_zapped'
 

hooks/utils.py

@@ -13,7 +13,7 @@ from charmhelpers.core.hookenv import (
     unit_get,
     cached
 )
-from charmhelpers.core.host import (
+from charmhelpers.fetch import (
     apt_install,
     filter_installed_packages
 )

revision

@@ -1 +1 @@
-100
+103