Use charm-internal _upgrade_keyring

This change includes a charms.ceph sync.

The _upgrade_keyring function was removed from charms.ceph
so this charm needs to use the already-existing, charm-
internal version.

Change-Id: Ia7cf352a2456dc85aca6f61d2e88327cd0c15f7e
Closes-Bug: #1845975

hooks/ceph_hooks.py

@@ -85,6 +85,7 @@ from utils import (
     get_blacklist,
     get_journal_devices,
     should_enable_discard,
+    _upgrade_keyring,
 )
 from charmhelpers.contrib.openstack.alternatives import install_alternative
 from charmhelpers.contrib.network.ip import (
@@ -124,7 +125,7 @@ CRON_CEPH_CHECK_FILE = '/etc/cron.d/check-osd-services'
 
 def check_for_upgrade():
 
-    if not os.path.exists(ceph._upgrade_keyring):
+    if not os.path.exists(_upgrade_keyring):
         log("Ceph upgrade keyring not detected, skipping upgrade checks.")
         return
 

lib/ceph/crush_utils.py

@@ -24,7 +24,7 @@ from charmhelpers.core.hookenv import (
 CRUSH_BUCKET = """root {name} {{
     id {id}    # do not change unnecessarily
     # weight 0.000
-    alg straw
+    alg straw2
     hash 0  # rjenkins1
 }}
 

lib/ceph/utils.py

@@ -40,6 +40,7 @@ from charmhelpers.core.host import (
     service_start,
     service_stop,
     CompareHostReleases,
+    write_file,
 )
 from charmhelpers.core.hookenv import (
     cached,
@@ -928,11 +929,13 @@ def is_osd_disk(dev):
 def start_osds(devices):
     # Scan for ceph block devices
     rescan_osd_devices()
-    if cmp_pkgrevno('ceph', "0.56.6") >= 0:
+    if (cmp_pkgrevno('ceph', '0.56.6') >= 0 and
-        # Use ceph-disk activate for directory based OSD's
+            cmp_pkgrevno('ceph', '14.2.0') < 0):
-        for dev_or_path in devices:
+            # Use ceph-disk activate for directory based OSD's
-            if os.path.exists(dev_or_path) and os.path.isdir(dev_or_path):
+            for dev_or_path in devices:
-                subprocess.check_call(['ceph-disk', 'activate', dev_or_path])
+                if os.path.exists(dev_or_path) and os.path.isdir(dev_or_path):
+                    subprocess.check_call(
+                        ['ceph-disk', 'activate', dev_or_path])
 
 
 def udevadm_settle():
@@ -950,13 +953,12 @@ def rescan_osd_devices():
 
     udevadm_settle()
 
-
+_client_admin_keyring = '/etc/ceph/ceph.client.admin.keyring'
-_bootstrap_keyring = "/var/lib/ceph/bootstrap-osd/ceph.keyring"
-_upgrade_keyring = "/var/lib/ceph/osd/ceph.client.osd-upgrade.keyring"
 
 
 def is_bootstrapped():
-    return os.path.exists(_bootstrap_keyring)
+    return os.path.exists(
+        '/var/lib/ceph/mon/ceph-{}/done'.format(socket.gethostname()))
 
 
 def wait_for_bootstrap():
@@ -964,36 +966,6 @@ def wait_for_bootstrap():
         time.sleep(3)
 
 
-def import_osd_bootstrap_key(key):
-    if not os.path.exists(_bootstrap_keyring):
-        cmd = [
-            "sudo",
-            "-u",
-            ceph_user(),
-            'ceph-authtool',
-            _bootstrap_keyring,
-            '--create-keyring',
-            '--name=client.bootstrap-osd',
-            '--add-key={}'.format(key)
-        ]
-        subprocess.check_call(cmd)
-
-
-def import_osd_upgrade_key(key):
-    if not os.path.exists(_upgrade_keyring):
-        cmd = [
-            "sudo",
-            "-u",
-            ceph_user(),
-            'ceph-authtool',
-            _upgrade_keyring,
-            '--create-keyring',
-            '--name=client.osd-upgrade',
-            '--add-key={}'.format(key)
-        ]
-        subprocess.check_call(cmd)
-
-
 def generate_monitor_secret():
     cmd = [
         'ceph-authtool',
@@ -1259,7 +1231,23 @@ def systemd():
     return CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) >= 'vivid'
 
 
+def use_bluestore():
+    """Determine whether bluestore should be used for OSD's
+
+    :returns: whether bluestore disk format should be used
+    :rtype: bool"""
+    if cmp_pkgrevno('ceph', '12.2.0') < 0:
+        return False
+    return config('bluestore')
+
+
 def bootstrap_monitor_cluster(secret):
+    """Bootstrap local ceph mon into the ceph cluster
+
+    :param secret: cephx secret to use for monitor authentication
+    :type secret: str
+    :raises: Exception if ceph mon cannot be bootstrapped
+    """
     hostname = socket.gethostname()
     path = '/var/lib/ceph/mon/ceph-{}'.format(hostname)
     done = '{}/done'.format(path)
@@ -1280,21 +1268,35 @@ def bootstrap_monitor_cluster(secret):
               perms=0o755)
         # end changes for Ceph >= 0.61.3
         try:
-            add_keyring_to_ceph(keyring,
+            _create_monitor(keyring,
-                                secret,
+                            secret,
-                                hostname,
+                            hostname,
-                                path,
+                            path,
-                                done,
+                            done,
-                                init_marker)
+                            init_marker)
-
+            _create_keyrings()
         except:
             raise
         finally:
             os.unlink(keyring)
 
 
-@retry_on_exception(3, base_delay=5)
+def _create_monitor(keyring, secret, hostname, path, done, init_marker):
-def add_keyring_to_ceph(keyring, secret, hostname, path, done, init_marker):
+    """Create monitor filesystem and enable and start ceph-mon process
+
+    :param keyring: path to temporary keyring on disk
+    :type keyring: str
+    :param secret: cephx secret to use for monitor authentication
+    :type: secret: str
+    :param hostname: hostname of the local unit
+    :type hostname: str
+    :param path: full path to ceph mon directory
+    :type path: str
+    :param done: full path to 'done' marker for ceph mon
+    :type done: str
+    :param init_marker: full path to 'init' marker for ceph mon
+    :type init_marker: str
+    """
     subprocess.check_call(['ceph-authtool', keyring,
                            '--create-keyring', '--name=mon.',
                            '--add-key={}'.format(secret),
@@ -1310,39 +1312,72 @@ def add_keyring_to_ceph(keyring, secret, hostname, path, done, init_marker):
         pass
 
     if systemd():
-        subprocess.check_call(['systemctl', 'enable', 'ceph-mon'])
+        if cmp_pkgrevno('ceph', '14.0.0') >= 0:
-        service_restart('ceph-mon')
+            systemd_unit = 'ceph-mon@{}'.format(socket.gethostname())
+        else:
+            systemd_unit = 'ceph-mon'
+        subprocess.check_call(['systemctl', 'enable', systemd_unit])
+        service_restart(systemd_unit)
     else:
         service_restart('ceph-mon-all')
 
-    # NOTE(jamespage): Later ceph releases require explicit
+
-    #                  call to ceph-create-keys to setup the
+@retry_on_exception(3, base_delay=5)
-    #                  admin keys for the cluster; this command
+def _create_keyrings():
-    #                  will wait for quorum in the cluster before
+    """Create keyrings for operation of ceph-mon units
-    #                  returning.
+
-    # NOTE(fnordahl): Explicitly run `ceph-crate-keys` for older
+    :raises: Exception if keyrings cannot be created
-    #                 ceph releases too.  This improves bootstrap
+    """
-    #                 resilience as the charm will wait for
+    if cmp_pkgrevno('ceph', '14.0.0') >= 0:
-    #                 presence of peer units before attempting
+        # NOTE(jamespage): At Nautilus, keys are created by the
-    #                 to bootstrap.  Note that charms deploying
+        #                  monitors automatically and just need
-    #                 ceph-mon service should disable running of
+        #                  exporting.
-    #                 `ceph-create-keys` service in init system.
+        output = str(subprocess.check_output(
-    cmd = ['ceph-create-keys', '--id', hostname]
+            [
-    if cmp_pkgrevno('ceph', '12.0.0') >= 0:
+                'sudo',
-        # NOTE(fnordahl): The default timeout in ceph-create-keys of 600
+                '-u', ceph_user(),
-        #                 seconds is not adequate.  Increase timeout when
+                'ceph',
-        #                 timeout parameter available.  For older releases
+                '--name', 'mon.',
-        #                 we rely on retry_on_exception decorator.
+                '--keyring',
-        #                 LP#1719436
+                '/var/lib/ceph/mon/ceph-{}/keyring'.format(
-        cmd.extend(['--timeout', '1800'])
+                    socket.gethostname()
-    subprocess.check_call(cmd)
+                ),
-    _client_admin_keyring = '/etc/ceph/ceph.client.admin.keyring'
+                'auth', 'get', 'client.admin',
-    osstat = os.stat(_client_admin_keyring)
+            ]).decode('UTF-8')).strip()
-    if not osstat.st_size:
+        if not output:
-        # NOTE(fnordahl): Retry will fail as long as this file exists.
+            # NOTE: key not yet created, raise exception and retry
-        #                 LP#1719436
+            raise Exception
-        os.remove(_client_admin_keyring)
+        write_file(_client_admin_keyring, output,
-        raise Exception
+                   owner=ceph_user(), group=ceph_user(),
+                   perms=0o400)
+    else:
+        # NOTE(jamespage): Later ceph releases require explicit
+        #                  call to ceph-create-keys to setup the
+        #                  admin keys for the cluster; this command
+        #                  will wait for quorum in the cluster before
+        #                  returning.
+        # NOTE(fnordahl): Explicitly run `ceph-create-keys` for older
+        #                 ceph releases too.  This improves bootstrap
+        #                 resilience as the charm will wait for
+        #                 presence of peer units before attempting
+        #                 to bootstrap.  Note that charms deploying
+        #                 ceph-mon service should disable running of
+        #                 `ceph-create-keys` service in init system.
+        cmd = ['ceph-create-keys', '--id', socket.gethostname()]
+        if cmp_pkgrevno('ceph', '12.0.0') >= 0:
+            # NOTE(fnordahl): The default timeout in ceph-create-keys of 600
+            #                 seconds is not adequate.  Increase timeout when
+            #                 timeout parameter available.  For older releases
+            #                 we rely on retry_on_exception decorator.
+            #                 LP#1719436
+            cmd.extend(['--timeout', '1800'])
+        subprocess.check_call(cmd)
+        osstat = os.stat(_client_admin_keyring)
+        if not osstat.st_size:
+            # NOTE(fnordahl): Retry will fail as long as this file exists.
+            #                 LP#1719436
+            os.remove(_client_admin_keyring)
+            raise Exception
 
 
 def update_monfs():
@@ -1427,6 +1462,10 @@ def osdize(dev, osd_format, osd_journal, ignore_errors=False, encrypt=False,
                    ignore_errors, encrypt,
                    bluestore, key_manager)
     else:
+        if cmp_pkgrevno('ceph', '14.0.0') >= 0:
+            log("Directory backed OSDs can not be created on Nautilus",
+                level=WARNING)
+            return
         osdize_dir(dev, encrypt, bluestore)
 
 
@@ -1555,7 +1594,7 @@ def _ceph_disk(dev, osd_format, osd_journal, encrypt=False, bluestore=False):
         cmd.append(osd_format)
 
     # NOTE(jamespage): enable experimental bluestore support
-    if cmp_pkgrevno('ceph', '12.2.0') >= 0 and bluestore:
+    if use_bluestore():
         cmd.append('--bluestore')
         wal = get_devices('bluestore-wal')
         if wal:
@@ -1567,7 +1606,7 @@ def _ceph_disk(dev, osd_format, osd_journal, encrypt=False, bluestore=False):
             cmd.append('--block.db')
             least_used_db = find_least_used_utility_device(db)
             cmd.append(least_used_db)
-    elif cmp_pkgrevno('ceph', '12.2.0') >= 0 and not bluestore:
+    elif cmp_pkgrevno('ceph', '12.1.0') >= 0 and not bluestore:
         cmd.append('--filestore')
 
     cmd.append(os.path.realpath(dev))
@@ -2704,6 +2743,14 @@ def dirs_need_ownership_update(service):
         if (curr_owner == expected_owner) and (curr_group == expected_group):
             continue
 
+        # NOTE(lathiat): when config_changed runs on reboot, the OSD might not
+        # yet be mounted or started, and the underlying directory the OSD is
+        # mounted to is expected to be owned by root. So skip the check. This
+        # may also happen for OSD directories for OSDs that were removed.
+        if (service == 'osd' and
+                not os.path.exists(os.path.join(child, 'magic'))):
+            continue
+
         log('Directory "%s" needs its ownership updated' % child, DEBUG)
         return True
 
@@ -2716,6 +2763,7 @@ UPGRADE_PATHS = collections.OrderedDict([
     ('hammer', 'jewel'),
     ('jewel', 'luminous'),
     ('luminous', 'mimic'),
+    ('mimic', 'nautilus'),
 ])
 
 # Map UCA codenames to ceph codenames
@@ -2731,6 +2779,7 @@ UCA_CODENAME_MAP = {
     'queens': 'luminous',
     'rocky': 'mimic',
     'stein': 'mimic',
+    'train': 'nautilus',
 }