Improve idempotency of block device processing

Resync charms.ceph to pickup improvements in recording
of block devices that have been processed as OSD devices
to support better idempotency of block device processing
codepaths.

This fixes a particularly nasty issue with osd-reformat
is set to True where the charm can wipe and re-prepare
an OSD device prior to the systemd unit actually booting
and mounting the OSD's associated filesystem.

This change also makes the osd-reformat option a boolean
option which is more accessible to users of the charm
via the CLI and the Juju GUI.

Change-Id: I578203aeebf6da2efc21a10d2e157324186e2a66
Depends-On: I2c6e9d5670c8d1d70584ae19b34eaf16be5dea19

config.yaml

@@ -128,15 +128,16 @@ options:
       Note that despite bluestore being the default for Ceph Luminous, if this
       option is False, OSDs will still use filestore.
   osd-reformat:
-    type: string
-    default:
+    type: boolean
+    default: False
     description: |
       By default, the charm will not re-format a device that already looks
       as if it might be an OSD device. This is a safeguard to try to
       prevent data loss.
       .
-      Specifying this option (any value) forces a reformat of any OSD devices
-      found which are not already mounted.
+      Enabling this option forces a reformat of any OSD devices found which
+      have not been processed by the unit previously or are not already
+      mounted.
   osd-encrypt:
     type: boolean
     default: False

lib/ceph/utils.py

@@ -76,6 +76,7 @@ from charmhelpers.contrib.openstack.utils import (
     get_os_codename_install_source,
 )
 from charmhelpers.contrib.storage.linux import lvm
+from charmhelpers.core.unitdata import kv
 
 CEPH_BASE_DIR = os.path.join(os.sep, 'var', 'lib', 'ceph')
 OSD_BASE_DIR = os.path.join(CEPH_BASE_DIR, 'osd')
@@ -1473,6 +1474,13 @@ def osdize(dev, osd_format, osd_journal, reformat_osd=False,
 
 def osdize_dev(dev, osd_format, osd_journal, reformat_osd=False,
                ignore_errors=False, encrypt=False, bluestore=False):
+    db = kv()
+    osd_devices = db.get('osd-devices', [])
+    if dev in osd_devices:
+        log('Device {} already processed by charm,'
+            ' skipping'.format(dev))
+        return
+
     if not os.path.exists(dev):
         log('Path {} does not exist - bailing'.format(dev))
         return
@@ -1515,12 +1523,28 @@ def osdize_dev(dev, osd_format, osd_journal, reformat_osd=False,
         log("osdize cmd: {}".format(cmd))
         subprocess.check_call(cmd)
     except subprocess.CalledProcessError:
+        try:
+            lsblk_output = subprocess.check_output(
+                ['lsblk', '-P']).decode('UTF-8')
+        except subprocess.CalledProcessError as e:
+            log("Couldn't get lsblk output: {}".format(e), ERROR)
         if ignore_errors:
             log('Unable to initialize device: {}'.format(dev), WARNING)
+            if lsblk_output:
+                log('lsblk output: {}'.format(lsblk_output), DEBUG)
         else:
             log('Unable to initialize device: {}'.format(dev), ERROR)
+            if lsblk_output:
+                log('lsblk output: {}'.format(lsblk_output), WARNING)
             raise
 
+    # NOTE: Record processing of device only on success to ensure that
+    #       the charm only tries to initialize a device of OSD usage
+    #       once during its lifetime.
+    osd_devices.append(dev)
+    db.set('osd-devices', osd_devices)
+    db.flush()
+
 
 def _ceph_disk(dev, osd_format, osd_journal, encrypt=False, bluestore=False):
     """

tests/basic_deployment.py

@@ -113,7 +113,7 @@ class CephOsdBasicDeployment(OpenStackAmuletDeployment):
         # Include a non-existent device as osd-devices is a whitelist,
         # and this will catch cases where proposals attempt to change that.
         ceph_osd_config = {
-            'osd-reformat': 'yes',
+            'osd-reformat': True,
             'ephemeral-unmount': '/mnt',
             'osd-devices': '/dev/vdb /srv/ceph /dev/test-non-existent'
         }