Improve idempotency of block device processing

Record details of each device that is successfully processed as
an OSD data device to ensure that they are not re-processed
during subsequent hook executions.

"successfully processed" in this case means that the call to
either ceph-disk or ceph-volume to prepare (and potentially
activate) the OSD completed without error.

This avoids potential data loss on reboot when osd-reformat is
set to True as the charm will only every process a block device
once.

Change-Id: I2c6e9d5670c8d1d70584ae19b34eaf16be5dea19

ceph/utils.py

@@ -73,6 +73,7 @@ from charmhelpers.contrib.storage.linux.utils import (
 from charmhelpers.contrib.openstack.utils import (
     get_os_codename_install_source,
 )
+from charmhelpers.core.unitdata import kv
 
 CEPH_BASE_DIR = os.path.join(os.sep, 'var', 'lib', 'ceph')
 OSD_BASE_DIR = os.path.join(CEPH_BASE_DIR, 'osd')
@@ -1450,6 +1451,13 @@ def osdize(dev, osd_format, osd_journal, reformat_osd=False,
 
 def osdize_dev(dev, osd_format, osd_journal, reformat_osd=False,
                ignore_errors=False, encrypt=False, bluestore=False):
+    db = kv()
+    osd_devices = db.get('osd-devices', [])
+    if dev in osd_devices:
+        log('Device {} already processed by charm,'
+            ' skipping'.format(dev))
+        return
+
     if not os.path.exists(dev):
         log('Path {} does not exist - bailing'.format(dev))
         return
@@ -1519,6 +1527,13 @@ def osdize_dev(dev, osd_format, osd_journal, reformat_osd=False,
             log('Unable to initialize device: {}'.format(dev), ERROR)
             raise
 
+    # NOTE: Record processing of device only on success to ensure that
+    #       the charm only tries to initialize a device of OSD usage
+    #       once during its lifetime.
+    osd_devices.append(dev)
+    db.set('osd-devices', osd_devices)
+    db.flush()
+
 
 def osdize_dir(path, encrypt=False, bluestore=False):
     """Ask ceph-disk to prepare a directory to become an osd.

unit_tests/test_utils.py

@@ -67,12 +67,13 @@ class CephTestCase(unittest.TestCase):
             call(['udevadm', 'settle']),
         ])
 
+    @patch.object(utils, 'kv')
     @patch.object(utils.subprocess, 'check_call')
     @patch.object(utils.os.path, 'exists')
     @patch.object(utils, 'is_device_mounted')
     @patch.object(utils, 'cmp_pkgrevno')
     @patch.object(utils, 'is_block_device')
-    def test_osdize_dev(self, _is_blk, _cmp, _mounted, _exists, _call):
+    def test_osdize_dev(self, _is_blk, _cmp, _mounted, _exists, _call, _kv):
         """Test that the dev osd is initialized correctly"""
         _is_blk.return_value = True
         _mounted.return_value = False