Implement key rotation on Quincy

This patchset implements key rotation for Quincy. This needs to be
done before merging to master as this action requires coordination
between ceph-mon and ceph-osd, and these use quincy/edge to run the
test bundles.

Change-Id: I2a49b89c7438626f55347b4201803496557b28ec
func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/1195

.gitreview

@@ -2,3 +2,5 @@
 host=review.opendev.org
 port=29418
 project=openstack/charm-ceph-osd.git
+
+defaultbranch=stable/quincy.2

hooks/ceph_hooks.py

@@ -83,6 +83,7 @@ from utils import (
     import_osd_upgrade_key,
     import_osd_removal_key,
     import_client_crash_key,
+    import_pending_key,
     get_host_ip,
     get_networks,
     assert_charm_supports_ipv6,
@@ -719,8 +720,23 @@ def get_bdev_enable_discard():
                           "bdev-enable-discard: %s") % bdev_enable_discard)
 
 
+def handle_pending_key(pending_key):
+    key_map = json.loads(pending_key)
+    for osd_id, key in key_map.items():
+        if not os.path.exists('/var/lib/ceph/osd/ceph-%s' % osd_id):
+            continue
+
+        import_pending_key(key, osd_id)
+        service_restart('ceph-osd@%s' % osd_id)
+
+
 @hooks.hook('mon-relation-changed')
 def mon_relation():
+    pending_key = relation_get('pending_key')
+    if pending_key:
+        handle_pending_key(pending_key)
+        return
+
     bootstrap_key = relation_get('osd_bootstrap_key')
     upgrade_key = relation_get('osd_upgrade_key')
     removal_key = relation_get('osd_disk_removal_key')

hooks/utils.py

@@ -85,15 +85,17 @@ def is_osd_bootstrap_ready():
     return os.path.exists(_bootstrap_keyring)
 
 
-def _import_key(key, path, name):
-    if not os.path.exists(path):
+def _import_key(key, path, name, override=False):
+    exists = os.path.exists(path)
+    if not exists or override:
+        create = ['--create-keyring'] if not exists else []
         cmd = [
             'sudo',
             '-u',
             ceph.ceph_user(),
             'ceph-authtool',
-            path,
-            '--create-keyring',
+            path
+        ] + create + [
             '--name={}'.format(name),
             '--add-key={}'.format(key)
         ]
@@ -140,6 +142,19 @@ def import_client_crash_key(key):
     _import_key(key, _client_crash_keyring, 'client.crash')
 
 
+def import_pending_key(key, osd_id):
+    """
+    Import a pending key, used for key rotation.
+
+    :param key: The pending cephx key that will replace the current one.
+    :type key: str
+    :param osd_id: The OSD id whose key will be replaced.
+    :type osd_id: str
+    :raises: subprocess.CalledProcessError"""
+    _import_key(key, '/var/lib/ceph/osd/ceph-%s/keyring' % osd_id,
+                'osd.%s' % osd_id, override=True)
+
+
 def render_template(template_name, context, template_dir=TEMPLATES_DIR):
     """Render Jinja2 template.
 

unit_tests/test_ceph_hooks.py

@@ -819,6 +819,18 @@ class CephHooksTestCase(unittest.TestCase):
             level=ceph_hooks.ERROR,
         )
 
+    @patch.object(ceph_hooks, 'service_restart')
+    @patch.object(ceph_hooks, 'import_pending_key')
+    @patch.object(ceph_hooks.os.path, 'exists')
+    def test_handle_pending_key(self, exists, import_pending_key,
+                                service_restart):
+        exists.return_value = True
+        pending_key = '{"0":"some-key"}'
+        ceph_hooks.handle_pending_key(pending_key)
+        exists.assert_called_with('/var/lib/ceph/osd/ceph-0')
+        import_pending_key.assert_called_with('some-key', '0')
+        service_restart.assert_called_with('ceph-osd@0')
+
 
 @patch.object(ceph_hooks, 'local_unit')
 @patch.object(ceph_hooks, 'relation_get')
@@ -903,3 +915,17 @@ class VaultLockerTestCase(unittest.TestCase):
         _cmp_pkgrevno.return_value = -1
         self.assertRaises(ValueError,
                           ceph_hooks.use_vaultlocker)
+
+
+class KeyRotationTestCase(unittest.TestCase):
+
+    @patch.object(ceph_hooks, 'import_pending_key')
+    @patch.object(ceph_hooks, 'service_restart')
+    @patch.object(ceph_hooks.os.path, 'exists')
+    def test_pending_key(self, exists, service_restart, import_pending_key):
+        pending_key = '{"1":"key1","2":"key2"}'
+        exists.side_effect = lambda x: x == '/var/lib/ceph/osd/ceph-2'
+
+        ceph_hooks.handle_pending_key(pending_key)
+        import_pending_key.assert_called_with("key2", "2")
+        service_restart.assert_called_with("ceph-osd@2")