Implement the 'rotate-key' action for managers

This patchset implements key rotation for managers only. The user
can specified either the full entity name (i.e: 'mgr.XXXX') or
simply 'mgr', which stands for the local manager.

After the entity's directory is located, a new pending key is
generated, the keyring file is mutated to include the new key and
then replaced in situ. Lastly, the manager service is restarted.

Note that Ceph only has one active manager at a certain point,
so it only makes sense to call this action on _every_ mon unit.

Change-Id: Ie24b3f30922fa5be6641e37635440891614539d5
func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/1195

actions.yaml

@@ -454,3 +454,10 @@ list-entities:
         - text
       default: text
       description: "The output format, either json, yaml or text (default)"
+rotate-key:
+  description: "Rotate the key of an entity in the Ceph cluster"
+  params:
+    entity:
+      type: string
+      description: The entity for which to rotate the key
+  required: [entity]

src/charm.py

@@ -230,6 +230,8 @@ class CephMonCharm(ops_openstack.core.OSBaseCharm):
                              ops_actions.get_erasure_profile.erasure_profile)
         self._observe_action(self.on.list_entities_action,
                              ops_actions.list_entities.list_entities)
+        self._observe_action(self.on.rotate_key_action,
+                             ops_actions.rotate_key.rotate_key)
 
         fw.observe(self.on.install, self.on_install)
         fw.observe(self.on.config_changed, self.on_config)

src/ops_actions/__init__.py

@@ -20,4 +20,5 @@ from . import (   # noqa: F401
     get_health,
     get_erasure_profile,
     list_entities,
+    rotate_key,
 )

src/ops_actions/list_entities.py

@@ -31,7 +31,7 @@ def list_entities(event):
         # since it sometimes contain escaped strings that are incompatible
         # with python's json module. This method of fetching entities is
         # simple enough and portable across Ceph versions.
-        out = subprocess.check_call(['sudo', 'ceph', 'auth', 'ls'])
+        out = subprocess.check_output(['sudo', 'ceph', 'auth', 'ls'])
         ret = []
 
         for line in out.decode('utf-8').split('\n'):

src/ops_actions/rotate_key.py

@@ -0,0 +1,103 @@
+#! /usr/bin/env python3
+#
+# Copyright 2024 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Rotate the key of one or more entities."""
+
+import configparser
+import json
+import logging
+import os
+import subprocess
+
+import charms.operator_libs_linux.v1.systemd as systemd
+
+
+logger = logging.getLogger(__name__)
+MGR_DIR = "/var/lib/ceph/mgr"
+
+
+def _find_mgr_path(base):
+    name = "ceph-" + base
+    try:
+        if name in os.listdir(MGR_DIR):
+            return MGR_DIR + "/" + name
+    except FileNotFoundError as exc:
+        logger.exception(exc)
+        return None
+
+
+def _create_key(entity, event):
+    try:
+        cmd = ["sudo", "ceph", "auth", "get-or-create-pending",
+               entity, "--format=json"]
+        out = subprocess.check_output(cmd).decode("utf-8")
+        return json.loads(out)[0]["pending_key"]
+    except (subprocess.SubprocessError, json.decoder.JSONDecodeError) as exc:
+        logger.exception(exc)
+        event.fail("Failed to create key: %s" % str(exc))
+        raise
+
+
+def _replace_keyring_file(path, entity, key, event):
+    path += "/keyring"
+    try:
+        c = configparser.ConfigParser(default_section=None)
+        c.read(path)
+        c[entity]["key"] = key
+
+        with open(path, "w") as file:
+            c.write(file)
+    except (KeyError, IOError) as exc:
+        logger.exception(exc)
+        event.fail("Failed to replace keyring file: %s" % str(exc))
+        raise
+
+
+def _restart_daemon(entity, event):
+    try:
+        systemd.service_restart(entity)
+    except systemd.SystemdError as exc:
+        logger.exception(exc)
+        event.fail("Failed to reload daemon: %s" % str(exc))
+        raise
+
+
+def rotate_key(event) -> None:
+    """Rotate the key of the specified entity."""
+    entity = event.params.get("entity")
+    if entity.startswith("mgr"):
+        if len(entity) > 3:
+            if entity[3] != '.':
+                event.fail("Invalid entity name: %s" % entity)
+                return
+            path = _find_mgr_path(entity[4:])
+            if path is None:
+                event.fail("Entity %s not found" % entity)
+                return
+        else:   # just 'mgr'
+            try:
+                path = MGR_DIR + "/" + os.listdir(MGR_DIR)[0]
+                entity = "mgr." + os.path.basename(path)[5:]   # skip 'ceph-'
+            except Exception:
+                event.fail("No managers found")
+                return
+
+        key = _create_key(entity, event)
+        _replace_keyring_file(path, entity, key, event)
+        _restart_daemon("ceph-mgr@%s.service" % entity[4:], event)
+        event.set_results({"message": "success"})
+    else:
+        event.fail("Unknown entity: %s" % entity)

tests/tests.yaml

@@ -39,3 +39,4 @@ tests:
   - zaza.openstack.charm_tests.ceph.tests.CephAuthTest
   - zaza.openstack.charm_tests.ceph.tests.CephMonActionsTest
   - zaza.openstack.charm_tests.ceph.mon.tests.CephPermissionUpgradeTest
+  - zaza.openstack.charm_tests.ceph.tests.CephMonKeyRotationTests

unit_tests/test_ceph_actions.py

@@ -18,6 +18,7 @@ import subprocess
 import test_utils
 import ops_actions.copy_pool as copy_pool
 import ops_actions.list_entities as list_entities
+import ops_actions.rotate_key as rotate_key
 
 with mock.patch('charmhelpers.contrib.hardening.harden.harden') as mock_dec:
     mock_dec.side_effect = (lambda *dargs, **dkwargs: lambda f:
@@ -294,9 +295,9 @@ class ListEntities(test_utils.CharmTestCase):
         self.harness.begin()
         self.addCleanup(self.harness.cleanup)
 
-    @mock.patch.object(list_entities.subprocess, 'check_call')
-    def test_list_entities(self, check_call):
-        check_call.return_value = b"""
+    @mock.patch.object(list_entities.subprocess, 'check_output')
+    def test_list_entities(self, check_output):
+        check_output.return_value = b"""
 client.admin
   key: AQAOwwFmTR3TNxAAIsdYgastd0uKntPtEnoWug==
 mgr.0
@@ -307,3 +308,51 @@ mgr.0
         event.set_results.assert_called_once_with(
             {"message": "client.admin\nmgr.0"}
         )
+
+
+# Needs to be outside as the decorator wouldn't find it otherwise.
+MGR_KEYRING_FILE = """
+[mgr.host-1]
+  key = old-key
+"""
+
+
+class RotateKey(test_utils.CharmTestCase):
+    """Run tests for action."""
+
+    def setUp(self):
+        self.harness = Harness(CephMonCharm)
+        self.harness.begin()
+        self.addCleanup(self.harness.cleanup)
+
+    def test_invalid_entity(self):
+        event = test_utils.MockActionEvent({'entity': '???'})
+        self.harness.charm.on_rotate_key_action(event)
+        event.fail.assert_called_once()
+
+    def test_invalid_mgr(self):
+        event = test_utils.MockActionEvent({'entity': 'mgr-123'})
+        self.harness.charm.on_rotate_key_action(event)
+        event.fail.assert_called_once()
+
+    @mock.patch('builtins.open', new_callable=mock.mock_open,
+                read_data=MGR_KEYRING_FILE)
+    @mock.patch.object(rotate_key.systemd, 'service_restart')
+    @mock.patch.object(rotate_key.subprocess, 'check_output')
+    @mock.patch.object(rotate_key.os, 'listdir')
+    def test_rotate_mgr_key(self, listdir, check_output, service_restart,
+                            _open):
+        listdir.return_value = ['ceph-host-1']
+        check_output.return_value = b'[{"pending_key": "new-key"}]'
+
+        event = test_utils.MockActionEvent({'entity': 'mgr.host-1'})
+        self.harness.charm.on_rotate_key_action(event)
+
+        event.set_results.assert_called_with({'message': 'success'})
+        listdir.assert_called_once_with('/var/lib/ceph/mgr')
+        check_output.assert_called_once()
+        service_restart.assert_called_once_with('ceph-mgr@host-1.service')
+
+        calls = any(x for x in _open.mock_calls
+                    if any(p is not None and 'new-key' in p for p in x.args))
+        self.assertTrue(calls)