Add `force-boot` action

This change adds a `force-boot` action which sets the `force_boot`
flag and restarts the RabbitMQ broker. This action can be used if a
broker refuses to start because the master of a queue is not
available.

Also add appropriate unit tests.

Change-Id: I8b01d1d668e18116c7f8b1fc56f197620a10c91f
Partial-Bug: #1828988
Signed-off-by: Nicolas Bock <nicolas.bock@canonical.com>

actions.yaml

@@ -42,3 +42,13 @@ forget-cluster-node:
     node:
       type: string
       description: Node name i.e. rabbit@<hostname>
+force-boot:
+  description: |
+    Set the `force_boot` flag and restart the RabbitMQ broker. This
+    action should be performed if a unit in the RabbitMQ cluster is
+    failing to boot after an uncontrolled shutdown of the cluster.
+    Note that units of a RabbitMQ cluster have to be booted in reverse
+    shutdown order. Also note that this potentially leads to a loss of
+    messages, in particular if the cluster received messages after the
+    unit was shut down.
+    See https://www.rabbitmq.com/clustering.html#restarting and LP: #1828988

actions/actions.py

@@ -34,6 +34,11 @@ def _add_path(path):
 _add_path(_root)
 _add_path(_hooks)
 
+from charmhelpers.core.host import (
+    service_start,
+    service_stop,
+)
+
 from charmhelpers.core.hookenv import (
     action_fail,
     action_set,
@@ -183,6 +188,32 @@ def list_unconsumed_queues(args):
     action_set({'unconsumed-queue-count': count})
 
 
+def force_boot(args):
+    """Set the force_boot flag and start RabbitMQ broker"""
+    try:
+        service_stop('rabbitmq-server')
+    except CalledProcessError as e:
+        action_set({'output': e.output})
+        action_fail('Failed to stop rabbitmqctl service')
+        return False
+
+    try:
+        force_boot = check_output(['rabbitmqctl', 'force_boot'],
+                                  universal_newlines=True)
+        action_set({'output': force_boot})
+    except CalledProcessError as e:
+        action_set({'output': e.output})
+        action_fail('Failed to run rabbitmqctl force_boot')
+        return False
+
+    try:
+        service_start('rabbitmq-server')
+    except CalledProcessError as e:
+        action_set({'output': e.output})
+        action_fail('Failed to start rabbitmqctl service after force_boot')
+        return False
+
+
 # A dictionary of all the defined actions to callables (which take
 # parsed arguments).
 ACTIONS = {
@@ -193,6 +224,7 @@ ACTIONS = {
     "complete-cluster-series-upgrade": complete_cluster_series_upgrade,
     "forget-cluster-node": forget_cluster_node,
     "list-unconsumed-queues": list_unconsumed_queues,
+    "force-boot": force_boot,
 }
 
 

actions/force-boot

@@ -0,0 +1 @@
+actions.py

unit_tests/test_actions.py

@@ -182,6 +182,55 @@ class ListUnconsumedQueuesTestCase(CharmTestCase):
             "Failed to query RabbitMQ vhost / queues")
 
 
+class ForceBootTestCase(CharmTestCase):
+    """Tests for force-boot action.
+
+    """
+
+    def setUp(self):
+        super(ForceBootTestCase, self).setUp(
+            actions, ["service_stop", "service_start", "check_output",
+                      "action_set", "action_fail"])
+
+    def test_force_boot_fail_stop(self):
+        self.service_stop.side_effect = \
+            actions.CalledProcessError(1, "Failure")
+        actions.force_boot([])
+        self.service_stop.assert_called()
+        self.service_stop.assert_called_once_with('rabbitmq-server')
+        self.action_set.assert_called()
+        self.action_fail.assert_called_once_with(
+            'Failed to stop rabbitmqctl service')
+
+    def test_force_boot_fail(self):
+        self.check_output.side_effect = \
+            actions.CalledProcessError(1, "Failure")
+        actions.force_boot([])
+        self.service_stop.assert_called()
+        self.check_output.assert_called()
+        self.action_set.assert_called()
+        self.action_fail.assert_called_once_with(
+            'Failed to run rabbitmqctl force_boot')
+
+    def test_force_boot_fail_start(self):
+        self.service_start.side_effect = \
+            actions.CalledProcessError(1, "Failure")
+        actions.force_boot([])
+        self.service_start.assert_called()
+        self.service_start.assert_called_once_with('rabbitmq-server')
+        self.action_set.assert_called()
+        self.action_fail.assert_called_once_with(
+            'Failed to start rabbitmqctl service after force_boot')
+
+    def test_force_boot(self):
+        output = 'Forcing boot for Mnesia dir ' \
+            '/var/lib/rabbitmq/mnesia/rabbit@juju-f52a8d-rabbitmq-12'
+        self.check_output.return_value = output
+        actions.force_boot([])
+        self.check_output.assert_called()
+        self.action_set.assert_called_once_with({'output': output})
+
+
 class MainTestCase(CharmTestCase):
 
     def setUp(self):