diff --git a/actions.yaml b/actions.yaml index 06927683..f60ca5bc 100644 --- a/actions.yaml +++ b/actions.yaml @@ -42,3 +42,13 @@ forget-cluster-node: node: type: string description: Node name i.e. rabbit@ +force-boot: + description: | + Set the `force_boot` flag and restart the RabbitMQ broker. This + action should be performed if a unit in the RabbitMQ cluster is + failing to boot after an uncontrolled shutdown of the cluster. + Note that units of a RabbitMQ cluster have to be booted in reverse + shutdown order. Also note that this potentially leads to a loss of + messages, in particular if the cluster received messages after the + unit was shut down. + See https://www.rabbitmq.com/clustering.html#restarting and LP: #1828988 diff --git a/actions/actions.py b/actions/actions.py index 729fbf4e..45da3190 100755 --- a/actions/actions.py +++ b/actions/actions.py @@ -34,6 +34,11 @@ def _add_path(path): _add_path(_root) _add_path(_hooks) +from charmhelpers.core.host import ( + service_start, + service_stop, +) + from charmhelpers.core.hookenv import ( action_fail, action_set, @@ -183,6 +188,32 @@ def list_unconsumed_queues(args): action_set({'unconsumed-queue-count': count}) +def force_boot(args): + """Set the force_boot flag and start RabbitMQ broker""" + try: + service_stop('rabbitmq-server') + except CalledProcessError as e: + action_set({'output': e.output}) + action_fail('Failed to stop rabbitmqctl service') + return False + + try: + force_boot = check_output(['rabbitmqctl', 'force_boot'], + universal_newlines=True) + action_set({'output': force_boot}) + except CalledProcessError as e: + action_set({'output': e.output}) + action_fail('Failed to run rabbitmqctl force_boot') + return False + + try: + service_start('rabbitmq-server') + except CalledProcessError as e: + action_set({'output': e.output}) + action_fail('Failed to start rabbitmqctl service after force_boot') + return False + + # A dictionary of all the defined actions to callables (which take # parsed arguments). ACTIONS = { @@ -193,6 +224,7 @@ ACTIONS = { "complete-cluster-series-upgrade": complete_cluster_series_upgrade, "forget-cluster-node": forget_cluster_node, "list-unconsumed-queues": list_unconsumed_queues, + "force-boot": force_boot, } diff --git a/actions/force-boot b/actions/force-boot new file mode 120000 index 00000000..405a394e --- /dev/null +++ b/actions/force-boot @@ -0,0 +1 @@ +actions.py \ No newline at end of file diff --git a/unit_tests/test_actions.py b/unit_tests/test_actions.py index 7b4b8ac6..8614d579 100644 --- a/unit_tests/test_actions.py +++ b/unit_tests/test_actions.py @@ -182,6 +182,55 @@ class ListUnconsumedQueuesTestCase(CharmTestCase): "Failed to query RabbitMQ vhost / queues") +class ForceBootTestCase(CharmTestCase): + """Tests for force-boot action. + + """ + + def setUp(self): + super(ForceBootTestCase, self).setUp( + actions, ["service_stop", "service_start", "check_output", + "action_set", "action_fail"]) + + def test_force_boot_fail_stop(self): + self.service_stop.side_effect = \ + actions.CalledProcessError(1, "Failure") + actions.force_boot([]) + self.service_stop.assert_called() + self.service_stop.assert_called_once_with('rabbitmq-server') + self.action_set.assert_called() + self.action_fail.assert_called_once_with( + 'Failed to stop rabbitmqctl service') + + def test_force_boot_fail(self): + self.check_output.side_effect = \ + actions.CalledProcessError(1, "Failure") + actions.force_boot([]) + self.service_stop.assert_called() + self.check_output.assert_called() + self.action_set.assert_called() + self.action_fail.assert_called_once_with( + 'Failed to run rabbitmqctl force_boot') + + def test_force_boot_fail_start(self): + self.service_start.side_effect = \ + actions.CalledProcessError(1, "Failure") + actions.force_boot([]) + self.service_start.assert_called() + self.service_start.assert_called_once_with('rabbitmq-server') + self.action_set.assert_called() + self.action_fail.assert_called_once_with( + 'Failed to start rabbitmqctl service after force_boot') + + def test_force_boot(self): + output = 'Forcing boot for Mnesia dir ' \ + '/var/lib/rabbitmq/mnesia/rabbit@juju-f52a8d-rabbitmq-12' + self.check_output.return_value = output + actions.force_boot([]) + self.check_output.assert_called() + self.action_set.assert_called_once_with({'output': output}) + + class MainTestCase(CharmTestCase): def setUp(self):