From 3a8c2b0dc713458ccea3d1ee2657a4ef448f7b2f Mon Sep 17 00:00:00 2001 From: Michele Baldessari Date: Wed, 14 Aug 2019 16:40:46 +0200 Subject: [PATCH] Make the rabbitmq-ready exec more stringent Currently we use the following command to determine if rabbit is up and running *and* ready to service requests: rabbitmqctl eval "rabbit_mnesia:is_clustered()." | grep -q true Now we have occasionally observed that rabbitmqctl policies commands which are executed after said exec['rabbitmq-ready'] will fail. One potential reason is that is_clustered() can return true *before* the rabbit app is actually running. In fact we can see it does return true even though the app is stopped: ()[root@controller-1 /]$ rabbitmqctl stop_app Stopping rabbit application on node rabbit@controller-1 ... ()[root@controller-1 /]$ rabbitmqctl eval 'rabbit_mnesia:is_clustered().' true Let's switch to a combination of commands that check for the cluster to be up *and* the rabbitmq app to be running: ()[root@controller-1 /]$ rabbitmqctl stop_app Stopping rabbit application on node rabbit@controller-1 ... ()[root@controller-1 /]$ rabbitmqctl eval 'rabbit_nodes:is_running(node(), rabbit).' false Suggested-By: Bogdan Dobrelya Closes-Bug: #1835615 Change-Id: I29f779145a39cd16374a91626f7fae1581a18224 --- manifests/profile/pacemaker/rabbitmq_bundle.pp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/manifests/profile/pacemaker/rabbitmq_bundle.pp b/manifests/profile/pacemaker/rabbitmq_bundle.pp index 4997cdce2..89a0bceb7 100644 --- a/manifests/profile/pacemaker/rabbitmq_bundle.pp +++ b/manifests/profile/pacemaker/rabbitmq_bundle.pp @@ -315,7 +315,9 @@ class tripleo::profile::pacemaker::rabbitmq_bundle ( # This grep makes sure the rabbit app in erlang is up and running # which is enough to guarantee that the user will eventually get # replicated around the cluster - $check_command = 'rabbitmqctl eval "rabbit_mnesia:is_clustered()." | grep -q true' + $cmd1 = 'rabbitmqctl eval "rabbit_nodes:is_running(node(), rabbit)." | grep -q true' + $cmd2 = 'rabbitmqctl eval "rabbit_mnesia:is_clustered()." | grep -q true' + $check_command = "${cmd1} && ${cmd2}" } exec { 'rabbitmq-ready':