Add rabbitmq service check

* Check whether rabbitmq is online before doing any rabbitmq queries
* Add new provider for custom service checks

Change-Id: I1fb27ab2d731dc3fa7daa3694c9da3a01f318695
Closes-Bug: 1591213

deployment/puppet/cluster/.fixtures.yml

@@ -14,3 +14,4 @@ fixtures:
     apt: "#{source_dir}/../apt"
     staging: "#{source_dir}/../staging"
     xinetd: "#{source_dir}/../xinetd"
+    osnailyfacter: "#{source_dir}/../osnailyfacter"

deployment/puppet/cluster/manifests/rabbitmq_ocf.pp

@@ -215,5 +215,8 @@ class cluster::rabbitmq_ocf (
     parameters          => $parameters,
     #    ocf_script_file     => $ocf_script_file,
   }
-  Service[$service_name] -> Rabbitmq_user <||>
+
+  ensure_resource('service_status', ['rabbitmq'], { 'ensure' => 'online', check_cmd => 'rabbitmqctl cluster_status'})
+
+  Service[$service_name] -> Service_status['rabbitmq'] -> Rabbitmq_user <||>
 }

deployment/puppet/osnailyfacter/lib/puppet/provider/service_status/service_status.rb

@@ -0,0 +1,42 @@
+Puppet::Type.type(:service_status).provide(:ruby) do
+  desc 'Wait for custom service to become online'
+
+  defaultfor :kernel => :linux
+
+  # get the current custom service status value
+  # @return [:online, :offline]
+  def ensure
+    debug 'Call: ensure'
+    out = status
+    debug "Return: #{out}"
+    out
+  end
+
+  # wait for custom service to switch to needed state
+  # @param value [:online, :offline]
+  def ensure=(value)
+    debug "Call: ensure=(#{value})"
+    debug "Waiting for custome service: '#{@resource[:name]}' to change its status to: '#{value}'"
+    @resource[:count].times do
+      if status == value
+        return true
+      end
+      sleep @resource[:step]
+    end
+    fail "Timeout waiting for custom service: '#{@resource[:name]}' to become: '#{value}' after #{@resource[:count] * @resource[:step]} seconds!"
+  end
+
+  # get custom service status
+  # @return [:online, :offline]
+  def status
+    rv = system(@resource[:check_cmd])
+    status = $?.exitstatus
+    debug "Got status: '#{status}'"
+    if status == @resource[:exitcode]
+      :online
+    else
+      :offline
+    end
+  end
+
+end

deployment/puppet/osnailyfacter/lib/puppet/type/service_status.rb

@@ -0,0 +1,54 @@
+Puppet::Type.newtype(:service_status) do
+  desc  'Wait for custom service to become online'
+
+  newparam(:name) do
+    desc 'The name of custom service to monitor'
+    isnamevar
+  end
+
+  newproperty(:ensure) do
+    desc 'Expected custom service status'
+    newvalues :online, :offline
+    defaultto :online
+  end
+
+  newparam(:check_cmd) do
+    desc 'Command to check status'
+  end
+
+  newparam(:exitcode) do
+    desc 'Supposed exitcode for check_cmd'
+    newvalues(/\d+/)
+    defaultto 0
+    munge do |n|
+      n.to_i
+    end
+  end
+
+  newparam(:count) do
+    desc 'How many times try to perform check?'
+    newvalues(/\d+/)
+    defaultto 100
+    munge do |n|
+      n.to_i
+    end
+  end
+
+  newparam(:step) do
+    desc 'How many seconds to wait between retries?'
+    newvalues(/\d+/)
+    defaultto 6
+    munge do |n|
+      n.to_i
+    end
+  end
+
+  newparam(:timeout) do
+    desc 'How long should we wait for a request to finish?'
+    newvalues(/\d+/)
+    defaultto 5
+    munge do |n|
+      n.to_i
+    end
+  end
+end

deployment/puppet/osnailyfacter/manifests/rabbitmq/rabbitmq.pp

@@ -125,9 +125,11 @@ class osnailyfacter::rabbitmq::rabbitmq {
     )
     $environment_variables = merge($environment_variables_init,{'NODENAME' => "rabbit@${fqdn_prefix}${hostname}"})
 
+    $rabbitmq_admin_enabled = pick($rabbit_hash['use_rabbitmq_admin'], true)
+
     if ($enabled) {
       class { '::rabbitmq':
-        admin_enable                => true,
+        admin_enable                => $rabbitmq_admin_enabled,
         management_port             => $management_port,
         repos_ensure                => false,
         package_provider            => $package_provider,
@@ -161,6 +163,13 @@ class osnailyfacter::rabbitmq::rabbitmq {
           tries       => 30,
           try_sleep   => 6,
       }
+
+      if $rabbitmq_admin_enabled {
+        ensure_resource('service_status', ['rabbitmq'], { 'ensure' => 'online', check_cmd => 'rabbitmqctl cluster_status'})
+
+        Service_status['rabbitmq'] -> Staging::File['rabbitmqadmin']
+      }
+
       # TODO(bogdando) contribute this to puppetlabs-rabbitmq
       # Start epmd as rabbitmq so it doesn't run as root when installing plugins
       exec { 'epmd_daemon':

tests/noop/spec/hosts/rabbitmq/rabbitmq_spec.rb

@@ -52,6 +52,7 @@ describe manifest do
     debug = Noop.hiera('debug', false)
     threads_max = 2*workers_max.to_i
     rabbit_hash = Noop.hiera_structure 'rabbit'
+    rabbitmq_admin_enabled = rabbit_hash.fetch(['use_rabbitmq_admin'], true)
     use_pacemaker = rabbit_hash.fetch(['pacemaker'], true)
     pid_file = rabbit_hash.fetch('pid_file', '/var/run/rabbitmq/p_pid')
     file_limit = rabbit_hash.fetch('file_limits', 100000)
@@ -75,7 +76,7 @@ describe manifest do
     })
 
     let (:params) do {
-      :admin_enable                => true,
+      :admin_enable                => rabbitmq_admin_enabled,
       :management_port             => management_port,
       :repos_ensure                => false,
       :package_provider            => $package_provider,
@@ -141,6 +142,12 @@ describe manifest do
       )
     end
 
+    if rabbitmq_admin_enabled
+      it 'should check rabbitmq status' do
+        should contain_service_status('rabbitmq').that_comes_before('Staging::File[rabbitmqadmin]')
+      end
+    end
+
     it 'should configure rabbit fence class' do
       if use_pacemaker
         should contain_class('cluster::rabbitmq_fence').with(
@@ -182,6 +189,7 @@ describe manifest do
         ).that_comes_before('Class[nova::rabbitmq]')
         should contain_class('cluster::rabbitmq_ocf').that_requires(
           'Class[rabbitmq::install]')
+        should contain_service_status('rabbitmq').that_requires('Service[rabbitmq-server]')
       end
     end