Don't delete nodes not reachable by MCollective

Nodes are checked for MCollective connectivity prior to deletion.
If any node is unreachable it will be put into 'error' state and
the entire deletion task will be aborted. This only occurs when
deleting individual nodes, not when deleting an entire cluster
or resetting it.

Change-Id: Ieda212d83c0354346db315eb7da27d416a728dbc
Closes-bug: #1449285

lib/astute/orchestrator.rb

@@ -194,6 +194,10 @@ module Astute
       cluster_id
     end
 
+    def check_for_offline_nodes(reporter, task_id, nodes)
+      PreDelete.check_for_offline_nodes(Context.new(task_id, reporter), nodes)
+    end
+
     def check_ceph_osds(reporter, task_id, nodes)
       PreDelete.check_ceph_osds(Context.new(task_id, reporter), nodes)
     end
@@ -204,7 +208,11 @@ module Astute
 
     def perform_pre_deletion_tasks(reporter, task_id, nodes, options={})
       result = {'status' => 'ready'}
+      # This option is no longer Ceph-specific and should be renamed
+      # FIXME(rmoe): https://bugs.launchpad.net/fuel/+bug/1454377
       if options[:check_ceph]
+        result = check_for_offline_nodes(reporter, task_id, nodes)
+        return result if result['status'] != 'ready'
         result = check_ceph_osds(reporter, task_id, nodes)
         return result if result['status'] != 'ready'
         result = remove_ceph_mons(reporter, task_id, nodes)

lib/astute/pre_delete.rb

@@ -126,5 +126,29 @@ module Astute
 
       answer
     end
+
+    def self.check_for_offline_nodes(ctx, nodes)
+      answer = {"status" => "ready"}
+      mco_nodes = nodes.collect { |n| n['id'] }
+
+      client = MClient.new(ctx, "version", mco_nodes, check_result=false)
+      responses = client.get_version
+      online_nodes = responses.map(&:results).collect { |r| r[:sender].to_i }
+      offline_nodes = (mco_nodes.to_set - online_nodes.to_set).to_a
+
+      if offline_nodes.present?
+        offline_nodes.map! { |e| {'uid' => e} }
+        msg = "MCollective is not running on nodes " \
+              "#{offline_nodes.collect {|n| n['uid'] }.join(',')}. " \
+              "MCollective must be running to properly delete a node."
+
+        answer = {'status' => 'error',
+                  'error' => msg,
+                  'error_nodes' => offline_nodes}
+      end
+
+      answer
+    end
+
   end
 end

spec/unit/dispatcher_spec.rb

@@ -27,7 +27,10 @@ describe Astute::Server::Dispatcher do
     end
 
     let (:orchestrator) do
-      Astute::Orchestrator.any_instance
+      orchestrator = Astute::Orchestrator.any_instance
+      orchestrator.stubs(:check_for_offline_nodes).returns({"status"=>"ready"})
+
+      orchestrator
     end
 
     let (:data) {

spec/unit/pre_delete_spec.rb

@@ -197,4 +197,40 @@ describe Astute::PreDelete do
 
   end # remove_ceph_mons
 
-end
+  context "verify that mcollective is running" do
+    let(:nodes) { [
+        {"id" => 1, "roles" => ["controller"]},
+        {"id" => 2, "roles" => ["compute"]}
+      ]
+    }
+    let(:error_result) do
+      msg = "MCollective is not running on nodes 2. " \
+            "MCollective must be running to properly delete a node."
+
+      {"status" => "error",
+       "error" => msg,
+       "error_nodes" => [{"uid" => 2}]
+      }
+    end
+
+    it "should prevent deletion of nodes when mcollective is not running" do
+      rs = mock()
+      rs.stubs(:map).returns([{:sender => "1"}])
+
+      mclient.expects(:get_version).returns(rs)
+      expect(Astute::PreDelete.check_for_offline_nodes(ctx, nodes)).to eq(error_result)
+    end
+
+    it "should allow deletion of nodes when mcollective is running" do
+      rs = mock()
+      rs.stubs(:map).returns( [
+        {:sender => "1"},
+        {:sender => "2"}
+      ])
+      mclient.expects(:get_version).returns(rs)
+
+      expect(Astute::PreDelete.check_for_offline_nodes(ctx, nodes)).to eq(success_result)
+    end
+  end
+
+end # describe