Ability to stop puppet deploy at every nodes

Change-Id: Ie8d0e82b04cc53f9cebe150415d00c1f3d9aa22e

lib/astute/deployment_engine.rb

@@ -45,6 +45,9 @@ module Astute
 
           # Sync puppet manifests and modules to every node (emulate puppet master)
           sync_puppet_manifests(part)
+
+          # Unlock puppet (can be lock if puppet was killed by user)
+          enable_puppet_deploy(part.map{ |n| n['uid'] })
         end
       rescue => e
         Astute.logger.error("Unexpected error #{e.message} traceback #{e.format_backtrace}")
@@ -162,6 +165,11 @@ module Astute
       return status.exitstatus, stdout, stderr
     end
 
+    def enable_puppet_deploy(node_uids)
+      puppetd = MClient.new(@ctx, "puppetd", node_uids)
+      puppetd.enable
+    end
+
     def nodes_status(nodes, status, data_to_merge)
       {
         'nodes' => nodes.map do |n|

lib/astute/orchestrator.rb

@@ -149,6 +149,12 @@ module Astute
       NodesRemover.new(Context.new(task_id, reporter), nodes, reboot).remove
     end
 
+    def stop_puppet_deploy(reporter, task_id, nodes)
+      nodes_uids = nodes.map { |n| n['uid'] }.uniq
+      puppetd = MClient.new(Context.new(task_id, reporter), "puppetd", nodes_uids)
+      puppetd.stop_and_disable
+    end
+
     def dump_environment(reporter, task_id, lastdump)
       Dump.dump_environment(Context.new(task_id, reporter), lastdump)
     end

mcagents/puppetd.ddl

@@ -29,6 +29,12 @@ action "last_run_summary", :description => "Get a summary of the last puppet run
            :display_as => "Versions"
 end
 
+action "stop_and_disable", :description => "Stop and disable puppet" do
+    output :output,
+           :description => "String indicating status",
+           :display_as => "Status"
+end
+
 action "enable", :description => "Enable puppet" do
     output :output,
            :description => "String indicating status",

mcagents/puppetd.rb

@@ -12,6 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+require 'timeout'
 
 module MCollective
   module Agent
@@ -43,6 +44,7 @@ module MCollective
                                                                                  -p #{@lockfile} \
                                                                                  /usr/bin/puppet apply /etc/puppet/manifests/site.pp"
         @last_summary = @config.pluginconf["puppet.summary"] || "/var/lib/puppet/state/last_run_summary.yaml"
+        @lockmcofile = "/tmp/mcopuppetd.lock"
       end
 
       action "last_run_summary" do
@@ -66,6 +68,10 @@ module MCollective
         set_status
       end
 
+      action "stop_and_disable" do
+        stop_and_disable
+      end
+
       private
 
       def last_run_summary
@@ -112,7 +118,7 @@ module MCollective
 
       def puppet_daemon_status
         err_msg = ""
-        alive = !!puppet_agent_pid
+        alive = !!puppet_pid
         locked = File.exists?(@lockfile)
         disabled = locked && File::Stat.new(@lockfile).zero?
 
@@ -137,30 +143,31 @@ module MCollective
       end
 
       def runonce
-        set_status
-        case (reply[:status])
-        when 'disabled' then     # can't run
-          reply.fail "Empty Lock file exists; puppet is disabled."
+        lock_file(@lockmcofile) do
+          set_status
+          case (reply[:status])
+          when 'disabled' then     # can't run
+            reply.fail "Empty Lock file exists; puppet is disabled."
 
-        when 'running' then      # can't run two simultaniously
-          reply.fail "Lock file and PID file exist; puppet is running."
+          when 'running' then      # can't run two simultaniously
+            reply.fail "Lock file and PID file exist; puppet is running."
 
-        when 'idling' then       # signal daemon
-          pid = puppet_agent_pid
-          begin
-            ::Process.kill('INT', pid)
-          rescue Errno::ESRCH => e
-            reply[:err_msg] = "Failed to signal the puppet apply daemon (process #{pid}): #{e}"
-          ensure
+          when 'idling' then       # signal daemon
+            pid = puppet_agent_pid
+            begin
+              ::Process.kill('INT', pid)
+            rescue Errno::ESRCH => e
+              reply[:err_msg] = "Failed to signal the puppet apply daemon (process #{pid}): #{e}"
+            ensure
+              runonce_background
+              reply[:output] = "Kill old idling puppet process #{pid})." + (reply[:output] || '')
+            end
+
+          when 'stopped' then      # just run
             runonce_background
-            reply[:output] = "Kill old idling puppet process #{pid})." + (reply[:output] || '')
+          else
+            reply.fail "Unknown puppet status: #{reply[:status]}"
           end
-
-        when 'stopped' then      # just run
-          runonce_background
-
-        else
-          reply.fail "Unknown puppet status: #{reply[:status]}"
         end
       end
 
@@ -181,6 +188,22 @@ module MCollective
         reply[:output] = "Called #{cmd}, " + output + (reply[:output] || '')
       end
 
+      def stop_and_disable
+        lock_file(@lockmcofile) do
+          case puppet_daemon_status
+          when 'stopped'
+            disable
+          when 'disabled'
+            reply[:output] = "Puppet already stoped and disabled"
+            return
+          else
+            kill_process
+            disable
+          end
+          reply[:output] = "Puppet stoped and disabled"
+        end
+      end
+
       def enable
         if File.exists?(@lockfile)
           stat = File::Stat.new(@lockfile)
@@ -211,10 +234,43 @@ module MCollective
         end
       end
 
-      def puppet_agent_pid
+      private
+
+      def kill_process
+        return if ['stopped', 'disabled'].include? puppet_daemon_status
+
+        begin
+          Timeout.timeout(30) do
+            Process.kill('TERM', puppet_pid)
+            while puppet_pid do
+              sleep 1
+            end
+          end
+        rescue Timeout::Error
+          Process.kill('KILL', puppet_pid)
+        end
+        #FIXME: Daemonized process do not update lock file when we send signal to kill him
+        raise "Should never happen. Some process block lock file in critical section" unless rm_file(@lockfile)
+      rescue => e
+        reply.fail "Failed to kill the puppet daemon (process #{puppet_pid}): #{e}"
+      end
+
+      def puppet_pid
         result = `ps -C puppet -o pid,comm --no-headers`.lines.first
         result && result.strip.split(' ')[0].to_i
       end
+
+      def lock_file(file_name, &block)
+        File.open(file_name, 'w+') do |f|
+          begin
+            f.flock File::LOCK_EX
+            yield
+          ensure
+            f.flock File::LOCK_UN
+          end
+        end
+      end
+
     end
   end
 end

spec/unit/deployment_engine_spec.rb

@@ -40,6 +40,7 @@ describe Astute::DeploymentEngine do
       deployer.stubs(:generate_ssh_keys)
       deployer.stubs(:upload_ssh_keys)
       deployer.stubs(:sync_puppet_manifests)
+      deployer.stubs(:enable_puppet_deploy)
     end
 
     it 'should generate and upload ssh keys' do
@@ -53,6 +54,15 @@ describe Astute::DeploymentEngine do
       deployer.deploy(nodes)
     end
 
+    it 'should enable puppet for all nodes' do
+      nodes = [{'uid' => 1, 'deployment_id' => 1}, {'uid' => 2}, {'uid' => 1}]
+      deployer.stubs(:deploy_piece)
+
+      deployer.expects(:enable_puppet_deploy).with([1,2]).returns()
+
+      deployer.deploy(nodes)
+    end
+
     it 'deploy nodes by order' do
       nodes = [{'uid' => 1, 'priority' => 10}, {'uid' => 2, 'priority' => 0}, {'uid' => 1, 'priority' => 15}]
 
@@ -144,6 +154,7 @@ describe Astute::DeploymentEngine do
       deployer.stubs(:deploy_piece)
       deployer.stubs(:generate_ssh_keys)
       deployer.stubs(:upload_ssh_keys)
+      deployer.stubs(:enable_puppet_deploy)
     end
 
     let(:nodes) { [{'uid' => 1, 'deployment_id' => 1, 'master_ip' => '10.20.0.2'}, {'uid' => 2}] }
@@ -167,6 +178,7 @@ describe Astute::DeploymentEngine do
       Astute.config.PUPPET_SSH_KEYS = ['nova']
       deployer.stubs(:deploy_piece)
       deployer.stubs(:sync_puppet_manifests)
+      deployer.stubs(:enable_puppet_deploy)
     end
 
     let(:nodes) { [{'uid' => 1, 'deployment_id' => 1}, {'uid' => 2}] }

spec/unit/nailyfact_deploy_spec.rb

@@ -51,6 +51,7 @@ describe "NailyFact DeploymentEngine" do
       deploy_engine.stubs(:generate_ssh_keys).with(deploy_data.first['deployment_id'])
       deploy_engine.stubs(:upload_ssh_keys).with(uniq_nodes_uid, deploy_data.first['deployment_id'])
       deploy_engine.stubs(:sync_puppet_manifests).with(deploy_data.uniq { |n| n['uid'] })
+      deploy_engine.stubs(:enable_puppet_deploy).with(uniq_nodes_uid)
     end
 
     context 'log parsing' do