From 0843a91d79fe6f547cc5817d96bd9ab655dfa446 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 18 Nov 2013 16:17:59 +0400 Subject: [PATCH] Ability to stop puppet deploy at every nodes Change-Id: Ie8d0e82b04cc53f9cebe150415d00c1f3d9aa22e --- lib/astute/deployment_engine.rb | 8 +++ lib/astute/orchestrator.rb | 6 ++ mcagents/puppetd.ddl | 6 ++ mcagents/puppetd.rb | 100 ++++++++++++++++++++++------ spec/unit/deployment_engine_spec.rb | 12 ++++ spec/unit/nailyfact_deploy_spec.rb | 1 + 6 files changed, 111 insertions(+), 22 deletions(-) diff --git a/lib/astute/deployment_engine.rb b/lib/astute/deployment_engine.rb index a456dd42..beecacd4 100644 --- a/lib/astute/deployment_engine.rb +++ b/lib/astute/deployment_engine.rb @@ -45,6 +45,9 @@ module Astute # Sync puppet manifests and modules to every node (emulate puppet master) sync_puppet_manifests(part) + + # Unlock puppet (can be lock if puppet was killed by user) + enable_puppet_deploy(part.map{ |n| n['uid'] }) end rescue => e Astute.logger.error("Unexpected error #{e.message} traceback #{e.format_backtrace}") @@ -162,6 +165,11 @@ module Astute return status.exitstatus, stdout, stderr end + def enable_puppet_deploy(node_uids) + puppetd = MClient.new(@ctx, "puppetd", node_uids) + puppetd.enable + end + def nodes_status(nodes, status, data_to_merge) { 'nodes' => nodes.map do |n| diff --git a/lib/astute/orchestrator.rb b/lib/astute/orchestrator.rb index 143ffcca..6613774e 100644 --- a/lib/astute/orchestrator.rb +++ b/lib/astute/orchestrator.rb @@ -149,6 +149,12 @@ module Astute NodesRemover.new(Context.new(task_id, reporter), nodes, reboot).remove end + def stop_puppet_deploy(reporter, task_id, nodes) + nodes_uids = nodes.map { |n| n['uid'] }.uniq + puppetd = MClient.new(Context.new(task_id, reporter), "puppetd", nodes_uids) + puppetd.stop_and_disable + end + def dump_environment(reporter, task_id, lastdump) Dump.dump_environment(Context.new(task_id, reporter), lastdump) end diff --git a/mcagents/puppetd.ddl b/mcagents/puppetd.ddl index 20e8e2c4..a7dd3de1 100644 --- a/mcagents/puppetd.ddl +++ b/mcagents/puppetd.ddl @@ -29,6 +29,12 @@ action "last_run_summary", :description => "Get a summary of the last puppet run :display_as => "Versions" end +action "stop_and_disable", :description => "Stop and disable puppet" do + output :output, + :description => "String indicating status", + :display_as => "Status" +end + action "enable", :description => "Enable puppet" do output :output, :description => "String indicating status", diff --git a/mcagents/puppetd.rb b/mcagents/puppetd.rb index 66100e0b..b864dd3c 100644 --- a/mcagents/puppetd.rb +++ b/mcagents/puppetd.rb @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +require 'timeout' module MCollective module Agent @@ -43,6 +44,7 @@ module MCollective -p #{@lockfile} \ /usr/bin/puppet apply /etc/puppet/manifests/site.pp" @last_summary = @config.pluginconf["puppet.summary"] || "/var/lib/puppet/state/last_run_summary.yaml" + @lockmcofile = "/tmp/mcopuppetd.lock" end action "last_run_summary" do @@ -66,6 +68,10 @@ module MCollective set_status end + action "stop_and_disable" do + stop_and_disable + end + private def last_run_summary @@ -112,7 +118,7 @@ module MCollective def puppet_daemon_status err_msg = "" - alive = !!puppet_agent_pid + alive = !!puppet_pid locked = File.exists?(@lockfile) disabled = locked && File::Stat.new(@lockfile).zero? @@ -137,30 +143,31 @@ module MCollective end def runonce - set_status - case (reply[:status]) - when 'disabled' then # can't run - reply.fail "Empty Lock file exists; puppet is disabled." + lock_file(@lockmcofile) do + set_status + case (reply[:status]) + when 'disabled' then # can't run + reply.fail "Empty Lock file exists; puppet is disabled." - when 'running' then # can't run two simultaniously - reply.fail "Lock file and PID file exist; puppet is running." + when 'running' then # can't run two simultaniously + reply.fail "Lock file and PID file exist; puppet is running." - when 'idling' then # signal daemon - pid = puppet_agent_pid - begin - ::Process.kill('INT', pid) - rescue Errno::ESRCH => e - reply[:err_msg] = "Failed to signal the puppet apply daemon (process #{pid}): #{e}" - ensure + when 'idling' then # signal daemon + pid = puppet_agent_pid + begin + ::Process.kill('INT', pid) + rescue Errno::ESRCH => e + reply[:err_msg] = "Failed to signal the puppet apply daemon (process #{pid}): #{e}" + ensure + runonce_background + reply[:output] = "Kill old idling puppet process #{pid})." + (reply[:output] || '') + end + + when 'stopped' then # just run runonce_background - reply[:output] = "Kill old idling puppet process #{pid})." + (reply[:output] || '') + else + reply.fail "Unknown puppet status: #{reply[:status]}" end - - when 'stopped' then # just run - runonce_background - - else - reply.fail "Unknown puppet status: #{reply[:status]}" end end @@ -181,6 +188,22 @@ module MCollective reply[:output] = "Called #{cmd}, " + output + (reply[:output] || '') end + def stop_and_disable + lock_file(@lockmcofile) do + case puppet_daemon_status + when 'stopped' + disable + when 'disabled' + reply[:output] = "Puppet already stoped and disabled" + return + else + kill_process + disable + end + reply[:output] = "Puppet stoped and disabled" + end + end + def enable if File.exists?(@lockfile) stat = File::Stat.new(@lockfile) @@ -211,10 +234,43 @@ module MCollective end end - def puppet_agent_pid + private + + def kill_process + return if ['stopped', 'disabled'].include? puppet_daemon_status + + begin + Timeout.timeout(30) do + Process.kill('TERM', puppet_pid) + while puppet_pid do + sleep 1 + end + end + rescue Timeout::Error + Process.kill('KILL', puppet_pid) + end + #FIXME: Daemonized process do not update lock file when we send signal to kill him + raise "Should never happen. Some process block lock file in critical section" unless rm_file(@lockfile) + rescue => e + reply.fail "Failed to kill the puppet daemon (process #{puppet_pid}): #{e}" + end + + def puppet_pid result = `ps -C puppet -o pid,comm --no-headers`.lines.first result && result.strip.split(' ')[0].to_i end + + def lock_file(file_name, &block) + File.open(file_name, 'w+') do |f| + begin + f.flock File::LOCK_EX + yield + ensure + f.flock File::LOCK_UN + end + end + end + end end end diff --git a/spec/unit/deployment_engine_spec.rb b/spec/unit/deployment_engine_spec.rb index 6a6395e9..24e6b01a 100644 --- a/spec/unit/deployment_engine_spec.rb +++ b/spec/unit/deployment_engine_spec.rb @@ -40,6 +40,7 @@ describe Astute::DeploymentEngine do deployer.stubs(:generate_ssh_keys) deployer.stubs(:upload_ssh_keys) deployer.stubs(:sync_puppet_manifests) + deployer.stubs(:enable_puppet_deploy) end it 'should generate and upload ssh keys' do @@ -53,6 +54,15 @@ describe Astute::DeploymentEngine do deployer.deploy(nodes) end + it 'should enable puppet for all nodes' do + nodes = [{'uid' => 1, 'deployment_id' => 1}, {'uid' => 2}, {'uid' => 1}] + deployer.stubs(:deploy_piece) + + deployer.expects(:enable_puppet_deploy).with([1,2]).returns() + + deployer.deploy(nodes) + end + it 'deploy nodes by order' do nodes = [{'uid' => 1, 'priority' => 10}, {'uid' => 2, 'priority' => 0}, {'uid' => 1, 'priority' => 15}] @@ -144,6 +154,7 @@ describe Astute::DeploymentEngine do deployer.stubs(:deploy_piece) deployer.stubs(:generate_ssh_keys) deployer.stubs(:upload_ssh_keys) + deployer.stubs(:enable_puppet_deploy) end let(:nodes) { [{'uid' => 1, 'deployment_id' => 1, 'master_ip' => '10.20.0.2'}, {'uid' => 2}] } @@ -167,6 +178,7 @@ describe Astute::DeploymentEngine do Astute.config.PUPPET_SSH_KEYS = ['nova'] deployer.stubs(:deploy_piece) deployer.stubs(:sync_puppet_manifests) + deployer.stubs(:enable_puppet_deploy) end let(:nodes) { [{'uid' => 1, 'deployment_id' => 1}, {'uid' => 2}] } diff --git a/spec/unit/nailyfact_deploy_spec.rb b/spec/unit/nailyfact_deploy_spec.rb index fa05baed..7a9129fe 100644 --- a/spec/unit/nailyfact_deploy_spec.rb +++ b/spec/unit/nailyfact_deploy_spec.rb @@ -51,6 +51,7 @@ describe "NailyFact DeploymentEngine" do deploy_engine.stubs(:generate_ssh_keys).with(deploy_data.first['deployment_id']) deploy_engine.stubs(:upload_ssh_keys).with(uniq_nodes_uid, deploy_data.first['deployment_id']) deploy_engine.stubs(:sync_puppet_manifests).with(deploy_data.uniq { |n| n['uid'] }) + deploy_engine.stubs(:enable_puppet_deploy).with(uniq_nodes_uid) end context 'log parsing' do