openstack
/
fuel-astute
Code Issues Proposed changes

Improvements for Nailgun hooks 

- implemenent raise workflow similar with critical
  roles (fail_deployment_on_error). Default - true;
- show detailed message for user in case of fail
  nailgun hook;
- implement support of cwd for execute_shell_command
  mclient agent and nailgun hooks;
- also refactring and tests.

Change-Id: I4458e7e66f62f6bf54509c3820e2ed0b69ea6315
Implements: blueprint cinder-neutron-plugins-in-fuel
This commit is contained in:
Vladimir Sharshov 2014-10-24 08:58:00 +04:00
parent 97eea90efe
commit c72dac7b31
8 changed files with 376 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
lib/astute/deployment_engine.rb
View File

@ -30,13 +30,12 @@ module Astute
begin
PreDeploymentActions.new(deployment_info, @ctx).process
NailgunHooks.new(pre_deployment, @ctx).process
rescue => e
Astute.logger.error("Unexpected error #{e.message} traceback #{e.format_backtrace}")
raise e
end
NailgunHooks.new(pre_deployment, @ctx).process
pre_node_actions = PreNodeActions.new(@ctx)
fail_deploy = false
@ -76,7 +75,22 @@ module Astute
end
# Post deployment hooks
NailgunHooks.new(post_deployment, @ctx).process
begin
NailgunHooks.new(post_deployment, @ctx).process
rescue => e
# We should fail all nodes in case of post deployment
# process. In other case they will not sending back
# for redeploy
nodes = deployment_info.uniq {|n| n['uid']}.map do |node|
{ 'uid' => node['uid'],
'status' => 'error',
'role' => 'hook',
'error_type' => 'deploy',
}
end
@ctx.report_and_update_status('nodes' => nodes)
raise e
end
PostDeploymentActions.new(deployment_info, @ctx).process
end

76
lib/astute/nailgun_hooks.rb
View File

@ -25,13 +25,29 @@ module Astute
@nailgun_hooks.sort_by { |f| f['priority'] }.each do |hook|
Astute.logger.info "Run hook #{hook.to_yaml}"
case hook['type']
success = case hook['type']
when 'sync' then sync_hook(hook)
when 'shell' then shell_hook(hook)
when 'upload_file' then upload_file_hook(hook)
when 'puppet' then puppet_hook(hook)
else raise "Unknown hook type #{hook['type']}"
end
is_raise_on_error = hook.fetch('fail_on_error', true)
if !success && is_raise_on_error
nodes = hook['uids'].map do |uid|
{ 'uid' => uid,
'status' => 'error',
'error_type' => 'deploy',
'role' => 'hook',
'hook' => hook['diagnostic_name']
}
end
@ctx.report_and_update_status('nodes' => nodes)
raise Astute::DeploymentEngineError,
"Failed to deploy plugin #{hook['diagnostic_name']}"
end
end
end
@ -43,27 +59,31 @@ module Astute
validate_presence(hook['parameters'], 'puppet_modules')
timeout = hook['parameters']['timeout'] || 300
cwd = hook['parameters']['cwd'] || "~/"
cwd = hook['parameters']['cwd'] || "/tmp"
shell_command = <<-PUPPET_CMD
cd #{cwd} &&
puppet apply --debug --verbose --logdest syslog
--modulepath=#{hook['parameters']['puppet_modules']}
#{hook['parameters']['puppet_manifest']}
PUPPET_CMD
shell_command.tr!("\n"," ")
is_success = true
perform_with_limit(hook['uids']) do |node_uids|
response = run_shell_command(
@ctx,
node_uids,
shell_command,
timeout
timeout,
cwd
)
if response[:data][:exit_code] != 0
Astute.logger.warn("Puppet run failed. Check puppet logs for details")
is_success = false
end
end
is_success
end #puppet_hook
def upload_file_hook(hook)
@ -73,9 +93,13 @@ module Astute
hook['parameters']['content'] = hook['parameters']['data']
is_success = true
perform_with_limit(hook['uids']) do |node_uids|
upload_file(@ctx, node_uids, hook['parameters'])
status = upload_file(@ctx, node_uids, hook['parameters'])
is_success = false if status == false
end
is_success
end
def shell_hook(hook)
@ -88,17 +112,22 @@ module Astute
shell_command = "cd #{cwd} && #{hook['parameters']['cmd']}"
is_success = true
perform_with_limit(hook['uids']) do |node_uids|
response = run_shell_command(
@ctx,
node_uids,
shell_command,
timeout
timeout,
cwd
)
if response[:data][:exit_code] != 0
Astute.logger.warn("Shell command failed. Check debug output for details")
is_success = false
end
end
is_success
end # shell_hook
@ -115,27 +144,32 @@ module Astute
rsync_options = '-c -r --delete'
rsync_cmd = "mkdir -p #{path} && rsync #{rsync_options} #{source} #{path}"
is_success = false
perform_with_limit(hook['uids']) do |node_uids|
sync_retries = 0
while sync_retries < 10
sync_retries += 1
10.times do |sync_retries|
response = run_shell_command(
@ctx,
node_uids,
rsync_cmd,
timeout
)
break if response[:data][:exit_code] == 0
if response[:data][:exit_code] == 0
is_success = true
break
end
Astute.logger.warn("Rsync problem. Try to repeat: #{sync_retries} attempt")
end
end
is_success
end # sync_hook
def validate_presence(data, key)
raise "Missing a required parameter #{key}" unless data[key].present?
end
def run_shell_command(context, node_uids, cmd, timeout=60)
def run_shell_command(context, node_uids, cmd, timeout=60, cwd="/tmp")
shell = MClient.new(context,
'execute_shell_command',
node_uids,
@ -144,15 +178,18 @@ module Astute
retries=1)
#TODO: return result for all nodes not only for first
response = shell.execute(:cmd => cmd).first
Astute.logger.debug("#{context.task_id}: cmd: #{cmd}
stdout: #{response[:data][:stdout]}
stderr: #{response[:data][:stderr]}
exit code: #{response[:data][:exit_code]}")
response = shell.execute(:cmd => cmd, :cwd => cwd).first
Astute.logger.debug(
"#{context.task_id}: cmd: #{cmd}\n" \
"cwd: #{cwd}\n" \
"stdout: #{response[:data][:stdout]}\n" \
"stderr: #{response[:data][:stderr]}\n" \
"exit code: #{response[:data][:exit_code]}")
response
rescue MClientTimeout, MClientError => e
Astute.logger.error("#{context.task_id}: cmd: #{cmd}
mcollective error: #{e.message}")
Astute.logger.error(
"#{context.task_id}: cmd: #{cmd} \n" \
"mcollective error: #{e.message}")
{:data => {}}
end
@ -176,8 +213,11 @@ module Astute
:group_owner => mco_params['group_owner'],
:dir_permissions => mco_params['dir_permissions']
)
true
rescue MClientTimeout, MClientError => e
Astute.logger.error("#{context.task_id}: mcollective upload_file agent error: #{e.message}")
false
end
def perform_with_limit(nodes, &block)

10
lib/astute/server/dispatcher.rb
View File

@ -76,7 +76,13 @@ module Astute
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], data['args']['task_uuid'])
begin
@orchestrator.task_deployment(reporter, data['args']['task_uuid'], data['args']['deployment_info'])
@orchestrator.task_deployment(
reporter,
data['args']['task_uuid'],
data['args']['deployment_info'],
data['args']['pre_deployment'] || [],
data['args']['post_deployment'] || []
)
reporter.report('status' => 'ready', 'progress' => 100)
rescue Timeout::Error
msg = "Timeout of deployment is exceeded."
@ -180,7 +186,7 @@ module Astute
Astute.logger.info "Try to kill running task #{target_task_uuid}"
service_data[:main_work_thread].kill
result = if service_data[:tasks_queue].current_task_method == 'deploy'
result = if ['deploy', 'task_deployment'].include? service_data[:tasks_queue].current_task_method
@orchestrator.stop_puppet_deploy(reporter, task_uuid, nodes)
@orchestrator.remove_nodes(reporter, task_uuid, data['args']['engine'], nodes)
else

6
lib/astute/server/server.rb
View File

@ -102,10 +102,12 @@ module Astute
abort_messages messages[(i + 1)..-1]
break
rescue => ex
Astute.logger.error "Error running RPC method #{message['method']}: #{ex.message}, trace: #{ex.backtrace.inspect}"
Astute.logger.error "Error running RPC method #{message['method']}: #{ex.message}, "
"trace: #{ex.format_backtrace}"
return_results message, {
'status' => 'error',
'error' => "Error occurred while running method '#{message['method']}'. Inspect Astute logs for the details"
'error' => "Method #{message['method']}. #{ex.message}.\n" \
"Inspect Astute logs for the details"
}
break
end

41
mcagents/execute_shell_command.ddl
View File

@ -8,23 +8,32 @@ metadata :name => "Execute shell command",
action "execute", :description => "Execute shell command" do
input :cmd,
:prompt => "Shell command",
:description => "Shell command for running",
:type => :string,
:validation => '.*',
:optional => false,
:maxlength => 0
input :cmd,
:prompt => "Shell command",
:description => "Shell command for running",
:type => :string,
:validation => '.*',
:optional => false,
:maxlength => 0
output :stdout,
:description => "Output from #{:cmd}",
:display_as => "Output"
input :cwd,
:prompt => "CWD",
:description => "Path to folder where command will be run",
:type => :string,
:validation => '.*',
:optional => true,
:default => '/tmp',
:maxlength => 0
output :stderr,
:description => "Stderr from #{:cmd}",
:display_as => "Stderr"
output :stdout,
:description => "Output from #{:cmd}",
:display_as => "Output"
output :exit_code,
:description => "Exit code of #{:cmd}",
:display_as => "Exit code"
output :stderr,
:description => "Stderr from #{:cmd}",
:display_as => "Stderr"
output :exit_code,
:description => "Exit code of #{:cmd}",
:display_as => "Exit code"
end

9
mcagents/execute_shell_command.rb
View File

@ -20,9 +20,12 @@ module MCollective
class Execute_shell_command < RPC::Agent
action 'execute' do
reply[:exit_code] = run(request[:cmd],
:stdout => :stdout,
:stderr => :stderr)
reply[:exit_code] = run(
request[:cmd],
:stdout => :stdout,
:stderr => :stderr,
:cwd => request.fetch(:cwd, '/tmp')
)
reply[:stdout] ||= ""
reply[:stderr] ||= ""
end

55
spec/unit/deployment_engine_spec.rb
View File

@ -80,21 +80,52 @@ describe Astute::DeploymentEngine do
deployer.deploy(nodes)
end
it 'should run pre and post deployment nailgun hooks run once for all cluster' do
pre_hook = mock('pre')
post_hook = mock('post')
hook_order = sequence('hook_order')
context 'nailgun hooks' do
it 'should run pre and post deployment nailgun hooks run once for all cluster' do
pre_hook = mock('pre')
post_hook = mock('post')
hook_order = sequence('hook_order')
Astute::NailgunHooks.expects(:new).with(pre_deployment, ctx).returns(pre_hook)
Astute::NailgunHooks.expects(:new).with(post_deployment, ctx).returns(post_hook)
Astute::NailgunHooks.expects(:new).with(pre_deployment, ctx).returns(pre_hook)
Astute::NailgunHooks.expects(:new).with(post_deployment, ctx).returns(post_hook)
Astute::PreDeploymentActions.any_instance.expects(:process).in_sequence(hook_order)
pre_hook.expects(:process).in_sequence(hook_order)
deployer.expects(:deploy_piece).in_sequence(hook_order)
post_hook.expects(:process).in_sequence(hook_order)
Astute::PostDeploymentActions.any_instance.expects(:process).in_sequence(hook_order)
Astute::PreDeploymentActions.any_instance.expects(:process).in_sequence(hook_order)
pre_hook.expects(:process).in_sequence(hook_order)
deployer.expects(:deploy_piece).in_sequence(hook_order)
post_hook.expects(:process).in_sequence(hook_order)
Astute::PostDeploymentActions.any_instance.expects(:process).in_sequence(hook_order)
deployer.deploy(nodes, pre_deployment, post_deployment)
deployer.deploy(nodes, pre_deployment, post_deployment)
end
it 'should not do additional update for node status if pre hooks failed' do
pre_hook = mock('pre')
Astute::NailgunHooks.expects(:new).with(pre_deployment, ctx).returns(pre_hook)
pre_hook.expects(:process).raises(Astute::DeploymentEngineError)
ctx.expects(:report_and_update_status).never
expect {deployer.deploy(nodes, pre_deployment, post_deployment)}.to raise_error(Astute::DeploymentEngineError)
end
it 'should update all nodes status to error if post hooks failed' do
pre_hook = mock('pre')
post_hook = mock('post')
Astute::NailgunHooks.expects(:new).with(pre_deployment, ctx).returns(pre_hook)
pre_hook.expects(:process)
Astute::NailgunHooks.expects(:new).with(post_deployment, ctx).returns(post_hook)
post_hook.expects(:process).raises(Astute::DeploymentEngineError)
ctx.expects(:report_and_update_status).with({
'nodes' => [
{'uid' => 1, 'status' => 'error', 'error_type' => 'deploy', 'role' => 'hook'},
{'uid' => 2, 'status' => 'error', 'error_type' => 'deploy', 'role' => 'hook'}
]
})
expect {deployer.deploy(nodes, pre_deployment, post_deployment)}.to raise_error(Astute::DeploymentEngineError)
end
end
it 'should run pre node hooks once for node' do

225
spec/unit/nailgun_hooks_spec.rb
View File

@ -32,6 +32,8 @@ describe Astute::NailgunHooks do
{
"priority" => 100,
"type" => "upload_file",
"fail_on_error" => false,
"diagnostic_name" => "upload-example-1.0",
"uids" => [2, 3],
"parameters" => {
"path" => "/etc/yum.repos.d/fuel_awesome_plugin-0.1.0.repo",
@ -44,9 +46,11 @@ describe Astute::NailgunHooks do
{
"priority" => 200,
"type" => "sync",
"fail_on_error" => false,
"diagnostic_name" => "sync-example-1.0",
"uids" => [1, 2],
"parameters" => {
"src" => "rsync => //10.20.0.2 => /plugins/fuel_awesome_plugin-0.1.0/deployment_scripts/",
"src" => "rsync://10.20.0.2/plugins/fuel_awesome_plugin-0.1.0/deployment_scripts/",
"dst" => "/etc/fuel/plugins/fuel_awesome_plugin-0.1.0/"
}
}
@ -56,6 +60,8 @@ describe Astute::NailgunHooks do
{
"priority" => 100,
"type" => "shell",
"fail_on_error" => false,
"diagnostic_name" => "shell-example-1.0",
"uids" => [1,2,3],
"parameters" => {
"cmd" => "./deploy.sh",
@ -69,6 +75,8 @@ describe Astute::NailgunHooks do
{
"priority" => 300,
"type" => "puppet",
"fail_on_error" => false,
"diagnostic_name" => "puppet-example-1.0",
"uids" => [1, 3],
"parameters" => {
"puppet_manifest" => "cinder_glusterfs.pp",
@ -127,6 +135,89 @@ describe Astute::NailgunHooks do
hooks.process
end
context 'critical hook' do
before(:each) do
hooks_data[2]['fail_on_error'] = true
ctx.stubs(:report_and_update_status)
end
it 'should raise exception if critical hook failed' do
hooks = Astute::NailgunHooks.new(hooks_data, ctx)
hooks.expects(:upload_file_hook).returns(true)
hooks.expects(:shell_hook).returns(false)
expect {hooks.process}.to raise_error(Astute::DeploymentEngineError, /Failed to deploy plugin shell-example-1.0/)
end
it 'should not process next hooks if critical hook failed' do
hooks = Astute::NailgunHooks.new(hooks_data, ctx)
hooks.expects(:upload_file_hook).returns(true)
hooks.expects(:shell_hook).returns(false)
hooks.expects(:sync_hook).never
hooks.expects(:puppet_hook).never
hooks.process rescue nil
end
it 'should process next hooks if non critical hook failed' do
hooks = Astute::NailgunHooks.new(hooks_data, ctx)
hooks.expects(:upload_file_hook).returns(false)
hooks.expects(:shell_hook).returns(true)
hooks.expects(:sync_hook).returns(false)
hooks.expects(:puppet_hook).returns(true)
hooks.process
end
it 'should report error node status if critical hook failed' do
hooks = Astute::NailgunHooks.new(hooks_data, ctx)
hooks.expects(:upload_file_hook).returns(true)
hooks.expects(:shell_hook).returns(false)
ctx.expects(:report_and_update_status).with(
{'nodes' =>
[
{ 'uid' => 1,
'status' => 'error',
'error_type' => 'deploy',
'role' => 'hook',
'hook' => "shell-example-1.0"
},
{ 'uid' => 2,
'status' => 'error',
'error_type' => 'deploy',
'role' => 'hook',
'hook' => "shell-example-1.0"
},
{ 'uid' => 3,
'status' => 'error',
'error_type' => 'deploy',
'role' => 'hook',
'hook' => "shell-example-1.0"
},
]
}
)
hooks.process rescue nil
end
it 'should not send report if non critical hook failed' do
hooks = Astute::NailgunHooks.new(hooks_data, ctx)
hooks.expects(:upload_file_hook).returns(false)
hooks.expects(:shell_hook).returns(true)
hooks.expects(:sync_hook).returns(false)
hooks.expects(:puppet_hook).returns(true)
ctx.expects(:report_and_update_status).never
hooks.process
end
end #hook
end #process
context '#shell_hook' do
@ -151,7 +242,8 @@ describe Astute::NailgunHooks do
ctx,
[1,2,3],
regexp_matches(/deploy/),
shell_hook['parameters']['timeout']
shell_hook['parameters']['timeout'],
shell_hook['parameters']['cwd']
)
.returns(:data => {:exit_code => 0})
@ -165,7 +257,8 @@ describe Astute::NailgunHooks do
ctx,
[1,2,3],
regexp_matches(/deploy/),
300
300,
shell_hook['parameters']['cwd']
)
.returns(:data => {:exit_code => 0})
@ -180,7 +273,8 @@ describe Astute::NailgunHooks do
ctx,
[1, 2],
regexp_matches(/deploy/),
shell_hook['parameters']['timeout']
shell_hook['parameters']['timeout'],
shell_hook['parameters']['cwd']
)
.returns(:data => {:exit_code => 0})
@ -188,13 +282,42 @@ describe Astute::NailgunHooks do
ctx,
[3],
regexp_matches(/deploy/),
shell_hook['parameters']['timeout']
shell_hook['parameters']['timeout'],
shell_hook['parameters']['cwd']
)
.returns(:data => {:exit_code => 0})
hooks.process
end
context 'process data from mcagent in case of critical hook' do
before(:each) do
shell_hook['fail_on_error'] = true
ctx.stubs(:report_and_update_status)
end
it 'if exit code eql 0 -> do not raise error' do
hooks = Astute::NailgunHooks.new([shell_hook], ctx)
hooks.expects(:run_shell_command).returns({:data => {:exit_code => 0}}).once
expect {hooks.process}.to_not raise_error
end
it 'if exit code not eql 0 -> raise error' do
hooks = Astute::NailgunHooks.new([shell_hook], ctx)
hooks.expects(:run_shell_command).returns({:data => {:exit_code => 1}}).once
expect {hooks.process}.to raise_error(Astute::DeploymentEngineError, /Failed to deploy plugin/)
end
it 'if exit code not presence -> raise error' do
hooks = Astute::NailgunHooks.new([shell_hook], ctx)
hooks.expects(:run_shell_command).returns({:data => {}}).once
expect {hooks.process}.to raise_error(Astute::DeploymentEngineError, /Failed to deploy plugin/)
end
end #context
end #shell_hook
context '#upload_file_hook' do
@ -259,6 +382,27 @@ describe Astute::NailgunHooks do
hooks.process
end
context 'process data from mcagent in case of critical hook' do
before(:each) do
upload_file_hook['fail_on_error'] = true
ctx.stubs(:report_and_update_status)
end
it 'mcagent success' do
hooks = Astute::NailgunHooks.new([upload_file_hook], ctx)
hooks.expects(:upload_file).returns(true).once
expect {hooks.process}.to_not raise_error
end
it 'mcagent fail' do
hooks = Astute::NailgunHooks.new([upload_file_hook], ctx)
hooks.expects(:upload_file).returns(false).once
expect {hooks.process}.to raise_error(Astute::DeploymentEngineError, /Failed to deploy plugin/)
end
end #context
end #upload_file_hook
context '#sync_hook' do
@ -333,6 +477,35 @@ describe Astute::NailgunHooks do
hooks.process
end
context 'process data from mcagent in case of critical hook' do
before(:each) do
sync_hook['fail_on_error'] = true
ctx.stubs(:report_and_update_status)
end
it 'if exit code eql 0 -> do not raise error' do
hooks = Astute::NailgunHooks.new([sync_hook], ctx)
hooks.expects(:run_shell_command).returns({:data => {:exit_code => 0}}).once
expect {hooks.process}.to_not raise_error
end
it 'if exit code not eql 0 -> raise error' do
hooks = Astute::NailgunHooks.new([sync_hook], ctx)
hooks.expects(:run_shell_command).returns({:data => {:exit_code => 1}}).times(10)
expect {hooks.process}.to raise_error(Astute::DeploymentEngineError, /Failed to deploy plugin/)
end
it 'if exit code not presence -> raise error' do
hooks = Astute::NailgunHooks.new([sync_hook], ctx)
hooks.expects(:run_shell_command).returns({:data => {}}).times(10)
expect {hooks.process}.to raise_error(Astute::DeploymentEngineError, /Failed to deploy plugin/)
end
end #context
end #sync_hook
context '#puppet_hook' do
@ -363,7 +536,8 @@ describe Astute::NailgunHooks do
ctx,
[1,3],
regexp_matches(/puppet/),
puppet_hook['parameters']['timeout']
puppet_hook['parameters']['timeout'],
puppet_hook['parameters']['cwd']
)
.returns(:data => {:exit_code => 0})
@ -377,7 +551,8 @@ describe Astute::NailgunHooks do
ctx,
[1,3],
regexp_matches(/puppet/),
300
300,
puppet_hook['parameters']['cwd']
)
.returns(:data => {:exit_code => 0})
@ -392,7 +567,8 @@ describe Astute::NailgunHooks do
ctx,
[1],
regexp_matches(/puppet/),
puppet_hook['parameters']['timeout']
puppet_hook['parameters']['timeout'],
puppet_hook['parameters']['cwd']
)
.returns(:data => {:exit_code => 0})
@ -400,12 +576,41 @@ describe Astute::NailgunHooks do
ctx,
[3],
regexp_matches(/puppet/),
puppet_hook['parameters']['timeout']
puppet_hook['parameters']['timeout'],
puppet_hook['parameters']['cwd']
)
.returns(:data => {:exit_code => 0})
hooks.process
end
end
context 'process data from mcagent in case of critical hook' do
before(:each) do
puppet_hook['fail_on_error'] = true
ctx.stubs(:report_and_update_status)
end
it 'if exit code eql 0 -> do not raise error' do
hooks = Astute::NailgunHooks.new([puppet_hook], ctx)
hooks.expects(:run_shell_command).returns({:data => {:exit_code => 0}}).once
expect {hooks.process}.to_not raise_error
end
it 'if exit code not eql 0 -> raise error' do
hooks = Astute::NailgunHooks.new([puppet_hook], ctx)
hooks.expects(:run_shell_command).returns({:data => {:exit_code => 1}}).once
expect {hooks.process}.to raise_error(Astute::DeploymentEngineError, /Failed to deploy plugin/)
end
it 'if exit code not presence -> raise error' do
hooks = Astute::NailgunHooks.new([puppet_hook], ctx)
hooks.expects(:run_shell_command).returns({:data => {}}).once
expect {hooks.process}.to raise_error(Astute::DeploymentEngineError, /Failed to deploy plugin/)
end
end #context
end # puppet_hook
end # 'describe'