openstack
/
fuel-astute
Code Issues Proposed changes

Support puppet hook processing by PuppetD 

Instead of using execute_shell_command magent
now puppet hooks will be processing by algorithm
which we used to run puppet for main deployment.

This changes give us ability to run complex
puppet manifests with a lot of modules.

Change-Id: I7201a35501802b342fa708ee9e8f29087ce4a302
Implements: blueprint cinder-neutron-plugins-in-fuel
This commit is contained in:
Vladimir Sharshov 2014-10-31 15:11:07 +03:00 committed by Vladimir Sharshov (warpc)
parent c72dac7b31
commit 49edd57968
8 changed files with 196 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lib/astute/exceptions.rb
View File

@ -11,6 +11,7 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
require 'timeout'
module Astute
@ -27,5 +28,7 @@ module Astute
class DeploymentEngineError < AstuteError; end
# MClient errors
class MClientError < AstuteError; end
# MClient timeout error
class MClientTimeout < Timeout::Error; end
end

2
lib/astute/mclient.rb
View File

@ -19,8 +19,6 @@ require 'timeout'
module Astute
class MClientTimeout < Timeout::Error; end
class MClient
include MCollective::RPC

51
lib/astute/nailgun_hooks.rb
View File

@ -57,27 +57,21 @@ module Astute
validate_presence(hook, 'uids')
validate_presence(hook['parameters'], 'puppet_manifest')
validate_presence(hook['parameters'], 'puppet_modules')
validate_presence(hook['parameters'], 'cwd')
timeout = hook['parameters']['timeout'] || 300
cwd = hook['parameters']['cwd'] || "/tmp"
shell_command = <<-PUPPET_CMD
puppet apply --debug --verbose --logdest syslog
--modulepath=#{hook['parameters']['puppet_modules']}
#{hook['parameters']['puppet_manifest']}
PUPPET_CMD
shell_command.tr!("\n"," ")
is_success = true
perform_with_limit(hook['uids']) do |node_uids|
response = run_shell_command(
result = run_puppet(
@ctx,
node_uids,
shell_command,
timeout,
cwd
hook['parameters']['puppet_manifest'],
hook['parameters']['puppet_modules'],
hook['parameters']['cwd'],
timeout
)
if response[:data][:exit_code] != 0
unless result
Astute.logger.warn("Puppet run failed. Check puppet logs for details")
is_success = false
end
@ -169,6 +163,28 @@ module Astute
raise "Missing a required parameter #{key}" unless data[key].present?
end
def run_puppet(context, node_uids, puppet_manifest, puppet_modules, cwd, timeout)
# Prevent send report status to Nailgun
hook_context = Context.new(context.task_id, HookReporter.new)
nodes = node_uids.map { |node_id| {'uid' => node_id.to_s, 'role' => 'hook'} }
Timeout::timeout(timeout) {
PuppetdDeployer.deploy(
hook_context,
nodes,
retries=2,
puppet_manifest,
puppet_modules,
cwd
)
}
!hook_context.status.has_value?('error')
rescue Astute::MClientTimeout, Astute::MClientError, Timeout::Error => e
Astute.logger.error("#{context.task_id}: puppet timeout error: #{e.message}")
false
end
def run_shell_command(context, node_uids, cmd, timeout=60, cwd="/tmp")
shell = MClient.new(context,
'execute_shell_command',
@ -196,7 +212,7 @@ module Astute
def upload_file(context, node_uids, mco_params={})
upload_mclient = Astute::MClient.new(context, "uploadfile", Array(node_uids))
mco_params['overwrite'] = false if mco_params['overwrite'].nil?
mco_params['overwrite'] = true if mco_params['overwrite'].nil?
mco_params['parents'] = true if mco_params['parents'].nil?
mco_params['permissions'] ||= '0644'
mco_params['user_owner'] ||= 'root'
@ -227,4 +243,11 @@ module Astute
end
end # class
class HookReporter
def report(msg)
Astute.logger.debug msg
end
end
end # module

16
lib/astute/puppetd.rb
View File

@ -19,11 +19,14 @@ require 'timeout'
module Astute
module PuppetdDeployer
def self.deploy(ctx, nodes, retries=2)
def self.deploy(ctx, nodes, retries=2, puppet_manifest=nil, puppet_modules=nil, cwd=nil)
@ctx = ctx
@nodes_roles = nodes.inject({}) { |h, n| h.merge({n['uid'] => n['role']}) }
@node_retries = nodes.inject({}) { |h, n| h.merge({n['uid'] => retries}) }
@nodes_roles = nodes.inject({}) { |h, n| h.merge({n['uid'].to_s => n['role']}) }
@node_retries = nodes.inject({}) { |h, n| h.merge({n['uid'].to_s => retries}) }
@nodes = nodes
@puppet_manifest = puppet_manifest || '/etc/puppet/manifests/site.pp'
@puppet_modules = puppet_modules || '/etc/puppet/modules'
@cwd = cwd || '/'
Astute.logger.debug "Waiting for puppet to finish deployment on all
nodes (timeout = #{Astute.config.PUPPET_TIMEOUT} sec)..."
@ -51,7 +54,12 @@ module Astute
@nodes.select { |n| stopped_uids.include? n['uid'] }
.group_by { |n| n['debug'] }
.each do |debug, stop_nodes|
puppetd(stop_nodes.map { |n| n['uid'] }).runonce(:puppet_debug => true)
puppetd(stop_nodes.map { |n| n['uid'] }).runonce(
:puppet_debug => true,
:manifest => @puppet_manifest,
:modules => @puppet_modules,
:cwd => @cwd
)
end
break if running_uids.empty?

2
lib/astute/server/server.rb
View File

@ -102,7 +102,7 @@ module Astute
abort_messages messages[(i + 1)..-1]
break
rescue => ex
Astute.logger.error "Error running RPC method #{message['method']}: #{ex.message}, "
Astute.logger.error "Error running RPC method #{message['method']}: #{ex.message}, " \
"trace: #{ex.format_backtrace}"
return_results message, {
'status' => 'error',

42
mcagents/puppetd.ddl
View File

@ -48,13 +48,41 @@ action "disable", :description => "Disable puppet" do
end
action "runonce", :description => "Invoke a single puppet run" do
#input :forcerun,
# :prompt => "Force puppet run",
# :description => "Should the puppet run happen immediately?",
# :type => :string,
# :validation => '^.+$',
# :optional => true,
# :maxlength => 5
input :manifest,
:prompt => "Path to manifest",
:description => "Path to manifest (relative or absolutely)",
:type => :string,
:validation => :shellsafe,
:optional => true,
:default => '/etc/puppet/manifests/site.pp',
:maxlength => 0
input :modules,
:prompt => "Path to modules folder",
:description => "Path to modules folder (relative or absolutely)",
:type => :string,
:validation => :shellsafe,
:optional => true,
:default => '/etc/puppet/modules',
:maxlength => 0
input :cwd,
:prompt => "CWD",
:description => "Path to folder where command will be run",
:type => :string,
:validation => :shellsafe,
:optional => true,
:default => '/tmp',
:maxlength => 0
input :puppet_debug,
:prompt => "Puppet debug",
:description => "Puppet run mode: debug",
:type => :boolean,
:validation => :typecheck,
:default => true,
:optional => true
output :output,
:description => "Output from puppet",

25
mcagents/puppetd.rb
View File

@ -39,11 +39,11 @@ module MCollective
@log = @config.pluginconf["puppetd.log"] || "/var/log/puppet.log"
@statefile = @config.pluginconf["puppetd.statefile"] || "/var/lib/puppet/state/state.yaml"
@pidfile = @config.pluginconf["puppet.pidfile"] || "/var/run/puppet/agent.pid"
@puppetd = @config.pluginconf["puppetd.puppetd"] || "/usr/sbin/daemonize -a \
-l #{@lockfile} \
-p #{@lockfile} \
/usr/bin/puppet apply \
/etc/puppet/manifests/site.pp"
@puppetd = @config.pluginconf["puppetd.puppetd"] ||
"/usr/sbin/daemonize -a \
-l #{@lockfile} \
-p #{@lockfile} "
@puppetd_agent = "/usr/bin/puppet apply"
@last_summary = @config.pluginconf["puppet.summary"] || "/var/lib/puppet/state/last_run_summary.yaml"
@lockmcofile = "/tmp/mcopuppetd.lock"
end
@ -173,7 +173,18 @@ module MCollective
end
def runonce_background
cmd = [@puppetd, "--logdest", 'syslog', '--trace', '--no-report']
cwd = request.fetch(:cwd, '/')
cmd = [
@puppetd,
"-c #{cwd}",
@puppetd_agent,
request.fetch(:manifest, '/etc/puppet/manifests/site.pp'),
"--modulepath=#{request.fetch(:modules, '/etc/puppet/modules')}",
'--logdest',
'syslog',
'--trace',
'--no-report'
]
unless request[:forcerun]
if @splaytime && @splaytime > 0
cmd << "--splaylimit" << @splaytime << "--splay"
@ -187,7 +198,7 @@ module MCollective
cmd = cmd.join(" ")
output = reply[:output] || ''
run(cmd, :stdout => :output, :chomp => true)
run(cmd, :stdout => :output, :chomp => true, :cwd => cwd)
reply[:output] = "Called #{cmd}, " + output + (reply[:output] || '')
end

143
spec/unit/nailgun_hooks_spec.rb
View File

@ -34,7 +34,7 @@ describe Astute::NailgunHooks do
"type" => "upload_file",
"fail_on_error" => false,
"diagnostic_name" => "upload-example-1.0",
"uids" => [2, 3],
"uids" => ['2', '3'],
"parameters" => {
"path" => "/etc/yum.repos.d/fuel_awesome_plugin-0.1.0.repo",
"data" => "[fuel_awesome_plugin-0.1.0]\\nname=Plugin fuel_awesome_plugin-0.1.0 repository\\nbaseurl=http => //10.20.0.2 => 8080/plugins/fuel_awesome_plugin-0.1.0/repositories/centos\\ngpgcheck=0"
@ -48,7 +48,7 @@ describe Astute::NailgunHooks do
"type" => "sync",
"fail_on_error" => false,
"diagnostic_name" => "sync-example-1.0",
"uids" => [1, 2],
"uids" => ['1', '2'],
"parameters" => {
"src" => "rsync://10.20.0.2/plugins/fuel_awesome_plugin-0.1.0/deployment_scripts/",
"dst" => "/etc/fuel/plugins/fuel_awesome_plugin-0.1.0/"
@ -62,7 +62,7 @@ describe Astute::NailgunHooks do
"type" => "shell",
"fail_on_error" => false,
"diagnostic_name" => "shell-example-1.0",
"uids" => [1,2,3],
"uids" => ['1','2','3'],
"parameters" => {
"cmd" => "./deploy.sh",
"cwd" => "/etc/fuel/plugins/fuel_awesome_plugin-0.1.0/",
@ -77,7 +77,7 @@ describe Astute::NailgunHooks do
"type" => "puppet",
"fail_on_error" => false,
"diagnostic_name" => "puppet-example-1.0",
"uids" => [1, 3],
"uids" => ['1', '3'],
"parameters" => {
"puppet_manifest" => "cinder_glusterfs.pp",
"puppet_modules" => "modules",
@ -112,7 +112,7 @@ describe Astute::NailgunHooks do
wrong_hook = [{
"priority" => 300,
"type" => "unknown",
"uids" => [1, 3],
"uids" => ['1', '3'],
"parameters" => {
"parameter" => "1"
}
@ -179,19 +179,19 @@ describe Astute::NailgunHooks do
ctx.expects(:report_and_update_status).with(
{'nodes' =>
[
{ 'uid' => 1,
{ 'uid' => '1',
'status' => 'error',
'error_type' => 'deploy',
'role' => 'hook',
'hook' => "shell-example-1.0"
},
{ 'uid' => 2,
{ 'uid' => '2',
'status' => 'error',
'error_type' => 'deploy',
'role' => 'hook',
'hook' => "shell-example-1.0"
},
{ 'uid' => 3,
{ 'uid' => '3',
'status' => 'error',
'error_type' => 'deploy',
'role' => 'hook',
@ -240,7 +240,7 @@ describe Astute::NailgunHooks do
hooks = Astute::NailgunHooks.new([shell_hook], ctx)
hooks.expects(:run_shell_command).once.with(
ctx,
[1,2,3],
['1','2','3'],
regexp_matches(/deploy/),
shell_hook['parameters']['timeout'],
shell_hook['parameters']['cwd']
@ -255,7 +255,7 @@ describe Astute::NailgunHooks do
hooks = Astute::NailgunHooks.new([shell_hook], ctx)
hooks.expects(:run_shell_command).once.with(
ctx,
[1,2,3],
['1','2','3'],
regexp_matches(/deploy/),
300,
shell_hook['parameters']['cwd']
@ -271,7 +271,7 @@ describe Astute::NailgunHooks do
hooks = Astute::NailgunHooks.new([shell_hook], ctx)
hooks.expects(:run_shell_command).once.with(
ctx,
[1, 2],
['1', '2'],
regexp_matches(/deploy/),
shell_hook['parameters']['timeout'],
shell_hook['parameters']['cwd']
@ -280,7 +280,7 @@ describe Astute::NailgunHooks do
hooks.expects(:run_shell_command).once.with(
ctx,
[3],
['3'],
regexp_matches(/deploy/),
shell_hook['parameters']['timeout'],
shell_hook['parameters']['cwd']
@ -347,7 +347,7 @@ describe Astute::NailgunHooks do
hooks.expects(:upload_file).once.with(
ctx,
[2, 3],
['2', '3'],
has_entries(
'content' => upload_file_hook['parameters']['data'],
'path' => upload_file_hook['parameters']['path']
@ -363,7 +363,7 @@ describe Astute::NailgunHooks do
hooks.expects(:upload_file).once.with(
ctx,
[2],
['2'],
has_entries(
'content' => upload_file_hook['parameters']['data'],
'path' => upload_file_hook['parameters']['path']
@ -372,7 +372,7 @@ describe Astute::NailgunHooks do
hooks.expects(:upload_file).once.with(
ctx,
[3],
['3'],
has_entries(
'content' => upload_file_hook['parameters']['data'],
'path' => upload_file_hook['parameters']['path']
@ -432,7 +432,7 @@ describe Astute::NailgunHooks do
hooks = Astute::NailgunHooks.new([sync_hook], ctx)
hooks.expects(:run_shell_command).once.with(
ctx,
[1,2],
['1','2'],
regexp_matches(/deploy/),
sync_hook['parameters']['timeout']
)
@ -446,7 +446,7 @@ describe Astute::NailgunHooks do
hooks = Astute::NailgunHooks.new([sync_hook], ctx)
hooks.expects(:run_shell_command).once.with(
ctx,
[1,2],
['1','2'],
regexp_matches(/rsync/),
300
)
@ -461,7 +461,7 @@ describe Astute::NailgunHooks do
hooks = Astute::NailgunHooks.new([sync_hook], ctx)
hooks.expects(:run_shell_command).once.with(
ctx,
[1],
['1'],
regexp_matches(/rsync/),
is_a(Integer)
)
@ -469,7 +469,7 @@ describe Astute::NailgunHooks do
hooks.expects(:run_shell_command).once.with(
ctx,
[2],
['2'],
regexp_matches(/rsync/),
is_a(Integer)
)
@ -530,16 +530,42 @@ describe Astute::NailgunHooks do
expect {hooks.process}.to raise_error(StandardError, /Missing a required parameter/)
end
it 'should run puppet command with timeout' do
it 'should validate presence of cwd parameter' do
puppet_hook['parameters'].delete('cwd')
hooks = Astute::NailgunHooks.new([puppet_hook], ctx)
hooks.expects(:run_shell_command).once.with(
ctx,
[1,3],
regexp_matches(/puppet/),
puppet_hook['parameters']['timeout'],
expect {hooks.process}.to raise_error(StandardError, /Missing a required parameter/)
end
it 'should run puppet command using main mechanism' do
hooks = Astute::NailgunHooks.new([puppet_hook], ctx)
PuppetdDeployer.expects(:deploy).once.with(
instance_of(Astute::Context),
[
{'uid' => '1', 'role' => 'hook'},
{'uid' => '3', 'role' => 'hook'}
],
retries=2,
puppet_hook['parameters']['puppet_manifest'],
puppet_hook['parameters']['puppet_modules'],
puppet_hook['parameters']['cwd']
)
.returns(:data => {:exit_code => 0})
Astute::Context.any_instance.stubs(:status).returns({'1' => 'success', '3' => 'success'})
hooks.process
end
it 'should run puppet command with timeout' do
hooks = Astute::NailgunHooks.new([puppet_hook], ctx)
hooks.expects(:run_puppet).once.with(
ctx,
['1','3'],
puppet_hook['parameters']['puppet_manifest'],
puppet_hook['parameters']['puppet_modules'],
puppet_hook['parameters']['cwd'],
puppet_hook['parameters']['timeout']
).returns(true)
Astute::Context.any_instance.stubs(:status).returns({'1' => 'success', '3' => 'success'})
hooks.process
end
@ -547,14 +573,15 @@ describe Astute::NailgunHooks do
it 'should use default timeout if it does not set' do
puppet_hook['parameters'].delete('timeout')
hooks = Astute::NailgunHooks.new([puppet_hook], ctx)
hooks.expects(:run_shell_command).once.with(
hooks.expects(:run_puppet).once.with(
ctx,
[1,3],
regexp_matches(/puppet/),
300,
puppet_hook['parameters']['cwd']
)
.returns(:data => {:exit_code => 0})
['1','3'],
puppet_hook['parameters']['puppet_manifest'],
puppet_hook['parameters']['puppet_modules'],
puppet_hook['parameters']['cwd'],
300
).returns(true)
Astute::Context.any_instance.stubs(:status).returns({'1' => 'success', '3' => 'success'})
hooks.process
end
@ -563,50 +590,60 @@ describe Astute::NailgunHooks do
Astute.config.MAX_NODES_PER_CALL = 1
hooks = Astute::NailgunHooks.new([puppet_hook], ctx)
hooks.expects(:run_shell_command).once.with(
hooks.expects(:run_puppet).once.with(
ctx,
[1],
regexp_matches(/puppet/),
puppet_hook['parameters']['timeout'],
puppet_hook['parameters']['cwd']
)
.returns(:data => {:exit_code => 0})
['1'],
puppet_hook['parameters']['puppet_manifest'],
puppet_hook['parameters']['puppet_modules'],
puppet_hook['parameters']['cwd'],
puppet_hook['parameters']['timeout']
).returns(true)
hooks.expects(:run_shell_command).once.with(
hooks.expects(:run_puppet).once.with(
ctx,
[3],
regexp_matches(/puppet/),
puppet_hook['parameters']['timeout'],
puppet_hook['parameters']['cwd']
)
.returns(:data => {:exit_code => 0})
['3'],
puppet_hook['parameters']['puppet_manifest'],
puppet_hook['parameters']['puppet_modules'],
puppet_hook['parameters']['cwd'],
puppet_hook['parameters']['timeout']
).returns(true)
Astute::Context.any_instance.stubs(:status).returns({'1' => 'success', '3' => 'success'})
hooks.process
end
it 'if mclient failed and task is not critical -> do not raise error' do
hooks = Astute::NailgunHooks.new([puppet_hook], ctx)
PuppetdDeployer.expects(:deploy).once.raises(Astute::MClientError)
expect {hooks.process}.to_not raise_error
end
context 'process data from mcagent in case of critical hook' do
before(:each) do
puppet_hook['fail_on_error'] = true
ctx.stubs(:report_and_update_status)
end
it 'if exit code eql 0 -> do not raise error' do
it 'if puppet success do not raise error' do
hooks = Astute::NailgunHooks.new([puppet_hook], ctx)
hooks.expects(:run_shell_command).returns({:data => {:exit_code => 0}}).once
PuppetdDeployer.expects(:deploy).once
Astute::Context.any_instance.stubs(:status).returns({'1' => 'success', '3' => 'success'})
expect {hooks.process}.to_not raise_error
end
it 'if exit code not eql 0 -> raise error' do
it 'if puppet fail -> raise error' do
hooks = Astute::NailgunHooks.new([puppet_hook], ctx)
hooks.expects(:run_shell_command).returns({:data => {:exit_code => 1}}).once
PuppetdDeployer.expects(:deploy).once
Astute::Context.any_instance.stubs(:status).returns({'1' => 'error', '3' => 'success'})
expect {hooks.process}.to raise_error(Astute::DeploymentEngineError, /Failed to deploy plugin/)
end
it 'if exit code not presence -> raise error' do
it 'if mclient failed -> raise error' do
hooks = Astute::NailgunHooks.new([puppet_hook], ctx)
hooks.expects(:run_shell_command).returns({:data => {}}).once
PuppetdDeployer.expects(:deploy).once.raises(Astute::MClientError)
expect {hooks.process}.to raise_error(Astute::DeploymentEngineError, /Failed to deploy plugin/)
end