Fail tolerance behavior for upload file tasks

Additional changes:

- decrease number of "reset undefined retries" messages;
- rewriting log messages for better understanding.

Change-Id: I17db392ac4c73a3c08505fcbaf17dbcce96ebd91
Blueprint: graph-concept-extension
This commit is contained in:
Vladimir Sharshov (warpc) 2017-02-28 04:50:19 +03:00
parent c489e972ff
commit 48ee1f7467
4 changed files with 30 additions and 11 deletions

View File

@ -41,7 +41,7 @@ module Astute
}
else
Astute.logger.warn("#{@ctx.task_id}: Failed to run shell "\
"#{details_for_log(cmd, timeout)} Error will not raise "\
"#{details_for_log(cmd, timeout)}. Error will not raise "\
"because shell was run without check")
{}
end

View File

@ -15,6 +15,7 @@
module Astute
class UploadFileMClient
attr_reader :ctx, :node_id
def initialize(ctx, node_id)
@ctx = ctx
@node_id = node_id
@ -36,11 +37,13 @@ module Astute
# @return [true, false] upload result
def upload_with_check(mco_params)
upload_mclient = upload_mclient(
:check_result => true,
:check_result => false,
:timeout => mco_params['timeout'],
:retries => mco_params['retries']
)
upload(mco_params, upload_mclient)
process_with_retries(:retries => mco_params['retries']) do
upload(mco_params, upload_mclient)
end
end
private
@ -61,17 +64,17 @@ module Astute
)
if results.present? && results.first[:statuscode] == 0
Astute.logger.debug("#{@ctx.task_id}: file was uploaded "\
Astute.logger.debug("#{ctx.task_id}: file was uploaded "\
"#{details_for_log(mco_params)} successfully")
true
else
Astute.logger.error("#{@ctx.task_id}: file was not uploaded "\
Astute.logger.error("#{ctx.task_id}: file was not uploaded "\
"#{details_for_log(mco_params)}: "\
"#{results.first[:msg] if results.present? }")
"#{results.present? ? results.first[:msg] : "node has not answered" }")
false
end
rescue MClientTimeout, MClientError => e
Astute.logger.error("#{@ctx.task_id}: file was not uploaded "\
Astute.logger.error("#{ctx.task_id}: file was not uploaded "\
"#{details_for_log(mco_params)}: #{e.message}")
false
end
@ -80,9 +83,9 @@ module Astute
# @return [Astute::MClient]
def upload_mclient(args={})
MClient.new(
@ctx,
ctx,
"uploadfile",
[@node_id],
[node_id],
args.fetch(:check_result, false),
args.fetch(:timeout, 2),
args.fetch(:retries, Astute.config.upload_retries)
@ -108,9 +111,23 @@ module Astute
# Return short useful info about node and shell task
# @return [String] detail info about upload task
def details_for_log(mco_params)
"#{mco_params['path']} on node #{@node_id} "\
"#{mco_params['path']} on node #{node_id} "\
"with timeout #{mco_params['timeout']}"
end
def process_with_retries(args={}, &block)
retries = args.fetch(:retries, 1) + 1
result = false
retries.times do |attempt|
result = block.call
break if result
Astute.logger.warn("#{ctx.task_id} Upload retry for node "\
"#{node_id}: attempt № #{attempt + 1}/#{retries}")
end
result
end
end
end

View File

@ -270,6 +270,8 @@ module Astute
# Reset undefined retries to original value
# @return [void]
def reset_undefined_retries!
return if @undefined_retries == @original_undefined_retries
Astute.logger.debug "Reset undefined retries to original "\
"value: #{@original_undefined_retries}"
@undefined_retries = @original_undefined_retries

View File

@ -47,7 +47,7 @@ module Astute
}]
})
else
info "Finished task: #{task} with status: #{status}"
info "Finished task #{task} #{"with status: #{task.status}" if task}"
setup_node_status
report_node_status
end