Allow to specify number of retries for puppet task

Now it's possible to specify a number of retries for
puppet task in granural deployment by specifying a new
parameter "retries":

  parameters:
    ...
    retries: 10

By default, there are 2 retries.

Change-Id: I819a019cf488fc1bd0d52b35d1693fcc088125cb
Partial-Bug: #1457794

lib/astute/config.rb

@@ -58,6 +58,7 @@ module Astute
     conf[:puppet_timeout] = 90 * 60       # maximum time it waits for single puppet run
     conf[:puppet_deploy_interval] = 2     # sleep for ## sec, then check puppet status again
     conf[:puppet_fade_timeout] = 120      # how long it can take for puppet to exit after dumping to last_run_summary
+    conf[:puppet_retries] = 2             # how many times astute will try to run puppet
     conf[:mc_retries] = 10                # MClient tries to call mcagent before failure
     conf[:mc_retry_interval] = 1          # MClient sleeps for ## sec between retries
     conf[:puppet_fade_interval] = 30      # retry every ## seconds to check puppet state if it was running

lib/astute/deployment_engine/granular_deployment.rb

@@ -64,7 +64,7 @@ class Astute::DeploymentEngine::GranularDeployment < Astute::DeploymentEngine
     Astute::PuppetTask.new(
       @hook_context,
       @nodes_by_uid[node_id], # Use single node uid instead of task['uids']
-      retries=2,
+      retries=task['parameters']['retries'] || Astute.config.puppet_retries,
       task['parameters']['puppet_manifest'],
       task['parameters']['puppet_modules'],
       task['parameters']['cwd'],

lib/astute/nailgun_hooks.rb

@@ -98,6 +98,7 @@ module Astute
       validate_presence(hook['parameters'], 'cwd')
 
       timeout = hook['parameters']['timeout'] || 300
+      retries = hook['parameters']['retries'] || Astute.config.puppet_retries
 
       ret = {'error' => nil}
       perform_with_limit(hook['uids']) do |node_uids|
@@ -107,7 +108,8 @@ module Astute
           hook['parameters']['puppet_manifest'],
           hook['parameters']['puppet_modules'],
           hook['parameters']['cwd'],
-          timeout
+          timeout,
+          retries
         )
         unless result
           ret['error'] = "Puppet run failed. Check puppet logs for details"
@@ -278,7 +280,7 @@ module Astute
       raise "Missing a required parameter #{key}" unless data[key].present?
     end
 
-    def run_puppet(context, node_uids, puppet_manifest, puppet_modules, cwd, timeout)
+    def run_puppet(context, node_uids, puppet_manifest, puppet_modules, cwd, timeout, retries)
       # Prevent send report status to Nailgun
       hook_context = Context.new(context.task_id, HookReporter.new, LogParser::NoParsing.new)
       nodes = node_uids.map { |node_id| {'uid' => node_id.to_s, 'role' => 'hook'} }
@@ -287,7 +289,7 @@ module Astute
         PuppetdDeployer.deploy(
           hook_context,
           nodes,
-          retries=2,
+          retries=retries,
           puppet_manifest,
           puppet_modules,
           cwd

spec/unit/granular_deployment_spec.rb

@@ -88,7 +88,8 @@ describe "Granular deployment engine" do
         "puppet_manifest" =>  "cinder_glusterfs.pp",
         "puppet_modules" =>  "modules",
         "cwd" => "/etc/fuel/plugins/plugin_name-1.0",
-        "timeout" =>  42
+        "timeout" =>  42,
+        "retries" => 123
       }
     }
   end
@@ -378,4 +379,19 @@ describe "Granular deployment engine" do
 
   end #deploy_nodes
 
+  describe '#puppet_hook' do
+    it 'should use retries from task parameters' do
+      deploy_engine.instance_variable_set(:@nodes_by_uid, {})
+      task = deploy_engine.puppet_task(1, puppet_hook)
+      expect(task.instance_variable_get(:@retries)).to eql(puppet_hook['parameters']['retries'])
+    end
+
+    it 'should use default retries if not provided' do
+      puppet_hook['parameters'].delete('retries')
+      deploy_engine.instance_variable_set(:@nodes_by_uid, {})
+      task = deploy_engine.puppet_task(1, puppet_hook)
+      expect(task.instance_variable_get(:@retries)).to eql(Astute.config.puppet_retries)
+    end
+  end
+
 end # 'describe'

spec/unit/nailgun_hooks_spec.rb

@@ -84,7 +84,8 @@ describe Astute::NailgunHooks do
         "puppet_manifest" =>  "cinder_glusterfs.pp",
         "puppet_modules" =>  "modules",
         "cwd" => "/etc/fuel/plugins/plugin_name-1.0",
-        "timeout" =>  42
+        "timeout" =>  42,
+        "retries" => 21
       }
     }
   end
@@ -911,7 +912,7 @@ describe Astute::NailgunHooks do
           {'uid' => '1', 'role' => 'hook'},
           {'uid' => '3', 'role' => 'hook'}
         ],
-        retries=2,
+        retries=puppet_hook['parameters']['retries'],
         puppet_hook['parameters']['puppet_manifest'],
         puppet_hook['parameters']['puppet_modules'],
         puppet_hook['parameters']['cwd']
@@ -929,7 +930,8 @@ describe Astute::NailgunHooks do
         puppet_hook['parameters']['puppet_manifest'],
         puppet_hook['parameters']['puppet_modules'],
         puppet_hook['parameters']['cwd'],
-        puppet_hook['parameters']['timeout']
+        puppet_hook['parameters']['timeout'],
+        puppet_hook['parameters']['retries']
       ).returns(true)
       Astute::Context.any_instance.stubs(:status).returns({'1' => 'success', '3' => 'success'})
 
@@ -945,7 +947,25 @@ describe Astute::NailgunHooks do
         puppet_hook['parameters']['puppet_manifest'],
         puppet_hook['parameters']['puppet_modules'],
         puppet_hook['parameters']['cwd'],
-        300
+        300,
+        puppet_hook['parameters']['retries']
+      ).returns(true)
+      Astute::Context.any_instance.stubs(:status).returns({'1' => 'success', '3' => 'success'})
+
+      hooks.process
+    end
+
+    it 'should use default retries if it does not set' do
+      puppet_hook['parameters'].delete('retries')
+      hooks = Astute::NailgunHooks.new([puppet_hook], ctx)
+      hooks.expects(:run_puppet).once.with(
+        ctx,
+        ['1','3'],
+        puppet_hook['parameters']['puppet_manifest'],
+        puppet_hook['parameters']['puppet_modules'],
+        puppet_hook['parameters']['cwd'],
+        puppet_hook['parameters']['timeout'],
+        Astute.config.puppet_retries,
       ).returns(true)
       Astute::Context.any_instance.stubs(:status).returns({'1' => 'success', '3' => 'success'})
 
@@ -962,7 +982,8 @@ describe Astute::NailgunHooks do
         puppet_hook['parameters']['puppet_manifest'],
         puppet_hook['parameters']['puppet_modules'],
         puppet_hook['parameters']['cwd'],
-        puppet_hook['parameters']['timeout']
+        puppet_hook['parameters']['timeout'],
+        puppet_hook['parameters']['retries']
       ).returns(true)
 
       hooks.expects(:run_puppet).once.with(
@@ -971,7 +992,8 @@ describe Astute::NailgunHooks do
         puppet_hook['parameters']['puppet_manifest'],
         puppet_hook['parameters']['puppet_modules'],
         puppet_hook['parameters']['cwd'],
-        puppet_hook['parameters']['timeout']
+        puppet_hook['parameters']['timeout'],
+        puppet_hook['parameters']['retries']
       ).returns(true)
 
       Astute::Context.any_instance.stubs(:status).returns({'1' => 'success', '3' => 'success'})