Adding retry ability to heatclient calls

All heatclient calls wrapped in execute_with_retry
method to avoid occasional errors

partially implements bp clients-calls-retry

Change-Id: I835fd79b5783f4a5271cec67b013ba8aeba3091a

sahara/service/heat/heat_engine.py

@@ -26,6 +26,7 @@ from sahara.service.heat import templates as ht
 from sahara.service import volumes
 from sahara.utils import cluster_progress_ops as cpo
 from sahara.utils import general as g
+from sahara.utils.openstack import base as b
 from sahara.utils.openstack import heat
 
 conductor = c.API
@@ -39,10 +40,10 @@ class HeatEngine(e.Engine):
 
     def _add_volumes(self, ctx, cluster):
         for instance in g.get_instances(cluster):
-            res_names = heat.client().resources.get(
+            res_names = heat.get_resource(
                 cluster.name, instance.instance_name).required_by
             for res_name in res_names:
-                vol_res = heat.client().resources.get(cluster.name, res_name)
+                vol_res = heat.get_resource(cluster.name, res_name)
                 if vol_res.resource_type == (('OS::Cinder::'
                                               'VolumeAttachment')):
                     volume_id = vol_res.physical_resource_id
@@ -177,7 +178,7 @@ class HeatEngine(e.Engine):
     def shutdown_cluster(self, cluster):
         """Shutdown specified cluster and all related resources."""
         try:
-            heat.client().stacks.delete(cluster.name)
+            b.execute_with_retries(heat.client().stacks.delete, cluster.name)
             stack = heat.get_stack(cluster.name)
             heat.wait_stack_completion(stack)
         except heat_exc.HTTPNotFound:

sahara/service/heat/templates.py

@@ -20,6 +20,7 @@ import six
 import yaml
 
 from sahara.utils import general as g
+from sahara.utils.openstack import base as b
 from sahara.utils.openstack import heat as h
 from sahara.utils.openstack import neutron
 
@@ -91,11 +92,11 @@ class ClusterTemplate(object):
             'template': main_tmpl}
 
         if not update_existing:
-            heat.stacks.create(**kwargs)
+            b.execute_with_retries(heat.stacks.create, **kwargs)
         else:
-            for stack in heat.stacks.list():
+            for stack in b.execute_with_retries(heat.stacks.list):
                 if stack.stack_name == self.cluster.name:
-                    stack.update(**kwargs)
+                    b.execute_with_retries(stack.update, **kwargs)
                     break
 
         return ClusterStack(self, h.get_stack(self.cluster.name))
@@ -337,10 +338,9 @@ class ClusterStack(object):
 
         count = self.tmpl.node_groups_extra[node_group.id]['node_count']
 
-        heat = h.client()
         for i in range(0, count):
             name = _get_inst_name(self.tmpl.cluster.name, node_group.name, i)
-            res = heat.resources.get(self.heat_stack.id, name)
+            res = h.get_resource(self.heat_stack.id, name)
             insts.append((name, res.physical_resource_id))
 
         return insts

sahara/utils/openstack/heat.py

@@ -49,7 +49,7 @@ def client():
 
 def get_stack(stack_name):
     heat = client()
-    for stack in heat.stacks.list():
+    for stack in base.execute_with_retries(heat.stacks.list):
         if stack.stack_name == stack_name:
             return stack
 
@@ -62,7 +62,12 @@ def wait_stack_completion(stack):
     # maybe is not set in heat database
     while stack.status in ['IN_PROGRESS', '']:
         context.sleep(1)
-        stack.get()
+        base.execute_with_retries(stack.get)
 
     if stack.status != 'COMPLETE':
         raise ex.HeatStackException(stack.stack_status)
+
+
+def get_resource(stack, resource):
+    return base.execute_with_retries(
+        client().resources.get, stack, resource)