Merge "Resolve issue with operating heat stack outputs"

sahara/service/heat/heat_engine.py

@@ -203,7 +203,9 @@ class HeatEngine(e.Engine):
         self._update_instance_count(stack, cluster, target_count)
         stack.instantiate(update_existing=update_stack,
                           disable_rollback=disable_rollback)
-        heat.wait_stack_completion(stack.heat_stack)
+        heat.wait_stack_completion(
+            stack.heat_stack,
+            is_update=update_stack, last_updated_time=stack.last_updated_time)
         return self._populate_cluster(cluster, stack)
 
     def _launch_instances(self, cluster, target_count, stages,

sahara/service/heat/templates.py

@@ -105,6 +105,7 @@ class ClusterStack(object):
         self.node_groups_extra = {}
         self.heat_stack = None
         self.files = {}
+        self.last_updated_time = None
 
     def add_node_group_extra(self, node_group_id, node_count,
                              gen_userdata_func):
@@ -145,6 +146,7 @@ class ClusterStack(object):
             b.execute_with_retries(heat.stacks.create, **kwargs)
         else:
             stack = h.get_stack(self.cluster.name)
+            self.last_updated_time = stack.updated_time
             LOG.debug("Updating Heat stack {stack} with args: "
                       "{args}".format(stack=stack, args=kwargs))
             b.execute_with_retries(stack.update, **kwargs)

sahara/tests/unit/utils/test_heat.py

@@ -23,16 +23,25 @@ from sahara.utils.openstack import heat as h
 class TestClusterStack(testtools.TestCase):
     @mock.patch("sahara.context.sleep", return_value=None)
     def test_wait_completion(self, _):
-        stack = FakeHeatStack('CREATE_IN_PROGRESS', 'CREATE_COMPLETE')
+        stack = FakeHeatStack('CREATE_IN_PROGRESS', ['CREATE_COMPLETE'])
         h.wait_stack_completion(stack)
 
-        stack = FakeHeatStack('UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE')
+        stack = FakeHeatStack('UPDATE_IN_PROGRESS', ['UPDATE_COMPLETE'])
         h.wait_stack_completion(stack)
 
-        stack = FakeHeatStack('DELETE_IN_PROGRESS', 'DELETE_COMPLETE')
+        stack = FakeHeatStack('DELETE_IN_PROGRESS', ['DELETE_COMPLETE'])
         h.wait_stack_completion(stack)
 
-        stack = FakeHeatStack('CREATE_IN_PROGRESS', 'CREATE_FAILED')
+        stack = FakeHeatStack('CREATE_COMPLETE', [
+            'CREATE_COMPLETE', 'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE'])
+        h.wait_stack_completion(stack, is_update=True)
+
+        stack = FakeHeatStack('UPDATE_COMPLETE', [
+            'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE'],
+            updated_time=-3)
+        h.wait_stack_completion(stack, is_update=True)
+
+        stack = FakeHeatStack('CREATE_IN_PROGRESS', ['CREATE_FAILED'])
         with testtools.ExpectedException(
                 ex.HeatStackException,
                 value_re=("Heat stack failed with status "
@@ -41,13 +50,19 @@ class TestClusterStack(testtools.TestCase):
 
 
 class FakeHeatStack(object):
-    def __init__(self, stack_status=None, new_status=None, stack_name=None):
+    def __init__(self, stack_status=None, new_statuses=None, stack_name=None,
+                 updated_time=None):
         self.stack_status = stack_status or ''
-        self.new_status = new_status or ''
+        self.new_statuses = new_statuses or []
+        self.idx = 0
         self.stack_name = stack_name or ''
+        self.updated_time = updated_time
 
     def get(self):
-        self.stack_status = self.new_status
+        self.stack_status = self.new_statuses[self.idx]
+        self.idx += 1
+        if self.idx > 0 and self.stack_status == 'UPDATE_COMPLETE':
+            self.updated_time = self.idx
 
     @property
     def status(self):

sahara/utils/openstack/heat.py

@@ -66,10 +66,20 @@ def get_stack(stack_name, raise_on_missing=True):
                                _('Failed to find stack %(stack)s'))
 
 
-def wait_stack_completion(stack):
+def _verify_completion(stack, is_update=False, last_update_time=None):
     # NOTE: expected empty status because status of stack
     # maybe is not set in heat database
-    while stack.status in ['IN_PROGRESS', '']:
+    if stack.status in ['IN_PROGRESS', '']:
+        return False
+    if is_update and stack.status == 'COMPLETE':
+        if stack.updated_time == last_update_time:
+            return False
+    return True
+
+
+def wait_stack_completion(stack, is_update=False, last_updated_time=None):
+    base.execute_with_retries(stack.get)
+    while not _verify_completion(stack, is_update, last_updated_time):
         context.sleep(1)
         base.execute_with_retries(stack.get)