diff --git a/sahara/service/heat/heat_engine.py b/sahara/service/heat/heat_engine.py index 444e7c4320..fe7c377d45 100644 --- a/sahara/service/heat/heat_engine.py +++ b/sahara/service/heat/heat_engine.py @@ -203,7 +203,9 @@ class HeatEngine(e.Engine): self._update_instance_count(stack, cluster, target_count) stack.instantiate(update_existing=update_stack, disable_rollback=disable_rollback) - heat.wait_stack_completion(stack.heat_stack) + heat.wait_stack_completion( + stack.heat_stack, + is_update=update_stack, last_updated_time=stack.last_updated_time) return self._populate_cluster(cluster, stack) def _launch_instances(self, cluster, target_count, stages, diff --git a/sahara/service/heat/templates.py b/sahara/service/heat/templates.py index eae24fecc9..d33ba292cb 100644 --- a/sahara/service/heat/templates.py +++ b/sahara/service/heat/templates.py @@ -105,6 +105,7 @@ class ClusterStack(object): self.node_groups_extra = {} self.heat_stack = None self.files = {} + self.last_updated_time = None def add_node_group_extra(self, node_group_id, node_count, gen_userdata_func): @@ -145,6 +146,7 @@ class ClusterStack(object): b.execute_with_retries(heat.stacks.create, **kwargs) else: stack = h.get_stack(self.cluster.name) + self.last_updated_time = stack.updated_time LOG.debug("Updating Heat stack {stack} with args: " "{args}".format(stack=stack, args=kwargs)) b.execute_with_retries(stack.update, **kwargs) diff --git a/sahara/tests/unit/utils/test_heat.py b/sahara/tests/unit/utils/test_heat.py index 5d11b54c10..970aefb70d 100644 --- a/sahara/tests/unit/utils/test_heat.py +++ b/sahara/tests/unit/utils/test_heat.py @@ -23,16 +23,25 @@ from sahara.utils.openstack import heat as h class TestClusterStack(testtools.TestCase): @mock.patch("sahara.context.sleep", return_value=None) def test_wait_completion(self, _): - stack = FakeHeatStack('CREATE_IN_PROGRESS', 'CREATE_COMPLETE') + stack = FakeHeatStack('CREATE_IN_PROGRESS', ['CREATE_COMPLETE']) h.wait_stack_completion(stack) - stack = FakeHeatStack('UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE') + stack = FakeHeatStack('UPDATE_IN_PROGRESS', ['UPDATE_COMPLETE']) h.wait_stack_completion(stack) - stack = FakeHeatStack('DELETE_IN_PROGRESS', 'DELETE_COMPLETE') + stack = FakeHeatStack('DELETE_IN_PROGRESS', ['DELETE_COMPLETE']) h.wait_stack_completion(stack) - stack = FakeHeatStack('CREATE_IN_PROGRESS', 'CREATE_FAILED') + stack = FakeHeatStack('CREATE_COMPLETE', [ + 'CREATE_COMPLETE', 'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE']) + h.wait_stack_completion(stack, is_update=True) + + stack = FakeHeatStack('UPDATE_COMPLETE', [ + 'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE'], + updated_time=-3) + h.wait_stack_completion(stack, is_update=True) + + stack = FakeHeatStack('CREATE_IN_PROGRESS', ['CREATE_FAILED']) with testtools.ExpectedException( ex.HeatStackException, value_re=("Heat stack failed with status " @@ -41,13 +50,19 @@ class TestClusterStack(testtools.TestCase): class FakeHeatStack(object): - def __init__(self, stack_status=None, new_status=None, stack_name=None): + def __init__(self, stack_status=None, new_statuses=None, stack_name=None, + updated_time=None): self.stack_status = stack_status or '' - self.new_status = new_status or '' + self.new_statuses = new_statuses or [] + self.idx = 0 self.stack_name = stack_name or '' + self.updated_time = updated_time def get(self): - self.stack_status = self.new_status + self.stack_status = self.new_statuses[self.idx] + self.idx += 1 + if self.idx > 0 and self.stack_status == 'UPDATE_COMPLETE': + self.updated_time = self.idx @property def status(self): diff --git a/sahara/utils/openstack/heat.py b/sahara/utils/openstack/heat.py index 09f2511ea7..b4bfcc9036 100644 --- a/sahara/utils/openstack/heat.py +++ b/sahara/utils/openstack/heat.py @@ -66,10 +66,20 @@ def get_stack(stack_name, raise_on_missing=True): _('Failed to find stack %(stack)s')) -def wait_stack_completion(stack): +def _verify_completion(stack, is_update=False, last_update_time=None): # NOTE: expected empty status because status of stack # maybe is not set in heat database - while stack.status in ['IN_PROGRESS', '']: + if stack.status in ['IN_PROGRESS', '']: + return False + if is_update and stack.status == 'COMPLETE': + if stack.updated_time == last_update_time: + return False + return True + + +def wait_stack_completion(stack, is_update=False, last_updated_time=None): + base.execute_with_retries(stack.get) + while not _verify_completion(stack, is_update, last_updated_time): context.sleep(1) base.execute_with_retries(stack.get)