Resolve issue with operating heat stack outputs
Sometimes, status of heat stack is not changed in DB right after executing updating of heat stack. Because of that sahara thinks that heat stack was successfully updated, but it's wrong, because heat stack updating still in progress. Because of that it's proposed to use field updated_time for verification of successful update of heat stack. We will store that field in variable, and will wait until that will be changed. Closes-bug: 1472536 Change-Id: Ifeee9a81ff1b37ff061de0ded371ae6f6bda4c95
This commit is contained in:
parent
2759be3127
commit
b4e5e12f7b
|
@ -203,7 +203,9 @@ class HeatEngine(e.Engine):
|
||||||
self._update_instance_count(stack, cluster, target_count)
|
self._update_instance_count(stack, cluster, target_count)
|
||||||
stack.instantiate(update_existing=update_stack,
|
stack.instantiate(update_existing=update_stack,
|
||||||
disable_rollback=disable_rollback)
|
disable_rollback=disable_rollback)
|
||||||
heat.wait_stack_completion(stack.heat_stack)
|
heat.wait_stack_completion(
|
||||||
|
stack.heat_stack,
|
||||||
|
is_update=update_stack, last_updated_time=stack.last_updated_time)
|
||||||
return self._populate_cluster(cluster, stack)
|
return self._populate_cluster(cluster, stack)
|
||||||
|
|
||||||
def _launch_instances(self, cluster, target_count, stages,
|
def _launch_instances(self, cluster, target_count, stages,
|
||||||
|
|
|
@ -95,6 +95,7 @@ class ClusterStack(object):
|
||||||
self.node_groups_extra = {}
|
self.node_groups_extra = {}
|
||||||
self.heat_stack = None
|
self.heat_stack = None
|
||||||
self.files = {}
|
self.files = {}
|
||||||
|
self.last_updated_time = None
|
||||||
|
|
||||||
def add_node_group_extra(self, node_group_id, node_count,
|
def add_node_group_extra(self, node_group_id, node_count,
|
||||||
gen_userdata_func):
|
gen_userdata_func):
|
||||||
|
@ -135,6 +136,7 @@ class ClusterStack(object):
|
||||||
b.execute_with_retries(heat.stacks.create, **kwargs)
|
b.execute_with_retries(heat.stacks.create, **kwargs)
|
||||||
else:
|
else:
|
||||||
stack = h.get_stack(self.cluster.name)
|
stack = h.get_stack(self.cluster.name)
|
||||||
|
self.last_updated_time = stack.updated_time
|
||||||
LOG.debug("Updating Heat stack {stack} with args: "
|
LOG.debug("Updating Heat stack {stack} with args: "
|
||||||
"{args}".format(stack=stack, args=kwargs))
|
"{args}".format(stack=stack, args=kwargs))
|
||||||
b.execute_with_retries(stack.update, **kwargs)
|
b.execute_with_retries(stack.update, **kwargs)
|
||||||
|
|
|
@ -23,16 +23,25 @@ from sahara.utils.openstack import heat as h
|
||||||
class TestClusterStack(testtools.TestCase):
|
class TestClusterStack(testtools.TestCase):
|
||||||
@mock.patch("sahara.context.sleep", return_value=None)
|
@mock.patch("sahara.context.sleep", return_value=None)
|
||||||
def test_wait_completion(self, _):
|
def test_wait_completion(self, _):
|
||||||
stack = FakeHeatStack('CREATE_IN_PROGRESS', 'CREATE_COMPLETE')
|
stack = FakeHeatStack('CREATE_IN_PROGRESS', ['CREATE_COMPLETE'])
|
||||||
h.wait_stack_completion(stack)
|
h.wait_stack_completion(stack)
|
||||||
|
|
||||||
stack = FakeHeatStack('UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE')
|
stack = FakeHeatStack('UPDATE_IN_PROGRESS', ['UPDATE_COMPLETE'])
|
||||||
h.wait_stack_completion(stack)
|
h.wait_stack_completion(stack)
|
||||||
|
|
||||||
stack = FakeHeatStack('DELETE_IN_PROGRESS', 'DELETE_COMPLETE')
|
stack = FakeHeatStack('DELETE_IN_PROGRESS', ['DELETE_COMPLETE'])
|
||||||
h.wait_stack_completion(stack)
|
h.wait_stack_completion(stack)
|
||||||
|
|
||||||
stack = FakeHeatStack('CREATE_IN_PROGRESS', 'CREATE_FAILED')
|
stack = FakeHeatStack('CREATE_COMPLETE', [
|
||||||
|
'CREATE_COMPLETE', 'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE'])
|
||||||
|
h.wait_stack_completion(stack, is_update=True)
|
||||||
|
|
||||||
|
stack = FakeHeatStack('UPDATE_COMPLETE', [
|
||||||
|
'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE'],
|
||||||
|
updated_time=-3)
|
||||||
|
h.wait_stack_completion(stack, is_update=True)
|
||||||
|
|
||||||
|
stack = FakeHeatStack('CREATE_IN_PROGRESS', ['CREATE_FAILED'])
|
||||||
with testtools.ExpectedException(
|
with testtools.ExpectedException(
|
||||||
ex.HeatStackException,
|
ex.HeatStackException,
|
||||||
value_re=("Heat stack failed with status "
|
value_re=("Heat stack failed with status "
|
||||||
|
@ -41,13 +50,19 @@ class TestClusterStack(testtools.TestCase):
|
||||||
|
|
||||||
|
|
||||||
class FakeHeatStack(object):
|
class FakeHeatStack(object):
|
||||||
def __init__(self, stack_status=None, new_status=None, stack_name=None):
|
def __init__(self, stack_status=None, new_statuses=None, stack_name=None,
|
||||||
|
updated_time=None):
|
||||||
self.stack_status = stack_status or ''
|
self.stack_status = stack_status or ''
|
||||||
self.new_status = new_status or ''
|
self.new_statuses = new_statuses or []
|
||||||
|
self.idx = 0
|
||||||
self.stack_name = stack_name or ''
|
self.stack_name = stack_name or ''
|
||||||
|
self.updated_time = updated_time
|
||||||
|
|
||||||
def get(self):
|
def get(self):
|
||||||
self.stack_status = self.new_status
|
self.stack_status = self.new_statuses[self.idx]
|
||||||
|
self.idx += 1
|
||||||
|
if self.idx > 0 and self.stack_status == 'UPDATE_COMPLETE':
|
||||||
|
self.updated_time = self.idx
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def status(self):
|
def status(self):
|
||||||
|
|
|
@ -66,10 +66,20 @@ def get_stack(stack_name, raise_on_missing=True):
|
||||||
_('Failed to find stack %(stack)s'))
|
_('Failed to find stack %(stack)s'))
|
||||||
|
|
||||||
|
|
||||||
def wait_stack_completion(stack):
|
def _verify_completion(stack, is_update=False, last_update_time=None):
|
||||||
# NOTE: expected empty status because status of stack
|
# NOTE: expected empty status because status of stack
|
||||||
# maybe is not set in heat database
|
# maybe is not set in heat database
|
||||||
while stack.status in ['IN_PROGRESS', '']:
|
if stack.status in ['IN_PROGRESS', '']:
|
||||||
|
return False
|
||||||
|
if is_update and stack.status == 'COMPLETE':
|
||||||
|
if stack.updated_time == last_update_time:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def wait_stack_completion(stack, is_update=False, last_updated_time=None):
|
||||||
|
base.execute_with_retries(stack.get)
|
||||||
|
while not _verify_completion(stack, is_update, last_updated_time):
|
||||||
context.sleep(1)
|
context.sleep(1)
|
||||||
base.execute_with_retries(stack.get)
|
base.execute_with_retries(stack.get)
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue