openstack
/
heat
Code Issues Proposed changes

Allow an update after a failure 

Change-Id: I41ce08c33780642c31b81763032d6c089e903c48
Implements: blueprint update-failure-recovery
Closes-bug: #1160052
This commit is contained in:
Zane Bitter 2014-08-26 18:37:09 -04:00
parent f80e5793f6
commit c2ffae8cd0
2 changed files with 246 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
heat/engine/stack.py
View File

@ -666,9 +666,8 @@ class Stack(collections.Mapping):
"Invalid action %s" % action)
return
if self.status != self.COMPLETE:
if (action == self.ROLLBACK and
self.state == (self.UPDATE, self.IN_PROGRESS)):
if self.status == self.IN_PROGRESS:
if action == self.ROLLBACK:
LOG.debug("Starting update rollback for %s" % self.name)
else:
self.state_set(action, self.FAILED,

255
heat/tests/test_parser.py
View File

@ -1847,17 +1847,6 @@ class StackTest(HeatTestCase):
self.assertEqual((self.stack.ROLLBACK, self.stack.COMPLETE),
self.stack.state)
def test_update_badstate(self):
self.stack = parser.Stack(self.ctx, 'test_stack', self.tmpl,
action=parser.Stack.CREATE,
status=parser.Stack.FAILED)
self.stack.store()
self.assertEqual((parser.Stack.CREATE, parser.Stack.FAILED),
self.stack.state)
self.stack.update(mock.Mock())
self.assertEqual((parser.Stack.UPDATE, parser.Stack.FAILED),
self.stack.state)
def test_resource_by_refid(self):
tmpl = {'HeatTemplateFormatVersion': '2012-12-12',
'Resources': {'AResource': {'Type': 'GenericResourceType'}}}
@ -2949,6 +2938,250 @@ class StackTest(HeatTestCase):
self.m.VerifyAll()
def test_update_failure_recovery(self):
'''
assertion:
check that rollback still works with dynamic metadata
this test fails the second instance
'''
class ResourceTypeA(generic_rsrc.ResourceWithProps):
count = 0
def handle_create(self):
ResourceTypeA.count += 1
self.resource_id_set('%s%d' % (self.name, self.count))
def handle_delete(self):
return super(ResourceTypeA, self).handle_delete()
resource._register_class('ResourceTypeA', ResourceTypeA)
tmpl = {'HeatTemplateFormatVersion': '2012-12-12',
'Resources': {
'AResource': {'Type': 'ResourceTypeA',
'Properties': {'Foo': 'abc'}},
'BResource': {'Type': 'ResourceWithPropsType',
'Properties': {
'Foo': {'Ref': 'AResource'}}}}}
tmpl2 = {'HeatTemplateFormatVersion': '2012-12-12',
'Resources': {
'AResource': {'Type': 'ResourceTypeA',
'Properties': {'Foo': 'smelly'}},
'BResource': {'Type': 'ResourceWithPropsType',
'Properties': {
'Foo': {'Ref': 'AResource'}}}}}
self.stack = parser.Stack(self.ctx, 'update_test_stack',
template.Template(tmpl),
disable_rollback=True)
self.stack.store()
self.stack.create()
self.assertEqual((parser.Stack.CREATE, parser.Stack.COMPLETE),
self.stack.state)
self.assertEqual('abc', self.stack['AResource'].properties['Foo'])
self.assertEqual('AResource1',
self.stack['BResource'].properties['Foo'])
self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_create')
self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_delete')
self.m.StubOutWithMock(ResourceTypeA, 'handle_delete')
# mock to make the replace fail when creating the second
# replacement resource
generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception)
# delete the old resource on the second update
generic_rsrc.ResourceWithProps.handle_delete()
ResourceTypeA.handle_delete()
generic_rsrc.ResourceWithProps.handle_create()
generic_rsrc.ResourceWithProps.handle_delete()
self.m.ReplayAll()
updated_stack = parser.Stack(self.ctx, 'updated_stack',
template.Template(tmpl2),
disable_rollback=True)
self.stack.update(updated_stack)
self.assertEqual((parser.Stack.UPDATE, parser.Stack.FAILED),
self.stack.state)
self.assertEqual('smelly', self.stack['AResource'].properties['Foo'])
self.stack = parser.Stack.load(self.ctx, self.stack.id)
updated_stack2 = parser.Stack(self.ctx, 'updated_stack',
template.Template(tmpl2),
disable_rollback=True)
self.stack.update(updated_stack2)
self.assertEqual((parser.Stack.UPDATE, parser.Stack.COMPLETE),
self.stack.state)
self.assertEqual('smelly', self.stack['AResource'].properties['Foo'])
self.assertEqual('AResource2',
self.stack['BResource'].properties['Foo'])
self.m.VerifyAll()
def test_update_failure_recovery_new_param(self):
'''
assertion:
check that rollback still works with dynamic metadata
this test fails the second instance
'''
class ResourceTypeA(generic_rsrc.ResourceWithProps):
count = 0
def handle_create(self):
ResourceTypeA.count += 1
self.resource_id_set('%s%d' % (self.name, self.count))
def handle_delete(self):
return super(ResourceTypeA, self).handle_delete()
resource._register_class('ResourceTypeA', ResourceTypeA)
tmpl = {
'HeatTemplateFormatVersion': '2012-12-12',
'Parameters': {
'abc-param': {'Type': 'String'}
},
'Resources': {
'AResource': {'Type': 'ResourceTypeA',
'Properties': {'Foo': {'Ref': 'abc-param'}}},
'BResource': {'Type': 'ResourceWithPropsType',
'Properties': {'Foo': {'Ref': 'AResource'}}}
}
}
env1 = environment.Environment({'abc-param': 'abc'})
tmpl2 = {
'HeatTemplateFormatVersion': '2012-12-12',
'Parameters': {
'smelly-param': {'Type': 'String'}
},
'Resources': {
'AResource': {'Type': 'ResourceTypeA',
'Properties': {'Foo': {'Ref': 'smelly-param'}}},
'BResource': {'Type': 'ResourceWithPropsType',
'Properties': {'Foo': {'Ref': 'AResource'}}}
}
}
env2 = environment.Environment({'smelly-param': 'smelly'})
self.stack = parser.Stack(self.ctx, 'update_test_stack',
template.Template(tmpl), env1,
disable_rollback=True)
self.stack.store()
self.stack.create()
self.assertEqual((parser.Stack.CREATE, parser.Stack.COMPLETE),
self.stack.state)
self.assertEqual('abc', self.stack['AResource'].properties['Foo'])
self.assertEqual('AResource1',
self.stack['BResource'].properties['Foo'])
self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_create')
self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_delete')
self.m.StubOutWithMock(ResourceTypeA, 'handle_delete')
# mock to make the replace fail when creating the second
# replacement resource
generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception)
# delete the old resource on the second update
generic_rsrc.ResourceWithProps.handle_delete()
ResourceTypeA.handle_delete()
generic_rsrc.ResourceWithProps.handle_create()
generic_rsrc.ResourceWithProps.handle_delete()
self.m.ReplayAll()
updated_stack = parser.Stack(self.ctx, 'updated_stack',
template.Template(tmpl2), env2,
disable_rollback=True)
self.stack.update(updated_stack)
self.assertEqual((parser.Stack.UPDATE, parser.Stack.FAILED),
self.stack.state)
self.assertEqual('smelly', self.stack['AResource'].properties['Foo'])
self.stack = parser.Stack.load(self.ctx, self.stack.id)
updated_stack2 = parser.Stack(self.ctx, 'updated_stack',
template.Template(tmpl2), env2,
disable_rollback=True)
self.stack.update(updated_stack2)
self.assertEqual((parser.Stack.UPDATE, parser.Stack.COMPLETE),
self.stack.state)
self.stack = parser.Stack.load(self.ctx, self.stack.id)
self.assertEqual('smelly', self.stack['AResource'].properties['Foo'])
self.assertEqual('AResource2',
self.stack['BResource'].properties['Foo'])
self.m.VerifyAll()
def test_create_failure_recovery(self):
'''
assertion:
check that rollback still works with dynamic metadata
this test fails the second instance
'''
class ResourceTypeA(generic_rsrc.ResourceWithProps):
count = 0
def handle_create(self):
ResourceTypeA.count += 1
self.resource_id_set('%s%d' % (self.name, self.count))
def handle_delete(self):
return super(ResourceTypeA, self).handle_delete()
resource._register_class('ResourceTypeA', ResourceTypeA)
tmpl = {'HeatTemplateFormatVersion': '2012-12-12',
'Resources': {
'AResource': {'Type': 'ResourceTypeA',
'Properties': {'Foo': 'abc'}},
'BResource': {'Type': 'ResourceWithPropsType',
'Properties': {
'Foo': {'Ref': 'AResource'}}}}}
self.stack = parser.Stack(self.ctx, 'update_test_stack',
template.Template(tmpl),
disable_rollback=True)
self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_create')
self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_delete')
self.m.StubOutWithMock(ResourceTypeA, 'handle_delete')
# create
generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception)
# update
generic_rsrc.ResourceWithProps.handle_delete()
generic_rsrc.ResourceWithProps.handle_create()
self.m.ReplayAll()
self.stack.store()
self.stack.create()
self.assertEqual((parser.Stack.CREATE, parser.Stack.FAILED),
self.stack.state)
self.assertEqual('abc', self.stack['AResource'].properties['Foo'])
updated_stack = parser.Stack(self.ctx, 'updated_stack',
template.Template(tmpl),
disable_rollback=True)
self.stack.update(updated_stack)
self.assertEqual((parser.Stack.UPDATE, parser.Stack.COMPLETE),
self.stack.state)
self.assertEqual('abc', self.stack['AResource'].properties['Foo'])
self.assertEqual('AResource1',
self.stack['BResource'].properties['Foo'])
self.m.VerifyAll()
def test_update_replace_parameters(self):
'''
assertion: