openstack
/
heat
Code Issues Proposed changes

Avoid loading nested stacks in memory where possible 

Prior to changing StackResource to do stack operations over RPC, we made
liberal use of the StackResource.nested() method to access the nested stack
that was likely always loaded in memory. Now that that is no longer
required, it adds additional memory overhead that we need not have. We can
now obtain the stack identifier without loading the stack, and that is
sufficient for performing operations over RPC.

The exceptions are prepare_abandon(), which cannot be done over RPC at
present, and get_output(), which may be addressed in a separate patch. The
gratuitous loading of the nested stack in TemplateResource.get_attribute()
is eliminated, so although it still ends up loading the nested stack in
many cases, it will no longer do so once get_output() stops doing it.

Change-Id: I669d2a077381d7e4e913f6ad1a86fb3f094da6c5
Co-Authored-By: Thomas Herve <therve@redhat.com>
Related-Bug: #1626675
(cherry picked from commit df889488ed)
This commit is contained in:
Zane Bitter 2016-10-10 15:11:42 -04:00 committed by Thomas Herve
parent 0078a72b10
commit 53137cb228
5 changed files with 81 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
heat/engine/resources/stack_resource.py
View File

@ -92,24 +92,23 @@ class StackResource(resource.Resource):
def _needs_update(self, after, before, after_props, before_props,
prev_resource, check_init_complete=True):
# Issue an update to the nested stack if the stack resource
# is able to update. If return true, let the individual
# resources in it decide if they need updating.
# FIXME (ricolin): seems currently can not call super here
if self.nested() is None and self.status == self.FAILED:
raise resource.UpdateReplace(self)
# If stack resource is in CHECK_FAILED state, raise UpdateReplace
# to replace the failed stack.
if self.state == (self.CHECK, self.FAILED):
raise resource.UpdateReplace(self)
if (check_init_complete and
self.nested() is None and
self.action == self.INIT and self.status == self.COMPLETE):
raise resource.UpdateReplace(self)
# If the nested stack has not been created, use the default
# implementation to determine if we need to replace the resource. Note
# that we do *not* return the result.
if self.resource_id is None:
super(StackResource, self)._needs_update(after, before,
after_props, before_props,
prev_resource,
check_init_complete)
# Always issue an update to the nested stack and let the individual
# resources in it decide if they need updating.
return True
def nested_identifier(self):
@ -450,8 +449,7 @@ class StackResource(resource.Resource):
self.physical_resource_name())
return {'target_action': self.stack.ROLLBACK}
nested_stack = self.nested()
if nested_stack is None:
if self.resource_id is None:
# if the create failed for some reason and the nested
# stack was not created, we need to create an empty stack
# here so that the update will work.
@ -464,18 +462,25 @@ class StackResource(resource.Resource):
self.create_with_template(empty_temp, {})
checker = scheduler.TaskRunner(_check_for_completion)
checker(timeout=self.stack.timeout_secs())
nested_stack = self.nested()
if timeout_mins is None:
timeout_mins = self.stack.timeout_mins
try:
status_data = stack_object.Stack.get_status(self.context,
self.resource_id)
except exception.NotFound:
raise resource.UpdateReplace(self)
action, status, status_reason, updated_time = status_data
kwargs = self._stack_kwargs(user_params, child_template)
cookie = {'previous': {
'updated_at': nested_stack.updated_time,
'state': nested_stack.state}}
'updated_at': updated_time,
'state': (action, status)}}
kwargs.update({
'stack_identity': dict(nested_stack.identifier()),
'stack_identity': dict(self.nested_identifier()),
'args': {rpc_api.PARAM_TIMEOUT: timeout_mins}
})
with self.translate_remote_exceptions:
@ -515,12 +520,10 @@ class StackResource(resource.Resource):
def delete_nested(self):
"""Delete the nested stack."""
stack = self.nested()
if stack is None:
stack_identity = self.nested_identifier()
if stack_identity is None:
return
stack_identity = dict(stack.identifier())
try:
if self.abandon_in_progress:
self.rpc_client().abandon_stack(self.context, stack_identity)
@ -537,34 +540,31 @@ class StackResource(resource.Resource):
return self._check_status_complete(self.DELETE)
def handle_suspend(self):
stack = self.nested()
if stack is None:
stack_identity = self.nested_identifier()
if stack_identity is None:
raise exception.Error(_('Cannot suspend %s, stack not created')
% self.name)
stack_identity = self.nested_identifier()
self.rpc_client().stack_suspend(self.context, dict(stack_identity))
def check_suspend_complete(self, cookie=None):
return self._check_status_complete(self.SUSPEND)
def handle_resume(self):
stack = self.nested()
if stack is None:
stack_identity = self.nested_identifier()
if stack_identity is None:
raise exception.Error(_('Cannot resume %s, stack not created')
% self.name)
stack_identity = self.nested_identifier()
self.rpc_client().stack_resume(self.context, dict(stack_identity))
def check_resume_complete(self, cookie=None):
return self._check_status_complete(self.RESUME)
def handle_check(self):
stack = self.nested()
if stack is None:
stack_identity = self.nested_identifier()
if stack_identity is None:
raise exception.Error(_('Cannot check %s, stack not created')
% self.name)
stack_identity = self.nested_identifier()
self.rpc_client().stack_check(self.context, dict(stack_identity))
def check_check_complete(self, cookie=None):
@ -574,7 +574,7 @@ class StackResource(resource.Resource):
self.abandon_in_progress = True
nested_stack = self.nested()
if nested_stack:
return self.nested().prepare_abandon()
return nested_stack.prepare_abandon()
return {}
@ -582,7 +582,9 @@ class StackResource(resource.Resource):
"""Return the specified Output value from the nested stack.
If the output key does not exist, raise an InvalidTemplateAttribute
exception.
exception. (Note that TemplateResource.get_attribute() relies on this
particular exception, not KeyError, being raised if the key does not
exist.)
"""
stack = self.nested()
if stack is None:

22
heat/engine/resources/template_resource.py
View File

@ -198,7 +198,7 @@ class TemplateResource(stack_resource.StackResource):
reported_excp = err
if t_data is None:
if self.nested() is not None:
if self.resource_id is not None:
t_data = jsonutils.dumps(self.nested().t.t)
if t_data is not None:
@ -296,17 +296,16 @@ class TemplateResource(stack_resource.StackResource):
self.child_params())
def get_reference_id(self):
if self.nested() is None:
if self.resource_id is None:
return six.text_type(self.name)
if 'OS::stack_id' in self.nested().outputs:
return self.nested().outputs['OS::stack_id'].get_value()
return self.nested().identifier().arn()
try:
return self.get_output('OS::stack_id')
except exception.InvalidTemplateAttribute:
return self.nested_identifier().arn()
def get_attribute(self, key, *path):
stack = self.nested()
if stack is None:
if self.resource_id is None:
return None
# first look for explicit resource.x.y
@ -314,9 +313,4 @@ class TemplateResource(stack_resource.StackResource):
return grouputils.get_nested_attrs(self, key, False, *path)
# then look for normal outputs
if key in stack.outputs:
return attributes.select_from_attribute(self.get_output(key), path)
# otherwise the key must be wrong.
raise exception.InvalidTemplateAttribute(resource=self.name,
key=key)
return attributes.select_from_attribute(self.get_output(key), path)

7
heat/tests/test_nested_stack.py
View File

@ -405,12 +405,13 @@ Outputs:
def test_handle_delete(self):
self.res.rpc_client = mock.MagicMock()
self.res.action = self.res.CREATE
self.res.nested = mock.MagicMock()
self.res.nested_identifier = mock.MagicMock()
stack_identity = identifier.HeatIdentifier(
self.ctx.tenant_id,
self.res.physical_resource_name(),
self.res.resource_id)
self.res.nested().identifier.return_value = stack_identity
self.res.nested_identifier.return_value = stack_identity
self.res.resource_id = stack_identity.stack_id
self.res.handle_delete()
self.res.rpc_client.return_value.delete_stack.assert_called_once_with(
self.ctx, self.res.nested().identifier(), cast=False)
self.ctx, stack_identity, cast=False)

4
heat/tests/test_provider_template.py
View File

@ -271,6 +271,7 @@ class ProviderTemplateTest(common.HeatTestCase):
"DummyResource2")
temp_res = template_resource.TemplateResource('test_t_res',
definition, stack)
temp_res.resource_id = 'dummy_id'
nested = mock.Mock()
nested.outputs = {'Blarg': {'Value': 'fluffy'}}
temp_res._nested = nested
@ -305,6 +306,7 @@ class ProviderTemplateTest(common.HeatTestCase):
"DummyResource")
temp_res = template_resource.TemplateResource('test_t_res',
definition, stack)
temp_res.resource_id = 'dummy_id'
self.assertIsNone(temp_res.validate())
nested = mock.Mock()
nested.outputs = {'Foo': {'Value': 'not-this',
@ -932,6 +934,7 @@ class TemplateDataTest(common.HeatTestCase):
def test_template_data_in_update_without_template_file(self):
self.res.action = self.res.UPDATE
self.res.resource_id = 'dummy_id'
self.res.nested = mock.MagicMock()
self.res.get_template_file = mock.Mock(
side_effect=exception.NotFound(
@ -942,6 +945,7 @@ class TemplateDataTest(common.HeatTestCase):
def test_template_data_in_create_without_template_file(self):
self.res.action = self.res.CREATE
self.res.nested = mock.MagicMock()
self.res.resource_id = 'dummy_id'
self.res.get_template_file = mock.Mock(
side_effect=exception.NotFound(
msg_fmt='Could not fetch remote template '

57
heat/tests/test_stack_resource.py
View File

@ -21,6 +21,7 @@ from oslo_serialization import jsonutils
import six
from heat.common import exception
from heat.common import identifier
from heat.common import template_format
from heat.engine import output
from heat.engine import resource
@ -205,7 +206,7 @@ class StackResourceTest(StackResourceBaseTest):
def test_abandon_nested_sends_rpc_abandon(self):
rpcc = mock.Mock()
self.parent_resource.rpc_client = rpcc
self.parent_resource.nested = mock.MagicMock()
self.parent_resource.resource_id = 'fake_id'
self.parent_resource.prepare_abandon()
self.parent_resource.delete_nested()
@ -507,7 +508,7 @@ class StackResourceTest(StackResourceBaseTest):
self.assertIsNone(self.parent_resource.delete_nested())
def test_delete_nested_not_found_nested_stack(self):
self.parent_resource._nested = mock.MagicMock()
self.parent_resource.resource_id = 'fake_id'
rpcc = mock.Mock()
self.parent_resource.rpc_client = rpcc
rpcc.return_value.delete_stack = mock.Mock(
@ -943,25 +944,27 @@ class WithTemplateTest(StackResourceBaseTest):
def test_update_with_template(self):
if self.adopt_data is not None:
return
nested = mock.MagicMock()
nested.updated_time = 'now_time'
nested.state = ('CREATE', 'COMPLETE')
nested.identifier.return_value = {'stack_identifier':
'stack-identifier'}
self.parent_resource.nested = mock.MagicMock(return_value=nested)
self.parent_resource._nested = nested
ident = identifier.HeatIdentifier(self.ctx.tenant_id, 'fake_name',
'pancakes')
self.parent_resource.resource_id = ident.stack_id
self.parent_resource.nested_identifier = mock.Mock(return_value=ident)
self.parent_resource.child_params = mock.Mock(
return_value=self.params)
rpcc = mock.Mock()
self.parent_resource.rpc_client = rpcc
rpcc.return_value._update_stack.return_value = {'stack_id': 'pancakes'}
self.parent_resource.update_with_template(
self.empty_temp, user_params=self.params,
timeout_mins=self.timeout_mins)
rpcc.return_value._update_stack.return_value = dict(ident)
status = ('CREATE', 'COMPLETE', '', 'now_time')
with self.patchobject(stack_object.Stack, 'get_status',
return_value=status):
self.parent_resource.update_with_template(
self.empty_temp, user_params=self.params,
timeout_mins=self.timeout_mins)
rpcc.return_value._update_stack.assert_called_once_with(
self.ctx,
stack_identity={'stack_identifier': 'stack-identifier'},
stack_identity=dict(ident),
template_id=self.IntegerMatch(),
template=None,
params=None,
@ -974,13 +977,10 @@ class WithTemplateTest(StackResourceBaseTest):
if self.adopt_data is not None:
return
nested = mock.MagicMock()
nested.updated_time = 'now_time'
nested.state = ('CREATE', 'COMPLETE')
nested.identifier.return_value = {'stack_identifier':
'stack-identifier'}
self.parent_resource.nested = mock.MagicMock(return_value=nested)
self.parent_resource._nested = nested
ident = identifier.HeatIdentifier(self.ctx.tenant_id, 'fake_name',
'pancakes')
self.parent_resource.resource_id = ident.stack_id
self.parent_resource.nested_identifier = mock.Mock(return_value=ident)
self.parent_resource.child_params = mock.Mock(
return_value=self.params)
@ -988,14 +988,19 @@ class WithTemplateTest(StackResourceBaseTest):
self.parent_resource.rpc_client = rpcc
remote_exc = StackValidationFailed_Remote(message='oops')
rpcc.return_value._update_stack.side_effect = remote_exc
self.assertRaises(exception.ResourceFailure,
self.parent_resource.update_with_template,
self.empty_temp, user_params=self.params,
timeout_mins=self.timeout_mins)
status = ('CREATE', 'COMPLETE', '', 'now_time')
with self.patchobject(stack_object.Stack, 'get_status',
return_value=status):
self.assertRaises(exception.ResourceFailure,
self.parent_resource.update_with_template,
self.empty_temp, user_params=self.params,
timeout_mins=self.timeout_mins)
template_id = self.IntegerMatch()
rpcc.return_value._update_stack.assert_called_once_with(
self.ctx,
stack_identity={'stack_identifier': 'stack-identifier'},
stack_identity=dict(ident),
template_id=template_id,
template=None,
params=None,