Get Instance/AutoScalingGroup attributes from nested stack outputs

Since Pike we've generated the templates for various nested stack resource
types with outputs that produce the attribute values that are referenced in
the parent stack, and a previous patch did the same for resource IDs. Use
these output values when available to calculate the attribute values. This
is efficient, because the all of the outputs are fetched together and
cached, and avoids using the grouputils functions that cause the nested
stack to get loaded into memory in the same process.

Fall back to the existing implementation (using grouputils) if the required
output isn't available (generally this would be due to the nested stack
being created on an earlier version of Heat that did not include the
outputs in the generated template, and not updated since).

The InstanceGroup and AutoScalingGroup implementations are more complex
than those of ResourceGroup and ResourceChain because they exclude failed
resources at runtime, and also must have a stable sort order for list-type
outputs that cannot be determined at template generation time. Therefore
the underlying output value is always a dict, and the attribute value is
calculated from it.

Change-Id: Ie4ad85068fdc16b8038a09df709ed7eb86434cfa
Partial-Bug: #1731349
This commit is contained in:
Zane Bitter 2018-01-08 17:23:12 -05:00
parent ac87bc7c79
commit 9f9605d927
4 changed files with 221 additions and 30 deletions

View File

@ -13,6 +13,8 @@
import six
from oslo_log import log as logging
from heat.common import exception
from heat.common import grouputils
from heat.common.i18n import _
@ -25,6 +27,8 @@ from heat.engine.resources.aws.autoscaling import autoscaling_group as aws_asg
from heat.engine import rsrc_defn
from heat.engine import support
LOG = logging.getLogger(__name__)
class HOTInterpreter(template.HOTemplate20150430):
def __new__(cls):
@ -195,26 +199,73 @@ class AutoScalingResourceGroup(aws_asg.AutoScalingGroup):
self)._create_template(num_instances, num_replace,
template_version=template_version)
def _attribute_output_name(self, *attr_path):
return ', '.join(six.text_type(a) for a in attr_path)
def get_attribute(self, key, *path):
if key == self.CURRENT_SIZE:
return grouputils.get_size(self)
if key == self.REFS:
refs = grouputils.get_member_refids(self)
return refs
if key == self.REFS_MAP:
members = grouputils.get_members(self)
refs_map = {m.name: m.resource_id for m in members}
return refs_map
if path:
members = grouputils.get_members(self)
attrs = ((rsrc.name, rsrc.FnGetAtt(*path)) for rsrc in members)
if key == self.OUTPUTS:
return dict(attrs)
if key == self.OUTPUTS_LIST:
return [value for name, value in attrs]
if key.startswith("resource."):
return grouputils.get_nested_attrs(self, key, True, *path)
op_key = key
op_path = path
keycomponents = None
if key == self.OUTPUTS_LIST:
op_key = self.OUTPUTS
elif key == self.REFS:
op_key = self.REFS_MAP
elif key.startswith("resource."):
keycomponents = key.split('.', 2)
if len(keycomponents) > 2:
op_path = (keycomponents[2],) + path
op_key = self.OUTPUTS if op_path else self.REFS_MAP
try:
output = self.get_output(self._attribute_output_name(op_key,
*op_path))
except (exception.NotFound,
exception.TemplateOutputError) as op_err:
LOG.debug('Falling back to grouputils due to %s', op_err)
if key == self.REFS:
return grouputils.get_member_refids(self)
if key == self.REFS_MAP:
members = grouputils.get_members(self)
return {m.name: m.resource_id for m in members}
if path and key in {self.OUTPUTS, self.OUTPUTS_LIST}:
members = grouputils.get_members(self)
attrs = ((rsrc.name,
rsrc.FnGetAtt(*path)) for rsrc in members)
if key == self.OUTPUTS:
return dict(attrs)
if key == self.OUTPUTS_LIST:
return [value for name, value in attrs]
if keycomponents is not None:
return grouputils.get_nested_attrs(self, key, True, *path)
else:
if key in {self.REFS, self.REFS_MAP}:
names = self._group_data().member_names(False)
if key == self.REFS:
return [output[n] for n in names if n in output]
else:
return {n: output[n] for n in names if n in output}
if path and key in {self.OUTPUTS_LIST, self.OUTPUTS}:
names = self._group_data().member_names(False)
if key == self.OUTPUTS_LIST:
return [output[n] for n in names if n in output]
else:
return {n: output[n] for n in names if n in output}
if keycomponents is not None:
names = list(self._group_data().member_names(False))
index = keycomponents[1]
try:
resource_name = names[int(index)]
return output[resource_name]
except (IndexError, KeyError):
raise exception.NotFound(_("Member '%(mem)s' not found "
"in group resource '%(grp)s'.")
% {'mem': index,
'grp': self.name})
raise exception.InvalidTemplateAttribute(resource=self.name,
key=key)
@ -238,7 +289,7 @@ class AutoScalingResourceGroup(aws_asg.AutoScalingGroup):
key = self.OUTPUTS
else:
key = self.REFS_MAP
output_name = ', '.join(six.text_type(a) for a in [key] + path)
output_name = self._attribute_output_name(key, *path)
value = None
if key == self.REFS_MAP:

View File

@ -14,7 +14,10 @@
import functools
import six
from oslo_log import log as logging
from heat.common import environment_format
from heat.common import exception
from heat.common import grouputils
from heat.common.i18n import _
from heat.common import short_id
@ -30,6 +33,8 @@ from heat.scaling import lbutils
from heat.scaling import rolling_update
from heat.scaling import template
LOG = logging.getLogger(__name__)
(SCALED_RESOURCE_TYPE,) = ('OS::Heat::ScaledResource',)
@ -391,6 +396,13 @@ class InstanceGroup(stack_resource.StackResource):
def get_reference_id(self):
return self.physical_resource_name_or_FnGetRefId()
def _group_data(self, refresh=False):
"""Return a cached GroupInspector object for the nested stack."""
if refresh or getattr(self, '_group_inspector', None) is None:
inspector = grouputils.GroupInspector.from_parent_resource(self)
self._group_inspector = inspector
return self._group_inspector
def _resolve_attribute(self, name):
"""Resolves the resource's attributes.
@ -398,8 +410,24 @@ class InstanceGroup(stack_resource.StackResource):
ip addresses.
"""
if name == self.INSTANCE_LIST:
return u','.join(inst.FnGetAtt('PublicIp') or '0.0.0.0'
for inst in grouputils.get_members(self)) or None
def listify(ips):
return u','.join(ips) or None
try:
output = self.get_output(name)
except (exception.NotFound,
exception.TemplateOutputError) as op_err:
LOG.debug('Falling back to grouputils due to %s', op_err)
else:
if isinstance(output, dict):
names = self._group_data().member_names(False)
return listify(output[n] for n in names if n in output)
else:
LOG.debug('Falling back to grouputils due to '
'old (list-style) output format')
return listify(inst.FnGetAtt('PublicIp') or '0.0.0.0'
for inst in grouputils.get_members(self))
def _nested_output_defns(self, resource_names, get_attr_fn, get_res_fn):
for attr in self.referenced_attrs():

View File

@ -389,6 +389,101 @@ class HeatScalingGroupAttrTest(common.HeatTestCase):
self.assertRaises(exception.InvalidTemplateAttribute,
self.group.FnGetAtt, 'InstanceList')
def _stub_get_attr(self, refids, attrs):
def ref_id_fn(res_name):
return refids[res_name]
def attr_fn(args):
res_name = args[0]
return attrs[res_name]
inspector = self.group._group_data()
member_names = sorted(refids if refids else attrs)
self.patchobject(inspector, 'member_names', return_value=member_names)
def get_output(output_name):
outputs = self.group._nested_output_defns(member_names,
attr_fn, ref_id_fn)
op_defns = {od.name: od for od in outputs}
self.assertIn(output_name, op_defns)
return op_defns[output_name].get_value()
orig_get_attr = self.group.FnGetAtt
def get_attr(attr_name, *path):
if not path:
attr = attr_name
else:
attr = (attr_name,) + path
# Mock referenced_attrs() so that _nested_output_definitions()
# will include the output required for this attribute
self.group.referenced_attrs = mock.Mock(return_value=[attr])
# Pass through to actual function under test
return orig_get_attr(attr_name, *path)
self.group.FnGetAtt = mock.Mock(side_effect=get_attr)
self.group.get_output = mock.Mock(side_effect=get_output)
def test_output_attribute_list(self):
values = {str(i): '2.1.3.%d' % i for i in range(1, 4)}
self._stub_get_attr({n: 'foo' for n in values}, values)
expected = [v for k, v in sorted(values.items())]
self.assertEqual(expected, self.group.FnGetAtt('outputs_list', 'Bar'))
def test_output_attribute_dict(self):
values = {str(i): '2.1.3.%d' % i for i in range(1, 4)}
self._stub_get_attr({n: 'foo' for n in values}, values)
self.assertEqual(values, self.group.FnGetAtt('outputs', 'Bar'))
def test_index_dotted_attribute(self):
values = {'ab'[i - 1]: '2.1.3.%d' % i for i in range(1, 3)}
self._stub_get_attr({'a': 'foo', 'b': 'bar'}, values)
self.assertEqual(values['a'], self.group.FnGetAtt('resource.0', 'Bar'))
self.assertEqual(values['b'], self.group.FnGetAtt('resource.1.Bar'))
self.assertRaises(exception.NotFound,
self.group.FnGetAtt, 'resource.2')
def test_output_refs(self):
values = {'abc': 'resource-1', 'def': 'resource-2'}
self._stub_get_attr(values, {})
expected = [v for k, v in sorted(values.items())]
self.assertEqual(expected, self.group.FnGetAtt('refs'))
def test_output_refs_map(self):
values = {'abc': 'resource-1', 'def': 'resource-2'}
self._stub_get_attr(values, {})
self.assertEqual(values, self.group.FnGetAtt('refs_map'))
def test_attribute_current_size(self):
mock_instances = self.patchobject(grouputils, 'get_size')
mock_instances.return_value = 3
self.assertEqual(3, self.group.FnGetAtt('current_size'))
def test_attribute_current_size_with_path(self):
mock_instances = self.patchobject(grouputils, 'get_size')
mock_instances.return_value = 4
self.assertEqual(4, self.group.FnGetAtt('current_size', 'name'))
class HeatScalingGroupAttrFallbackTest(common.HeatTestCase):
def setUp(self):
super(HeatScalingGroupAttrFallbackTest, self).setUp()
t = template_format.parse(inline_templates.as_heat_template)
self.stack = utils.parse_stack(t, params=inline_templates.as_params)
self.group = self.stack['my-group']
self.assertIsNone(self.group.validate())
# Raise NotFound when getting output, to force fallback to old-school
# grouputils functions
self.group.get_output = mock.Mock(side_effect=exception.NotFound)
def test_output_attribute_list(self):
mock_members = self.patchobject(grouputils, 'get_members')
members = []
@ -448,16 +543,6 @@ class HeatScalingGroupAttrTest(common.HeatTestCase):
self.assertEqual(output,
self.group.FnGetAtt('outputs', 'Bar'))
def test_attribute_current_size(self):
mock_instances = self.patchobject(grouputils, 'get_size')
mock_instances.return_value = 3
self.assertEqual(3, self.group.FnGetAtt('current_size'))
def test_attribute_current_size_with_path(self):
mock_instances = self.patchobject(grouputils, 'get_size')
mock_instances.return_value = 4
self.assertEqual(4, self.group.FnGetAtt('current_size', 'name'))
def test_index_dotted_attribute(self):
mock_members = self.patchobject(grouputils, 'get_members')
self.group.nested = mock.Mock()
@ -465,7 +550,7 @@ class HeatScalingGroupAttrTest(common.HeatTestCase):
output = []
for ip_ex in six.moves.range(0, 2):
inst = mock.Mock()
inst.name = str(ip_ex)
inst.name = 'ab'[ip_ex]
inst.FnGetAtt.return_value = '2.1.3.%d' % ip_ex
output.append('2.1.3.%d' % ip_ex)
members.append(inst)

View File

@ -134,6 +134,33 @@ class TestInstanceGroup(common.HeatTestCase):
self.instance_group.resize.assert_called_once_with(5)
def test_attributes(self):
get_output = mock.Mock(return_value={'z': '2.1.3.1',
'x': '2.1.3.2',
'c': '2.1.3.3'})
self.instance_group.get_output = get_output
inspector = self.instance_group._group_data()
inspector.member_names = mock.Mock(return_value=['z', 'x', 'c'])
res = self.instance_group._resolve_attribute('InstanceList')
self.assertEqual('2.1.3.1,2.1.3.2,2.1.3.3', res)
get_output.assert_called_once_with('InstanceList')
def test_attributes_format_fallback(self):
self.instance_group.get_output = mock.Mock(return_value=['2.1.3.2',
'2.1.3.1',
'2.1.3.3'])
mock_members = self.patchobject(grouputils, 'get_members')
instances = []
for ip_ex in six.moves.range(1, 4):
inst = mock.Mock()
inst.FnGetAtt.return_value = '2.1.3.%d' % ip_ex
instances.append(inst)
mock_members.return_value = instances
res = self.instance_group._resolve_attribute('InstanceList')
self.assertEqual('2.1.3.1,2.1.3.2,2.1.3.3', res)
def test_attributes_fallback(self):
self.instance_group.get_output = mock.Mock(
side_effect=exception.NotFound)
mock_members = self.patchobject(grouputils, 'get_members')
instances = []
for ip_ex in six.moves.range(1, 4):