openstack
/
deb-python-tosca-parser
Code Issues Proposed changes

Add input validation in substitution_mapping class 

Add input validation in class of substitution_mapping according to
specification of
http://docs.oasis-open.org/tosca/TOSCA-Simple-Profile-YAML/v1.0/
TOSCA-Simple-Profile-YAML-v1.0.html:
1) The properties of substituted node template which be mapped must be
in the inputs of nested service template which defines substutition
mappings;
2) The inputs of nested service template which are not in properties of
the substituted node template must have default values.
3) If the properties of node_type is required and no default value,
 must provide inputs for them;
4) Property names and the input names must be the same.

This patch is related to bp:https://review.openstack.org/#/c/345492/

Change-Id: Ib928434ec67661e689ac80cca1749b53d17e4ba8
Signed-off-by: shangxdy <shang.xiaodong@zte.com.cn>
This commit is contained in:
shangxdy 2016-09-09 00:31:15 +08:00
parent 63384ef235
commit 9baadf93b5
7 changed files with 289 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
toscaparser/common/exception.py
View File

@ -114,11 +114,21 @@ class UnknownInputError(TOSCAException):
msg_fmt = _('Unknown input "%(input_name)s".')
class MissingRequiredInputError(TOSCAException):
msg_fmt = _('%(what)s is missing required input definition '
'of input "%(input_name)s".')
class MissingRequiredParameterError(TOSCAException):
msg_fmt = _('%(what)s is missing required parameter for input: '
msg_fmt = _('%(what)s is missing required parameter for input '
'"%(input_name)s".')
class MissingDefaultValueError(TOSCAException):
msg_fmt = _('%(what)s is missing required default value '
'of input "%(input_name)s".')
class InvalidPropertyValueError(TOSCAException):
msg_fmt = _('Value of property "%(what)s" is invalid.')

79
toscaparser/substitution_mappings.py
View File

@ -14,9 +14,12 @@ import logging
from toscaparser.common.exception import ExceptionCollector
from toscaparser.common.exception import InvalidNodeTypeError
from toscaparser.common.exception import MissingDefaultValueError
from toscaparser.common.exception import MissingRequiredFieldError
from toscaparser.common.exception import MissingRequiredInputError
from toscaparser.common.exception import UnknownFieldError
from toscaparser.elements.nodetype import NodeType
from toscaparser.utils.gettextutils import _
log = logging.getLogger('tosca')
@ -47,7 +50,7 @@ class SubstitutionMappings(object):
@property
def type(self):
if self.sub_mapping_def:
return self.sub_mapping_def['node_type']
return self.sub_mapping_def.get(self.NODE_TYPE)
@classmethod
def get_node_type(cls, sub_mapping_def):
@ -66,9 +69,16 @@ class SubstitutionMappings(object):
def requirements(self):
return self.sub_mapping_def.get(self.REQUIREMENTS)
@property
def node_definition(self):
return NodeType(self.node_type, self.custom_defs)
def _validate(self):
# Basic validation
self._validate_keys()
self._validate_type()
# SubstitutionMapping class syntax validation
self._validate_inputs()
self._validate_capabilities()
self._validate_requirements()
@ -79,7 +89,7 @@ class SubstitutionMappings(object):
for key in self.sub_mapping_def.keys():
if key not in self.SECTIONS:
ExceptionCollector.appendException(
UnknownFieldError(what='SubstitutionMappings',
UnknownFieldError(what=_('SubstitutionMappings'),
field=key))
def _validate_type(self):
@ -94,24 +104,59 @@ class SubstitutionMappings(object):
node_type_def = self.custom_defs.get(node_type)
if not node_type_def:
ExceptionCollector.appendException(
InvalidNodeTypeError(what=node_type_def))
InvalidNodeTypeError(what=node_type))
def _validate_inputs(self):
"""validate the inputs of substitution mappings."""
"""validate the inputs of substitution mappings.
# The inputs in service template which defines substutition mappings
# must be in properties of node template which is mapped or provide
# defualt value. Currently the input.name is not restrict to be the
# same as properte's name in specification, but they should be equal
# for current implementation.
property_names = list(self.sub_mapped_node_template
.get_properties().keys()
if self.sub_mapped_node_template else [])
for input in self.inputs:
if input.name not in property_names and input.default is None:
The inputs defined by the topology template have to match the
properties of the node type or the substituted node. If there are
more inputs than the substituted node has properties, default values
must be defined for those inputs.
"""
all_inputs = set([input.name for input in self.inputs])
required_properties = set([p.name for p in
self.node_definition.
get_properties_def_objects()
if p.required and p.default is None])
# Must provide inputs for required properties of node type.
for property in required_properties:
# Check property which is 'required' and has no 'default' value
if property not in all_inputs:
ExceptionCollector.appendException(
UnknownFieldError(what='SubstitutionMappings',
field=input.name))
MissingRequiredInputError(
what=_('SubstitutionMappings with node_type ')
+ self.node_type,
input_name=property))
# If the optional properties of node type need to be customized by
# substituted node, it also is necessary to define inputs for them,
# otherwise they are not mandatory to be defined.
customized_parameters = set(self.sub_mapped_node_template
.get_properties().keys()
if self.sub_mapped_node_template else [])
all_properties = set(self.node_definition.get_properties_def())
for parameter in customized_parameters - all_inputs:
if parameter in all_properties:
ExceptionCollector.appendException(
MissingRequiredInputError(
what=_('SubstitutionMappings with node_type ')
+ self.node_type,
input_name=parameter))
# Additional inputs are not in the properties of node type must
# provide default values. Currently the scenario may not happen
# because of parameters validation in nodetemplate, here is a
# guarantee.
for input in self.inputs:
if input.name in all_inputs - all_properties \
and input.default is None:
ExceptionCollector.appendException(
MissingDefaultValueError(
what=_('SubstitutionMappings with node_type ')
+ self.node_type,
input_name=input.name))
def _validate_capabilities(self):
"""validate the capabilities of substitution mappings."""

76
toscaparser/tests/data/topology_template/validate/queuingsubsystem_invalid_input.yaml Normal file
View File

@ -0,0 +1,76 @@
tosca_definitions_version: tosca_simple_yaml_1_0
description: >
This template is a test template which contains invalid input needed for substitution mappings.
The required properties without default value in substituted node template which be mapped must be
as inputs of nested service template which defines substutition mappings, and the inputs of nested
service template which are not in the properties of the substituted node template must provide
default values.
This template provides an additional input of server_port1/my_cpus/my_input which are not defined
in example.QueuingSubsystem, and the default value are 8080/2/123, all of these are right. But the
required property of server_port defined in example.QueuingSubsystem is not appeared in inputs
definiton, so will raise excepton of "MissingRequiredInputError".
imports:
- ../definitions.yaml
topology_template:
description: Template of a database including its hosting stack.
inputs:
server_ip:
type: string
description: IP address of the message queuing server to receive messages from.
default: 127.0.0.1
server_port1:
type: integer
description: Port to be used for receiving messages.
default: 8080
my_cpus:
type: integer
description: Number of CPUs for the server.
default: 2
constraints:
- valid_values: [ 1, 2, 4, 8 ]
my_input:
type: integer
description: test for input validation.
default: 123
substitution_mappings:
node_type: example.QueuingSubsystem
node_templates:
tran_app:
type: example.QueuingSubsystem
properties:
server_ip: { get_input: server_ip }
server_port: { get_input: server_port1 }
requirements:
- host:
node: server
server:
type: tosca.nodes.Compute
capabilities:
host:
properties:
disk_size: 10 GB
num_cpus: { get_input: my_cpus }
mem_size: 4096 MB
os:
properties:
architecture: x86_64
type: Linux
distribution: Ubuntu
version: 14.04
outputs:
receiver_ip:
description: private IP address of the message receiver application
value: { get_attribute: [ server, private_address ] }
groups:
tran_server_group:
members: [ tran_app, server ]
type: tosca.groups.Root

24
toscaparser/tests/data/topology_template/validate/system_invalid_input.yaml Normal file
View File

@ -0,0 +1,24 @@
tosca_definitions_version: tosca_simple_yaml_1_0
imports:
- queuingsubsystem_invalid_input.yaml
topology_template:
description: Test template with invalid input.
inputs:
mq_server_ip:
type: string
default: 127.0.0.1
description: IP address of the message queuing server to receive messages from.
mq_server_port:
type: integer
default: 8080
description: Port to be used for receiving messages.
node_templates:
mq:
type: example.QueuingSubsystem
properties:
server_ip: { get_input: mq_server_ip }
server_port: { get_input: mq_server_port }

100
toscaparser/tests/test_topology_template.py
View File

@ -12,9 +12,12 @@
import os
from toscaparser.common import exception
from toscaparser.substitution_mappings import SubstitutionMappings
from toscaparser.tests.base import TestCase
from toscaparser.topology_template import TopologyTemplate
from toscaparser.tosca_template import ToscaTemplate
from toscaparser.utils.gettextutils import _
import toscaparser.utils.yamlparser
YAML_LOADER = toscaparser.utils.yamlparser.load_yaml
@ -52,6 +55,18 @@ class TopologyTemplateTest(TestCase):
custom_defs.update(self._get_custom_def('capability_types'))
return custom_defs
def _get_custom_types(self):
custom_types = {}
def_file = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"data/topology_template/definitions.yaml")
custom_type = YAML_LOADER(def_file)
node_types = custom_type['node_types']
for name in node_types:
defintion = node_types[name]
custom_types[name] = defintion
return custom_types
def test_description(self):
expected_desc = 'Template of a database including its hosting stack.'
self.assertEqual(expected_desc, self.topo.description)
@ -162,3 +177,88 @@ class TopologyTemplateTest(TestCase):
self.assertEqual(
len(system_tosca_template.
nested_tosca_templates_with_topology), 4)
self.assertTrue(system_tosca_template.has_nested_templates())
def test_invalid_keyname(self):
tpl_snippet = '''
substitution_mappings:
node_type: example.DatabaseSubsystem
capabilities:
database_endpoint: [ db_app, database_endpoint ]
requirements:
receiver1: [ tran_app, receiver1 ]
invalid_key: 123
'''
sub_mappings = (toscaparser.utils.yamlparser.
simple_parse(tpl_snippet))['substitution_mappings']
expected_message = _(
'SubstitutionMappings contains unknown field '
'"invalid_key". Refer to the definition '
'to verify valid values.')
err = self.assertRaises(
exception.UnknownFieldError,
lambda: SubstitutionMappings(sub_mappings, None, None,
None, None, None))
self.assertEqual(expected_message, err.__str__())
def test_missing_required_keyname(self):
tpl_snippet = '''
substitution_mappings:
capabilities:
database_endpoint: [ db_app, database_endpoint ]
requirements:
receiver1: [ tran_app, receiver1 ]
'''
sub_mappings = (toscaparser.utils.yamlparser.
simple_parse(tpl_snippet))['substitution_mappings']
expected_message = _('SubstitutionMappings used in topology_template '
'is missing required field "node_type".')
err = self.assertRaises(
exception.MissingRequiredFieldError,
lambda: SubstitutionMappings(sub_mappings, None, None,
None, None, None))
self.assertEqual(expected_message, err.__str__())
def test_invalid_nodetype(self):
tpl_snippet = '''
substitution_mappings:
node_type: example.DatabaseSubsystem1
capabilities:
database_endpoint: [ db_app, database_endpoint ]
requirements:
receiver1: [ tran_app, receiver1 ]
'''
sub_mappings = (toscaparser.utils.yamlparser.
simple_parse(tpl_snippet))['substitution_mappings']
custom_defs = self._get_custom_types()
expected_message = _('Node type "example.DatabaseSubsystem1" '
'is not a valid type.')
err = self.assertRaises(
exception.InvalidNodeTypeError,
lambda: SubstitutionMappings(sub_mappings, None, None,
None, None, custom_defs))
self.assertEqual(expected_message, err.__str__())
def test_system_with_input_validation(self):
tpl_path0 = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"data/topology_template/validate/system_invalid_input.yaml")
tpl_path1 = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"data/topology_template/validate/"
"queuingsubsystem_invalid_input.yaml")
errormsg = _('SubstitutionMappings with node_type '
'example.QueuingSubsystem is missing '
'required input definition of input "server_port".')
# It's invalid in nested template.
self.assertRaises(exception.ValidationError,
lambda: ToscaTemplate(tpl_path0))
exception.ExceptionCollector.assertExceptionMessage(
exception.MissingRequiredInputError, errormsg)
# Subtemplate deploy standaolone is also invalid.
self.assertRaises(exception.ValidationError,
lambda: ToscaTemplate(tpl_path1))
exception.ExceptionCollector.assertExceptionMessage(
exception.MissingRequiredInputError, errormsg)

17
toscaparser/tosca_template.py
View File

@ -14,6 +14,7 @@
import logging
import os
from copy import deepcopy
from toscaparser.common.exception import ExceptionCollector
from toscaparser.common.exception import InvalidTemplateVersion
from toscaparser.common.exception import MissingRequiredFieldError
@ -224,12 +225,14 @@ class ToscaTemplate(object):
for fname, tosca_tpl in self.nested_tosca_tpls_with_topology.items():
for nodetemplate in self.nodetemplates:
if self._is_sub_mapped_node(nodetemplate, tosca_tpl):
parsed_params = self._get_params_for_nested_template(
nodetemplate)
topology_tpl = tosca_tpl.get(TOPOLOGY_TEMPLATE)
topology_with_sub_mapping = TopologyTemplate(
topology_tpl,
self._get_all_custom_defs(),
self.relationship_types,
self.parsed_params,
parsed_params,
nodetemplate)
if topology_with_sub_mapping.substitution_mappings:
# Record nested topo templates in top level template
@ -311,13 +314,23 @@ class ToscaTemplate(object):
else:
return False
def _get_params_for_nested_template(self, nodetemplate):
"""Return total params for nested_template."""
parsed_params = deepcopy(self.parsed_params) \
if self.parsed_params else {}
if nodetemplate:
for pname in nodetemplate.get_properties():
parsed_params.update({pname:
nodetemplate.get_property_value(pname)})
return parsed_params
def get_sub_mapping_node_type(self, tosca_tpl):
"""Return substitution mappings node type."""
if tosca_tpl:
return TopologyTemplate.get_sub_mapping_node_type(
tosca_tpl.get(TOPOLOGY_TEMPLATE))
def has_substitution_mappings(self):
def _has_substitution_mappings(self):
"""Return True if the template has valid substitution mappings."""
return self.topology_template is not None and \
self.topology_template.substitution_mappings is not None

1
toscaparser/utils/gettextutils.py
View File

@ -19,4 +19,5 @@ _t = gettext.translation('tosca-parser', localedir=_localedir,
def _(msg):
# type: (object) -> object
return _t.gettext(msg)