Set property required flag to true as default

The required key of every property that did not explicitely have
one already needed to have one defaulted to true as per spec.
This resulted in finding errors in TOSCA schema and tox tests
that needed to be corrected.

Change-Id: Iaefaa5dacb7d75b58c1d83e33842050de1a0f092

toscaparser/elements/TOSCA_definition_1_0.yaml

@@ -506,12 +506,14 @@ tosca.capabilities.Endpoint:
   properties:
     protocol:
       type: string
+      required: true
       default: tcp
     port:
       type: tosca.datatypes.network.PortDef
       required: false
     secure:
       type: boolean
+      required: false
       default: false
     url_path:
       type: string
@@ -524,6 +526,7 @@ tosca.capabilities.Endpoint:
       required: false
     initiator:
       type: string
+      required: false
       default: source
       constraints:
         - valid_values: [source, target, peer]
@@ -792,14 +795,18 @@ tosca.datatypes.network.PortSpec:
         - valid_values: [ udp, tcp, igmp ]
     target:
       type: PortDef
+      required: false
     target_range:
       type: range
+      required: false
       constraints:
         - in_range: [ 1, 65535 ]
     source:
       type: PortDef
+      required: false
     source_range:
       type: range
+      required: false
       constraints:
         - in_range: [ 1, 65535 ]
 

toscaparser/elements/nodetype.py

@@ -141,6 +141,8 @@ class NodeType(StatefulEntityType):
         if caps is None:
             caps = self.get_value(self.CAPABILITIES, None, True)
         if caps:
+            # 'name' is symbolic name of the capability
+            # 'value' is a dict { 'type': <capability type name> }
             for name, value in caps.items():
                 ctype = value.get('type')
                 cap = CapabilityTypeDef(name, ctype, self.type,

toscaparser/elements/property_definition.py

@@ -24,7 +24,7 @@ class PropertyDef(object):
                                PROPERTY_KEYNAME_STATUS) = \
         ('default', 'required', 'status')
 
-    PROPERTY_REQUIRED_DEFAULT = False
+    PROPERTY_REQUIRED_DEFAULT = True
 
     VALID_REQUIRED_VALUES = ['true', 'false']
     VALID_STATUS_VALUES = (PROPERTY_STATUS_SUPPORTED,

toscaparser/tests/data/datatypes/custom_datatype_def.yaml

@@ -20,6 +20,7 @@ data_types:
           - min_length: 2
       gender:
         type: string
+        required: false
         default: unknown
 
   tosca.my.datatypes.People:
@@ -27,10 +28,12 @@ data_types:
     properties:
       addresses:
         type: map
+        required: false
         entry_schema:
           type: string
       contacts:
         type: list
+        required: false
         entry_schema:
           type: tosca.my.datatypes.ContactInfo
 
@@ -44,5 +47,7 @@ data_types:
           - min_length: 2
       contact_email:
         type: string
+        required: false
       contact_phone:
         type: string
+        required: false

toscaparser/tests/data/datatypes/test_custom_datatypes_in_current_template.yaml

@@ -20,6 +20,7 @@ data_types:
           - min_length: 2
       gender:
         type: string
+        required: false
         default: unknown
 
   tosca.my.datatypes.People:
@@ -27,10 +28,12 @@ data_types:
     properties:
       addresses:
         type: map
+        required: false
         entry_schema:
           type: string
       contacts:
         type: list
+        required: false
         entry_schema:
           type: tosca.my.datatypes.ContactInfo
 
@@ -44,8 +47,10 @@ data_types:
           - min_length: 2
       contact_email:
         type: string
+        required: false
       contact_phone:
         type: string
+        required: false
 
 topology_template:
   node_templates:

toscaparser/tests/data/topology_template/system.yaml

@@ -21,8 +21,11 @@ topology_template:
     trans1:
       type: example.TransactionSubsystem
       properties:
-        # to be updated when substitution_mapping is implemented
+        # TODO to be updated when substitution_mapping is implemented
         # mq_server_ip: { get_attribute: [ mq, server_ip ] }
+        # for now, we will use the loopback address to avoid errors as
+        # this property is required in the schema
+        mq_server_ip: 127.0.0.1
         receiver_port: 8080
       # capabilities:
         # message_receiver:
@@ -33,8 +36,11 @@ topology_template:
     trans2:
       type: example.TransactionSubsystem
       properties:
-        # to be updated when substitution_mapping is implemented
+        # TODO to be updated when substitution_mapping is implemented
         # mq_server_ip: { get_attribute: [ mq, server_ip ] }
+        # for now, we will use the loopback address to avoid errors as
+        # this property is required in the schema
+        mq_server_ip: 127.0.0.1
         receiver_port: 8080
       # capabilities:
         # message_receiver:

toscaparser/tests/test_datatypes.py

@@ -42,10 +42,12 @@ class DataTypeTest(TestCase):
       properties:
         addresses:
           type: map
+          required: false
           entry_schema:
             type: string
         contacts:
           type: list
+          required: false
           entry_schema:
             type: tosca.my.datatypes.ContactInfo
 
@@ -69,6 +71,15 @@ class DataTypeTest(TestCase):
         value = yamlparser.simple_parse(value_snippet)
         self.assertEqual(value, {})
 
+    # TODO(Matt) - opened as bug 1555300
+    # Need a test for PortSpec normative data type
+    # that tests the spec. requirement: "A valid PortSpec
+    # must have at least one of the following properties:
+    # target, target_range, source or source_range."
+    # TODO(Matt) - opened as bug 1555310
+    # test PortSpec value for source and target
+    # against the source_range and target_range
+    # when specified.
     def test_built_in_datatype(self):
         value_snippet = '''
         private_network:

toscaparser/tests/test_properties.py

@@ -10,6 +10,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from testtools import matchers
+
 from toscaparser.common import exception
 from toscaparser.elements.property_definition import PropertyDef
 from toscaparser.nodetemplate import NodeTemplate
@@ -180,10 +182,12 @@ class PropertyTest(TestCase):
     def test_timestamp_invalid(self):
         test_property_schema = {'type': 'timestamp'}
         # invalid timestamp - day out of range
-        propertyInstance = Property('test_property', '2015-04-115T02:59:43.1Z',
+        value = '2015-04-115T02:59:43.1Z'
+        propertyInstance = Property('test_property', value,
                                     test_property_schema)
         error = self.assertRaises(ValueError, propertyInstance.validate)
-        self.assertEqual(_('day is out of range for month'), str(error))
+        expected_message = (_('"%s" is not a valid timestamp.') % value)
+        self.assertThat(str(error), matchers.StartsWith(expected_message))
 
     def test_required(self):
         test_property_schema = {'type': 'string'}

toscaparser/tests/test_toscadef.py

@@ -63,6 +63,11 @@ class ToscaDefTest(TestCase):
         self.assertIn(ifaces.LIFECYCLE_SHORTNAME, group_type.interfaces)
 
     def test_capabilities(self):
+        # Assure the normative Compute node type
+        # has all the required Capability types
+        # regardless of symbloc name
+        # TODO(Matt) - since Compute IS a normative node type
+        # we SHOULD test symbolic capability names as well
         self.assertEqual(
             ['tosca.capabilities.Container',
              'tosca.capabilities.Node',
@@ -70,34 +75,29 @@ class ToscaDefTest(TestCase):
              'tosca.capabilities.Scalable',
              'tosca.capabilities.network.Bindable'],
             sorted([c.type for c in compute_type.get_capabilities_objects()]))
+        # Assure the normative Network node type
+        # hsa all the required Capability types
+        # TODO(Matt) - since Network IS a normative node type
+        # we SHOULD test symbolic capability names as well
         self.assertEqual(
             ['tosca.capabilities.Node',
              'tosca.capabilities.network.Linkable'],
             sorted([c.type for c in network_type.get_capabilities_objects()]))
-        endpoint_properties = ['initiator', 'network_name', 'port',
-                               'port_name', 'ports', 'protocol',
-                               'secure', 'url_path']
+
+        # Assure the normative WebServer node type's
+        # Endpoint cap. has all required property names
+        # Note: we are testing them in alphabetic sort order
         endpoint_props_def_objects = \
             self._get_capability_properties_def_objects(
                 webserver_type.get_capabilities_objects(),
                 'tosca.capabilities.Endpoint')
+        # Assure WebServer's Endpoint capability's properties have their
+        # required keyname value set correctly
         self.assertEqual(
-            endpoint_properties,
-            sorted([p.name for p in endpoint_props_def_objects]))
-        for p in endpoint_props_def_objects:
-            if p.name in endpoint_properties:
-                self.assertFalse(p.required)
-        endpoint_props_def = self._get_capability_properties_def(
-            webserver_type.get_capabilities_objects(),
-            'tosca.capabilities.Endpoint')
-        self.assertEqual(
-            endpoint_properties,
-            sorted(endpoint_props_def.keys()))
-        endpoint_prop_def = self._get_capability_property_def(
-            webserver_type.get_capabilities_objects(),
-            'tosca.capabilities.Endpoint',
-            'initiator')
-        self.assertEqual(None, endpoint_prop_def)
+            [('initiator', False), ('network_name', False), ('port', False),
+             ('port_name', False), ('ports', False), ('protocol', True),
+             ('secure', False), ('url_path', False)],
+            sorted([(p.name, p.required) for p in endpoint_props_def_objects]))
 
         os_props = self._get_capability_properties_def_objects(
             compute_type.get_capabilities_objects(),
@@ -139,14 +139,6 @@ class ToscaDefTest(TestCase):
                 break
         return properties_def
 
-    def _get_capability_property_def(self, caps, type, property):
-        property_def = None
-        for cap in caps:
-            if cap.type == type:
-                property_def = cap.get_property_def_value(property)
-                break
-        return property_def
-
     def test_properties_def(self):
         self.assertEqual(
             ['name', 'password', 'port', 'user'],

toscaparser/tests/test_toscatpl.py

@@ -348,6 +348,11 @@ class ToscaTemplateTest(TestCase):
             NotImplementedError,
             lambda: NodeTemplate(tpl_name, nodetemplates).relationships)
 
+    # Test the following:
+    # 1. Custom node type derived from 'WebApplication' named 'TestApp'
+    #    with a custom Capability Type 'TestCapability'
+    # 2. Same as #1, but referencing a custom 'TestCapability' Capability Type
+    #    that is not defined
     def test_custom_capability_type_definition(self):
         tpl_snippet = '''
         node_templates:
@@ -358,7 +363,7 @@ class ToscaTemplateTest(TestCase):
                 properties:
                   test: 1
         '''
-        # custom definition with capability type definition
+        # custom node type definition with custom capability type definition
         custom_def = '''
         tosca.nodes.WebApplication.TestApp:
           derived_from: tosca.nodes.WebApplication
@@ -383,7 +388,7 @@ class ToscaTemplateTest(TestCase):
             expected_capabilities,
             sorted(tpl.get_capabilities().keys()))
 
-        # custom definition without capability type definition
+        # custom definition without valid capability type definition
         custom_def = '''
         tosca.nodes.WebApplication.TestApp:
           derived_from: tosca.nodes.WebApplication

toscaparser/utils/validateutils.py

@@ -89,7 +89,19 @@ def validate_boolean(value):
 
 
 def validate_timestamp(value):
-    return dateutil.parser.parse(value)
+    try:
+        # Note: we must return our own exception message
+        # as dateutil's parser returns different types / values on
+        # different systems. OSX, for example, returns a tuple
+        # containing a different error message than Linux
+        dateutil.parser.parse(value)
+    except Exception as e:
+        original_err_msg = str(e)
+        log.error(original_err_msg)
+        ExceptionCollector.appendException(
+            ValueError(_('"%(val)s" is not a valid timestamp. "%(msg)s"') %
+                       {'val': value, 'msg': original_err_msg}))
+    return
 
 
 class TOSCAVersionProperty(object):