From a6e7af984c75b94960860e29977b4c9195217ee3 Mon Sep 17 00:00:00 2001 From: Eric K Date: Tue, 3 Jul 2018 16:44:10 -0700 Subject: [PATCH] explicit data types - part I changes: - Basic data types. - Compute schema from data type specification in data source drivers. Limited to hdict translator for now. - Data type is checked and enforced by data source driver when ingesting data. - Policy engine can access type information through compile.Schema interface. Future work: - Data type schema computation from other translator types - Data type for generated IDs - Additional standard data types (UUID, etc.) - Loading of additional data types specified in data source drivers - Adding type information to data source drivers partially-implements: blueprint explicit-data-types Change-Id: I054cae5b9b138ab269c526c9c926fb2838a19314 --- congress/data_types.py | 240 ++++++++++++++++++ congress/datalog/compile.py | 31 ++- congress/datasources/datasource_driver.py | 133 +++++++--- congress/datasources/datasource_utils.py | 9 +- congress/tests/datalog/test_compiler.py | 20 ++ .../datasources/test_datasource_driver.py | 66 ++++- .../policy_engines/test_agnostic_dse2.py | 4 +- congress/tests/test_data_types.py | 31 +++ 8 files changed, 484 insertions(+), 50 deletions(-) create mode 100644 congress/data_types.py create mode 100644 congress/tests/test_data_types.py diff --git a/congress/data_types.py b/congress/data_types.py new file mode 100644 index 000000000..5a4c6fb76 --- /dev/null +++ b/congress/data_types.py @@ -0,0 +1,240 @@ +# Copyright (c) 2018 VMware, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import collections +import ipaddress +import json +import six + + +TypeNullabilityTuple = collections.namedtuple( + 'TypeNullabilityTuple', 'type nullable') + + +def nullable(marshal): + '''decorator to make marshal function accept None value''' + def func(cls, value): + if value is None: + return None + else: + return marshal(cls, value) + return func + + +class UnqualifiedNameStr(abc.ABCMeta): + '''metaclass to make str(Type) == Type''' + def __str__(self): + return self.__name__ + + +@six.add_metaclass(UnqualifiedNameStr) +class CongressDataType(object): + @classmethod + @abc.abstractmethod + def marshal(cls, value): + '''Validate a value as valid for this type. + + :Raises ValueError: if the value is not valid for this type + ''' + raise NotImplementedError + + @classmethod + def least_ancestor(cls, target_types): + '''Find this type's least ancestor among target_types + + This method helps a data consumer find the least common ancestor of + this type among the types the data consumer supports. + + :param supported_types: iterable collection of types + :returns: the subclass of CongressDataType which is the least ancestor + ''' + target_types = frozenset(target_types) + current_class = cls + try: + while current_class not in target_types: + current_class = current_class._get_parent() + return current_class + except cls.CongressDataTypeNoParent: + return None + + @classmethod + def convert_to_ancestor(cls, value, ancestor_type): + '''Convert this type's exchange value to ancestor_type's exchange value + + Generally there is no actual conversion because descendant type value + is directly interpretable as ancestor type value. The only exception + is the conversion from non-string descendents to string. This + conversion is needed by Agnostic engine does not support boolean. + + .. warning:: undefined behavior if ancestor_type is not an ancestor of + this type. + ''' + if ancestor_type == Str: + return json.dumps(value) + else: + if cls.least_ancestor([ancestor_type]) is None: + raise cls.CongressDataTypeHierarchyError + else: + return value + + @classmethod + def _get_parent(cls): + congress_parents = [parent for parent in cls.__bases__ + if issubclass(parent, CongressDataType)] + if len(congress_parents) == 1: + return congress_parents[0] + elif len(congress_parents) == 0: + raise cls.CongressDataTypeNoParent( + 'No parent type found for {0}'.format(cls)) + else: + raise cls.CongressDataTypeHierarchyError( + 'More than one parent type found for {0}: {1}' + .format(cls, congress_parents)) + + class CongressDataTypeNoParent(TypeError): + pass + + class CongressDataTypeHierarchyError(TypeError): + pass + + +class Scalar(CongressDataType): + '''Most general type, emcompassing all JSON scalar values''' + + ACCEPTED_VALUE_TYPES = [ + six.string_types, six.text_type, six.integer_types, float, bool] + + @classmethod + @nullable + def marshal(cls, value): + for type in cls.ACCEPTED_VALUE_TYPES: + if isinstance(value, type): + return value + raise ValueError('Input value (%s) is of %s instead of one of the ' + 'expected types %s' + % (value, type(value), cls.ACCEPTED_VALUE_TYPES)) + + +class Str(Scalar): + + @classmethod + @nullable + def marshal(cls, value): + if not isinstance(value, six.string_types): + raise ValueError('Input value (%s) is of %s instead of expected %s' + % (value, type(value), six.string_types)) + return value + + +class Bool(Scalar): + + @classmethod + @nullable + def marshal(cls, value): + if not isinstance(value, bool): + raise ValueError('Input value (%s) is of %s instead of expected %s' + % (value, type(value), bool)) + return value + + +class Int(Scalar): + + @classmethod + @nullable + def marshal(cls, value): + if isinstance(value, int): + return value + elif isinstance(value, float) and value.is_integer(): + return int(value) + else: + raise ValueError('Input value (%s) is of %s instead of expected %s' + ' or %s' % (value, type(value), int, float)) + + +class Float(Scalar): + + @classmethod + @nullable + def marshal(cls, value): + if isinstance(value, float): + return value + elif isinstance(value, int): + return float(value) + else: + raise ValueError('Input value (%s) is of %s instead of expected %s' + ' or %s' % (value, type(value), int, float)) + + +class IPAddress(Str): + + @classmethod + @nullable + def marshal(cls, value): + try: + return str(ipaddress.IPv4Address(value)) + except ipaddress.AddressValueError: + try: + ipv6 = ipaddress.IPv6Address(value) + if ipv6.ipv4_mapped: + return str(ipv6.ipv4_mapped) + else: + return str(ipv6) + except ipaddress.AddressValueError: + raise ValueError('Input value (%s) is not interprable ' + 'as an IP address' % value) + + +@six.add_metaclass(abc.ABCMeta) +class CongressTypeFiniteDomain(object): + '''Abstract base class for a Congress type of bounded domain. + + Each type inheriting from this class must have a class variable DOMAIN + which is a frozenset of the set of values allowed in the type. + ''' + pass + + +def create_congress_str_enum_type(class_name, enum_items): + '''Return a sub-type of CongressStr + + representing a string from a fixed, finite domain. + ''' + + for item in enum_items: + if not isinstance(item, six.string_types): + raise ValueError + + class NewType(Str, CongressTypeFiniteDomain): + DOMAIN = frozenset(enum_items) + + @classmethod + @nullable + def marshal(cls, value): + if value not in cls.DOMAIN: + raise ValueError( + 'Input value (%s) is not in the expected domain of values ' + '%s' % (value, cls.DOMAIN)) + return value + + NewType.__name__ = class_name + return NewType + + +NetworkDirection = create_congress_str_enum_type( + 'NetworkDirection', ('ingress', 'egress')) + +TYPES = [Scalar, Str, Bool, Int, Float, IPAddress] + +TYPE_NAME_TO_TYPE_CLASS = {str(type_obj): type_obj for type_obj in TYPES} diff --git a/congress/datalog/compile.py b/congress/datalog/compile.py index 639bb2cf8..b55d2c533 100644 --- a/congress/datalog/compile.py +++ b/congress/datalog/compile.py @@ -27,6 +27,7 @@ from six.moves import range from oslo_log import log as logging from oslo_utils import uuidutils +from congress import data_types from congress.datalog import analysis from congress.datalog import base from congress.datalog import builtin @@ -82,8 +83,8 @@ class Schema(object): def __contains__(self, tablename): return tablename in self.map - @classmethod - def col(self, cols): + @staticmethod + def _col(cols): # For Datasource tables, columns would be in the format - # {'name': 'colname', 'desc': 'description'} if len(cols) and isinstance(cols[0], dict): @@ -91,6 +92,20 @@ class Schema(object): else: return [x for x in cols] + @staticmethod + def _type(cols): + # For Datasource tables, columns would be in the format - + # {'name': 'colname', 'desc': 'description', + # 'type': 'typename', 'nullable': True/False} + if len(cols) and isinstance(cols[0], dict): + return [data_types.TypeNullabilityTuple( + data_types.TYPE_NAME_TO_TYPE_CLASS.get( + x.get('type', str(data_types.Scalar))), + x.get('nullable', True)) for x in cols] + else: + return [data_types.TypeNullabilityTuple(data_types.Scalar, True) + for x in cols] + def columns(self, tablename): """Returns the list of column names for the given TABLENAME. @@ -99,7 +114,17 @@ class Schema(object): if tablename not in self.map.keys(): return cols = self.map[tablename] - return Schema.col(cols) + return Schema._col(cols) + + def types(self, tablename): + """Returns the list of column names for the given TABLENAME. + + Return None if the tablename's columns are unknown. + """ + if tablename not in self.map.keys(): + return + cols = self.map[tablename] + return Schema._type(cols) def arity(self, tablename): """Returns the number of columns for the given TABLENAME. diff --git a/congress/datasources/datasource_driver.py b/congress/datasources/datasource_driver.py index 9c3961bf0..c0468143d 100644 --- a/congress/datasources/datasource_driver.py +++ b/congress/datasources/datasource_driver.py @@ -113,7 +113,7 @@ class DataSourceDriver(data_service.DataService): specified with a sub-translator, that value is included as a column in the top-level translator's table. - Using both parent-key and id-col at the same time is redudant, so + Using both parent-key and id-col at the same time is redundant, so DataSourceDriver will reject that configuration. The example translator expects an object such as: @@ -268,6 +268,8 @@ class DataSourceDriver(data_service.DataService): IN_LIST = 'in-list' OBJECTS_EXTRACT_FN = 'objects-extract-fn' DESCRIPTION = 'desc' + DATA_TYPE = 'data-type' + NULLABLE = 'nullable' # Name of the column name and desc when using a parent key. PARENT_KEY_COL_NAME = 'parent_key' @@ -284,7 +286,7 @@ class DataSourceDriver(data_service.DataService): LIST_PARAMS = (TRANSLATION_TYPE, TABLE_NAME, PARENT_KEY, ID_COL, VAL_COL, TRANSLATOR, PARENT_COL_NAME, OBJECTS_EXTRACT_FN, PARENT_KEY_DESC, VAL_COL_DESC) - VALUE_PARAMS = (TRANSLATION_TYPE, EXTRACT_FN) + VALUE_PARAMS = (TRANSLATION_TYPE, EXTRACT_FN, DATA_TYPE, NULLABLE) TRANSLATION_TYPE_PARAMS = (TRANSLATION_TYPE,) VALID_TRANSLATION_TYPES = (HDICT, VDICT, LIST, VALUE) @@ -463,7 +465,7 @@ class DataSourceDriver(data_service.DataService): self._table_deps[translator[self.TABLE_NAME]] = related_tables self._validate_translator(translator, related_tables) self._translators.append(translator) - self._schema.update(self._get_schema(translator, {})) + self._schema.update(self._get_schema(translator, {}).schema) def get_translator(self, translator_name): """Get a translator. @@ -488,12 +490,16 @@ class DataSourceDriver(data_service.DataService): """ return self._translators + SCHEMA_RETURN_TUPLE = collections.namedtuple('SchemaReturnTuple', + 'schema id_type') + @classmethod - def _get_schema_hdict(cls, translator, schema): + def _get_schema_hdict(cls, translator, schema, parent_key_type=None): tablename = translator[cls.TABLE_NAME] parent_key = translator.get(cls.PARENT_KEY, None) id_col = translator.get(cls.ID_COL, None) field_translators = translator[cls.FIELD_TRANSLATORS] + parent_col_name = None columns = [] # columns here would be list of dictionaries. @@ -504,25 +510,61 @@ class DataSourceDriver(data_service.DataService): parent_col_name = translator.get(cls.PARENT_COL_NAME, cls.PARENT_KEY_COL_NAME) desc = translator.get(cls.PARENT_KEY_DESC) - columns.append(ds_utils.add_column(parent_col_name, desc)) + columns.append(ds_utils.add_column( + parent_col_name, desc, type=parent_key_type)) - for field_translator in field_translators: + # Sort with fields lacking parent-key coming first so that the + # subtranslators that need a parent field will be able to get them + # from the fields processed first + + field_translators_with_order = [ + (index, trans) for index, trans in enumerate(field_translators)] + field_translators_sorted = sorted( + field_translators_with_order, key=cmp_to_key( + cls._compare_tuple_by_subtranslator)) + + columns_indexed = {} + + def get_current_table_col_type(name): + if parent_col_name and parent_col_name == name: + return parent_key_type + elif name == cls._id_col_name(id_col): + return None # FIXME(ekcs): return type for ID col + else: + [type] = [column_schema.get('type') for column_schema in + columns_indexed.values() + if column_schema.get('name') == name] + return type + + for (index, field_translator) in field_translators_sorted: col = field_translator.get( cls.COL, field_translator[cls.FIELDNAME]) desc = field_translator.get(cls.DESCRIPTION) subtranslator = field_translator[cls.TRANSLATOR] - if cls.PARENT_KEY not in subtranslator: - columns.append(ds_utils.add_column(col, desc)) - cls._get_schema(subtranslator, schema) + if cls.PARENT_KEY in subtranslator: + # TODO(ekcs): disallow nullable parent key + cls._get_schema(subtranslator, schema, + parent_key_type=get_current_table_col_type( + subtranslator[cls.PARENT_KEY])) + else: + field_type = subtranslator.get(cls.DATA_TYPE) + nullable = subtranslator.get(cls.NULLABLE, True) + columns_indexed[index] = ds_utils.add_column( + col, desc, field_type, nullable) + cls._get_schema(subtranslator, schema) + + for index in range(0, len(field_translators)): + if index in columns_indexed: + columns.append(columns_indexed[index]) if tablename in schema: raise exception.InvalidParamException( "table %s already in schema" % tablename) schema[tablename] = tuple(columns) - return schema + return cls.SCHEMA_RETURN_TUPLE(schema, None) @classmethod - def _get_schema_vdict(cls, translator, schema): + def _get_schema_vdict(cls, translator, schema, parent_key_type=None): tablename = translator[cls.TABLE_NAME] parent_key = translator.get(cls.PARENT_KEY, None) id_col = translator.get(cls.ID_COL, None) @@ -546,10 +588,10 @@ class DataSourceDriver(data_service.DataService): new_schema = new_schema + (value_col,) schema[tablename] = new_schema - return schema + return cls.SCHEMA_RETURN_TUPLE(schema, None) @classmethod - def _get_schema_list(cls, translator, schema): + def _get_schema_list(cls, translator, schema, parent_key_type=None): tablename = translator[cls.TABLE_NAME] parent_key = translator.get(cls.PARENT_KEY, None) id_col = translator.get(cls.ID_COL, None) @@ -572,31 +614,36 @@ class DataSourceDriver(data_service.DataService): ds_utils.add_column(value_col, val_desc)) else: schema[tablename] = (ds_utils.add_column(value_col, val_desc), ) - return schema + return cls.SCHEMA_RETURN_TUPLE(schema, None) @classmethod - def _get_schema(cls, translator, schema): - """Returns the schema of a translator. + def _get_schema(cls, translator, schema, parent_key_type=None): + """Returns named tuple with values: + + schema: the schema of a translator, + id_type: the data type of the id-col, or None of absent Note: this method uses the argument schema to store data in since this method words recursively. It might be worthwhile in the future to refactor this code so this is not required. + + :param parent_key_type: passes down the column data type which the + translator refers to as parent-key """ cls.check_translation_type(translator.keys()) translation_type = translator[cls.TRANSLATION_TYPE] if translation_type == cls.HDICT: - cls._get_schema_hdict(translator, schema) + return cls._get_schema_hdict(translator, schema, parent_key_type) elif translation_type == cls.VDICT: - cls._get_schema_vdict(translator, schema) + return cls._get_schema_vdict(translator, schema, parent_key_type) elif translation_type == cls.LIST: - cls._get_schema_list(translator, schema) + return cls._get_schema_list(translator, schema, parent_key_type) elif translation_type == cls.VALUE: - pass + return cls.SCHEMA_RETURN_TUPLE(schema, None) else: raise AssertionError('Unexpected translator type %s' % translation_type) - return schema @classmethod def get_schema(cls): @@ -745,17 +792,25 @@ class DataSourceDriver(data_service.DataService): return h @classmethod - def _extract_value(cls, obj, extract_fn): + def _extract_value(cls, obj, extract_fn, data_type, nullable=True): # Reads a VALUE object and returns (result_rows, h) if extract_fn is None: extract_fn = lambda x: x value = extract_fn(obj) # preserve type if possible; convert to str if not Hashable - if isinstance(value, collections.Hashable): - return value - else: - return str(value) + if not isinstance(value, collections.Hashable): + value = str(value) + + # check that data type matches if specified in translator + if data_type is not None and value is not None: + value = data_type.marshal(value) + + return value + + @classmethod + def _compare_tuple_by_subtranslator(cls, x, y): + return cls._compare_subtranslator(x[1], y[1]) @classmethod def _compare_subtranslator(cls, x, y): @@ -779,8 +834,9 @@ class DataSourceDriver(data_service.DataService): if subtrans[cls.TRANSLATION_TYPE] == cls.VALUE: extract_fn = subtrans.get(cls.EXTRACT_FN, None) - converted_values = tuple([cls._extract_value(o, extract_fn) - for o in obj]) + data_type = subtrans.get(cls.DATA_TYPE) + converted_values = tuple( + [cls._extract_value(o, extract_fn, data_type) for o in obj]) if id_col: h = cls._compute_id(id_col, obj, converted_values) new_tuples = [(table, (h, v)) for v in converted_values] @@ -844,8 +900,10 @@ class DataSourceDriver(data_service.DataService): if subtrans[cls.TRANSLATION_TYPE] == cls.VALUE: extract_fn = subtrans.get(cls.EXTRACT_FN, None) - converted_items = tuple([(k, cls._extract_value(v, extract_fn)) - for k, v in obj.items()]) + data_type = subtrans.get(cls.DATA_TYPE) + converted_items = tuple( + [(k, cls._extract_value(v, extract_fn, data_type)) + for k, v in obj.items()]) if id_col: h = cls._compute_id(id_col, obj, converted_items) new_tuples = [(table, (h,) + i) for i in converted_items] @@ -932,9 +990,18 @@ class DataSourceDriver(data_service.DataService): subtranslator = field_translator[cls.TRANSLATOR] if subtranslator[cls.TRANSLATION_TYPE] == cls.VALUE: extract_fn = subtranslator.get(cls.EXTRACT_FN) - v = cls._extract_value( - cls._get_value(obj, field, selector), extract_fn) - hdict_row[col_name] = v + data_type = subtranslator.get(cls.DATA_TYPE) + nullable = subtranslator.get(cls.NULLABLE, True) + try: + v = cls._extract_value( + cls._get_value(obj, field, selector), + extract_fn, data_type, nullable) + hdict_row[col_name] = v + except TypeError as exc: + arg0 = "While translating field: %s, column: %s; " \ + "%s" % (field, col_name, exc.args[0]) + exc.args = tuple([arg0]) + exc.args[1:] + raise else: assert translator[cls.TRANSLATION_TYPE] in (cls.HDICT, cls.VDICT, diff --git a/congress/datasources/datasource_utils.py b/congress/datasources/datasource_utils.py index 6488c8c21..18dcfa118 100644 --- a/congress/datasources/datasource_utils.py +++ b/congress/datasources/datasource_utils.py @@ -63,9 +63,14 @@ def update_state_on_changed(root_table_name): return outer -def add_column(colname, desc=None): +def add_column(colname, desc=None, type=None, nullable=True): """Adds column in the form of dict.""" - return {'name': colname, 'desc': desc} + col_dict = {'name': colname, 'desc': desc} + if type is not None: + col_dict['type'] = str(type) + if not nullable: + col_dict['nullable'] = False + return col_dict def inspect_methods(client, api_prefix): diff --git a/congress/tests/datalog/test_compiler.py b/congress/tests/datalog/test_compiler.py index 49603cfbd..cf2481e9f 100644 --- a/congress/tests/datalog/test_compiler.py +++ b/congress/tests/datalog/test_compiler.py @@ -19,6 +19,7 @@ from __future__ import absolute_import import copy +from congress import data_types from congress.datalog import analysis from congress.datalog import base as datalogbase from congress.datalog import compile @@ -972,3 +973,22 @@ class TestDependencyGraph(base.TestCase): self.assertEqual(set(g.tables_with_modal('execute')), set()) g.undo_changes(chgs) self.assertEqual(set(g.tables_with_modal('execute')), set(['p'])) + + +class TestSchema(base.TestCase): + + def test_schema_columns(self): + test_schema = compile.Schema({ + 'p': (1, 2, 3), + 'q': ({'name': 'a', 'type': 'Str'}, + {'name': 'b', 'nullable': False})}, + complete=True) + self.assertEqual(test_schema.columns('p'), + [1, 2, 3]) + self.assertEqual(test_schema.columns('q'), + ['a', 'b']) + self.assertEqual([(data_types.Scalar, True), (data_types.Scalar, True), + (data_types.Scalar, True)], + test_schema.types('p')) + self.assertEqual([(data_types.Str, True), (data_types.Scalar, False)], + test_schema.types('q')) diff --git a/congress/tests/datasources/test_datasource_driver.py b/congress/tests/datasources/test_datasource_driver.py index 7214129f8..6ebe83753 100644 --- a/congress/tests/datasources/test_datasource_driver.py +++ b/congress/tests/datasources/test_datasource_driver.py @@ -24,6 +24,7 @@ import eventlet import mock from oslo_utils import uuidutils +from congress import data_types from congress.datasources import datasource_driver from congress.datasources import datasource_utils from congress.db import db_ds_table_data @@ -40,6 +41,9 @@ class TestDatasourceDriver(base.TestCase): super(TestDatasourceDriver, self).setUp() self.val_trans = {'translation-type': 'VALUE'} + def typed_value_trans(self, type): + return {'translation-type': 'VALUE', 'data-type': type} + def compute_hash(self, obj): s = json.dumps(sorted(obj, key=(lambda x: str(type(x)) + repr(x))), sort_keys=True) @@ -72,6 +76,16 @@ class TestDatasourceDriver(base.TestCase): self.assertEqual(params, expected) def test_in_list_results_hdict_hdict(self): + class Type1(object): + @classmethod + def marshal(cls, value): + return value + + class Type2(object): + @classmethod + def marshal(cls, value): + return value + ports_fixed_ips_translator = { 'translation-type': 'HDICT', 'table-name': 'fixed-ips', @@ -79,7 +93,8 @@ class TestDatasourceDriver(base.TestCase): 'selector-type': 'DICT_SELECTOR', 'in-list': True, 'field-translators': - ({'fieldname': 'ip_address', 'translator': self.val_trans}, + ({'fieldname': 'ip_address', + 'translator': self.typed_value_trans(Type2)}, {'fieldname': 'subnet_id', 'translator': self.val_trans})} ports_translator = { @@ -87,7 +102,8 @@ class TestDatasourceDriver(base.TestCase): 'table-name': 'ports', 'selector-type': 'DICT_SELECTOR', 'field-translators': - ({'fieldname': 'id', 'translator': self.val_trans}, + ({'fieldname': 'id', + 'translator': self.typed_value_trans(Type2)}, {'fieldname': 'fixed_ips', 'translator': ports_fixed_ips_translator})} @@ -1093,14 +1109,17 @@ class TestDatasourceDriver(base.TestCase): 'field-translators': ( {'fieldname': 'a', 'col': 'a1', - 'translator': self.val_trans}, + 'translator': + self.typed_value_trans( + data_types.Bool)}, {'fieldname': 'b', 'col': 'b1', 'translator': self.val_trans})}}, {'fieldname': 'testfield2', 'translator': {'translation-type': 'HDICT', 'table-name': 'subtable2', - 'id-col': 'id2', + 'parent-key': 'zparent_col3', + 'parent-col-name': 'id2', 'field-translators': ( {'fieldname': 'c', 'col': 'c1', @@ -1109,7 +1128,7 @@ class TestDatasourceDriver(base.TestCase): 'col': 'd1', 'translator': self.val_trans})}}, {'fieldname': 'ztestfield3', 'col': 'zparent_col3', - 'translator': self.val_trans}, + 'translator': self.typed_value_trans(data_types.Str)}, {'fieldname': 'testfield4', 'col': 'parent_col4', 'translator': {'translation-type': 'VALUE', 'extract-fn': lambda x: x.id}}, @@ -1143,9 +1162,9 @@ class TestDatasourceDriver(base.TestCase): self.assertEqual(7, len(schema)) self.assertEqual(({'name': 'id1', 'desc': None}, - {'name': 'a1', 'desc': None}, + {'name': 'a1', 'desc': None, 'type': 'Bool'}, {'name': 'b1', 'desc': None}), schema['subtable1']) - self.assertEqual(({'name': 'id2', 'desc': None}, + self.assertEqual(({'name': 'id2', 'desc': None, 'type': 'Str'}, {'name': 'c1', 'desc': None}, {'name': 'd1', 'desc': None}), schema['subtable2']) self.assertEqual(('id3', 'key3', 'value3'), schema['subtable3']) @@ -1157,8 +1176,7 @@ class TestDatasourceDriver(base.TestCase): 'desc': None},), schema['subtable6']) self.assertEqual( ({'name': 'parent_col1', 'desc': None}, - {'name': 'testfield2', 'desc': None}, - {'name': 'zparent_col3', 'desc': None}, + {'name': 'zparent_col3', 'desc': None, 'type': 'Str'}, {'name': 'parent_col4', 'desc': None}, {'name': 'parent_col5', 'desc': None}, {'name': 'parent_col6', 'desc': None}, @@ -1218,6 +1236,36 @@ class TestDatasourceDriver(base.TestCase): ({'name': 'parent_key', 'desc': None}, {'name': 'val', 'desc': None}), schema['subtable']) + def test_get_schema_with_hdict_parent_and_id_col_in_subtranslator(self): + class TestDriver(datasource_driver.DataSourceDriver): + subtranslator = {'translation-type': 'LIST', + 'table-name': 'subtable', + 'id-col': 'id', 'val-col': 'val', + 'translator': self.val_trans} + + translator = {'translation-type': 'HDICT', + 'table-name': 'testtable', + 'selector-type': 'DICT_SELECTOR', + 'field-translators': ({'fieldname': 'unique_key', + 'translator': self.val_trans}, + {'fieldname': 'sublist', + 'translator': subtranslator})} + + TRANSLATORS = [translator] + + def __init__(self): + super(TestDriver, self).__init__('', None) + + schema = TestDriver().get_schema() + + self.assertEqual(2, len(schema)) + self.assertEqual( + ({'desc': None, 'name': 'unique_key'}, + {'desc': None, 'name': 'sublist'}), schema['testtable']) + self.assertEqual( + ({'desc': None, 'name': 'id'}, + {'desc': None, 'name': 'val'}), schema['subtable']) + def test_get_schema_with_hdict_id_function(self): class TestDriver(datasource_driver.DataSourceDriver): translator = { diff --git a/congress/tests/policy_engines/test_agnostic_dse2.py b/congress/tests/policy_engines/test_agnostic_dse2.py index d83dda67a..e0eeaea68 100644 --- a/congress/tests/policy_engines/test_agnostic_dse2.py +++ b/congress/tests/policy_engines/test_agnostic_dse2.py @@ -48,9 +48,7 @@ class TestDseRuntime(base.SqlTestCase): node.invoke_service_rpc = mock.MagicMock() node.invoke_service_rpc.return_value = [ - ['id1', 'name1', 'status1'], - ['id2', 'name2', 'status2'], - ] + 0, [['id1', 'name1', 'status1'], ['id2', 'name2', 'status2']]] # loaded rule is disabled subscriptions = engine2.subscription_list() diff --git a/congress/tests/test_data_types.py b/congress/tests/test_data_types.py new file mode 100644 index 000000000..dc402f4a8 --- /dev/null +++ b/congress/tests/test_data_types.py @@ -0,0 +1,31 @@ +# Copyright (c) 2018 VMware +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import + +import testtools + +from congress import data_types + + +class TestDataTypes(testtools.TestCase): + + def test_congress_str_nullable(self): + self.assertEqual(data_types.Str.marshal('test-str-value'), + 'test-str-value') + self.assertIsNone(data_types.Str.marshal(None)) + self.assertRaises(ValueError, data_types.Str.marshal, True)