From e8a1ece4f4021cda8b5e9ec4fd181a169ffd7a97 Mon Sep 17 00:00:00 2001 From: Shachar Snapiri Date: Wed, 29 Nov 2017 13:36:10 +0200 Subject: [PATCH] Add JsonSchema printer Added OpenApiSchema json printer Change-Id: I2cfcaf1d2fbe87ea80362f620b12596c26d52d5e Partial-Bug: #1734146 --- dragonflow/cli/df_model.py | 324 +++++++++++++++--- dragonflow/db/model_framework.py | 17 +- dragonflow/tests/unit/test_model_framework.py | 13 + 3 files changed, 291 insertions(+), 63 deletions(-) diff --git a/dragonflow/cli/df_model.py b/dragonflow/cli/df_model.py index 5724d3f15..2ce6590a4 100644 --- a/dragonflow/cli/df_model.py +++ b/dragonflow/cli/df_model.py @@ -15,115 +15,194 @@ from __future__ import print_function import abc import argparse import contextlib -import re import six import sys from jsonmodels import fields +from oslo_serialization import jsonutils from dragonflow.db import field_types from dragonflow.db import model_framework from dragonflow.db.models import all # noqa +STRING_TYPE = 'string' +NUMBER_TYPE = 'number' +FLOAT_TYPE = 'float' +BOOL_TYPE = 'boolean' +ENUM_TYPE = 'enum' +BASIC_TYPES = (STRING_TYPE, NUMBER_TYPE, FLOAT_TYPE, BOOL_TYPE, ENUM_TYPE) + + @six.add_metaclass(abc.ABCMeta) class ModelsPrinter(object): + """Abstract base class for the different format printers. + + Every specific format printer should inherit from this class and + implement the methods for the specific output format. + + All the models are handled one after the other, and it is guaranteed + that if a model depends on another model, the dependant model will + be handled _after_ the model it depends on. + """ + def __init__(self, fh): + """Basic constructor for the base class. + + :param fh: file handler of the output stream to write to + :type fh: file object + """ self._output = fh def _print(self, *args, **kwargs): print(*args, file=self._output, **kwargs) def output_start(self): - """ - Called once on the beginning of the processing. - Should be used for initializations of any kind + """Handler called once before any processing is done. + + This should be used for initialization or to print any prefix the + specific output format requires. """ pass def output_end(self): - """ - Called once on the end of the processing. - Should be used for cleanup and leftover printing of any kind + """Handler called once after all processing is done. + + This should be used for cleanup or to print any remaining data or + suffixes the specific output format requires. """ pass def model_start(self, model_name): - """ - Called once for every model, before any field. + """Handler called once per model, before processing the model. + + This should be used to clean/initialize any data specific for the + handling of the model. + + :param model_name: the name of the model + :type model_name: string """ pass def model_end(self, model_name): - """ - Called once for every model, after all model processing is done. + """Handler called once per model, after processing the model. + + This should be used to cleanup or to print any remaining data or + suffixes specific for the handling of the model. + + :param model_name: the name of the model + :type model_name: string """ pass def fields_start(self): - """ - Called once for every model, before all fields. + """Handler called once per model, before processing the fields. + + This should be used to initialize any data specific for the + handling of the model fields. """ pass def fields_end(self): - """ - Called once for every model, after all fields. + """Handler called once per model, after processing all the fields. + + This should be used to cleanup any data specific for the handling + of the model fields. """ pass @abc.abstractmethod def handle_field(self, field_name, field_type, is_required, is_embedded, is_single, restrictions): - """ - Called once for every field in a model. + """Handler called once per field in a model. + + This should be used to print the specific field. + :param field_name: the name of the field + :type field_name: string + :param field_type: string - the type of the field (e.g. number) + :type field_type: string + :param is_required: True iff the field is a required field for the + specific model + :type is_required: bool + :param is_embedded: True iff the field is not a reference to another + object, but rather an object that is part of the model + :type is_embedded: bool + :param is_single: True iff the field is single-valued + :type is_single: bool + :param restrictions: representation of any restrictions on the field + (e.g. list of possible values for enum) + :type restrictions: string """ pass def indexes_start(self): - """ - Called once for every model, before all indexes. - Not called if no indexes exist + """Handler called once per model, before processing the indexes. + + The indexes are fields that are marked as indexes, thus are a subset + of the list of model indexes. + This should be used to initialize any data specific for the + handling of the model indexes. + In case there are no indexes for this model, this method will not be + called. """ pass def indexes_end(self): - """ - Called once for every model, after all indexes. - Not called if no indexes exist + """Handler called once per model, after processing all the indexes. + + This should be used to cleanup any data specific for the handling + of the model indexes. + In case there are no indexes for this model, this method will not be + called. """ pass @abc.abstractmethod def handle_index(self, index_name): - """ - Called once for every index in a model. + """Handler called once per index in a model. + + This should be used to print the specific index. + :param index_name: the name of the index field + :type index_name: string """ pass def events_start(self): - """ - Called once for every model, before all events. - Not called if no events exist + """Handler called once per model, before processing the events. + + This should be used to initialize any data specific for the + handling of the model events. + In case there are no events for this model, this method will not be + called. """ pass def events_end(self): - """ - Called once for every model, after all events. - Not called if no events exist + """Handler called once per model, after processing all the events. + + This should be used to cleanup any data specific for the handling + of the model events. + In case there are no events for this model, this method will not be + called. """ pass @abc.abstractmethod def handle_event(self, event_name): - """ - Called once for every event in a model. + """Handler called once per event in a model. + + This should be used to print the specific event. + :param event_name: the name of the event + :type event_name: string """ pass class PlaintextPrinter(ModelsPrinter): + """ModelPrinter that prints to simple plaintext format. + + This printer prints the models in the most simple way. + """ def __init__(self, fh): super(PlaintextPrinter, self).__init__(fh) @@ -142,11 +221,10 @@ class PlaintextPrinter(ModelsPrinter): def handle_field(self, field_name, field_type, is_required, is_embedded, is_single, restrictions): restriction_str = ' {}'.format(restrictions) if restrictions else '' - print('{name} : {type}{restriction}{required}{to_many}'.format( - name=field_name, type=field_type, - restriction=restriction_str, + self._print('{name} : {type}{restriction}{required}{multi}'.format( + name=field_name, type=field_type, restriction=restriction_str, required=', Required' if is_required else '', - to_many=', Multi' if not is_single else '', + multi=', Multi' if not is_single else '', embedded=', Embedded' if is_embedded else '')) def indexes_start(self): @@ -165,7 +243,10 @@ class PlaintextPrinter(ModelsPrinter): class UMLPrinter(ModelsPrinter): - """PlantUML format printer""" + """ModelPrinter that prints to UML format. + + This printer prints the models in PlantUML format. + """ def __init__(self, fh): super(UMLPrinter, self).__init__(fh) self._model = '' @@ -205,7 +286,7 @@ class UMLPrinter(ModelsPrinter): restriction_str = ' {}'.format(restrictions) if restrictions else '' name = '{}'.format(field_name) if is_required else field_name self._print(' +{name} : {type} {restriction}'.format( - name=name, type=field_type, restriction=restriction_str)) + name=name, type=field_type, restriction=restriction_str)) self._dependencies.add((self._model, field_type, field_name, is_single, is_embedded)) @@ -222,30 +303,135 @@ class UMLPrinter(ModelsPrinter): self._print(' {}'.format(event_name)) -class DfModelParser(object): +class OASPrinter(ModelsPrinter): + """ModelPrinter that prints to JSON format + + This printer prints the models in JSON format. + Specifically, it uses the OpenApiSchema format. + """ + _OPENAPI_VERSION = '3.0.0' + _MODEL_SCHEMA_VERSION = '0.0.1' + _SCHEMA_BASE_PATH = '#/components/schemas' + _INFO_TITLE = 'DragonFlow Schema' + _INFO_DESC = 'jsonschma representation of the DragonFlow model' + _LIC_NAME = 'Apache 2.0' + _LIC_URL = 'http://www.apache.org/licenses/LICENSE-2.0.html' + + def __init__(self, fh): + super(OASPrinter, self).__init__(fh) + self._required = list() + self._base_types = BASIC_TYPES + self._models_obj = dict() + self._model = dict() + + def output_start(self): + info = dict() + license = dict() + paths = dict() + schemas = dict() + components = dict() + self._models_obj['openapi'] = OASPrinter._OPENAPI_VERSION + self._models_obj['info'] = info + info['title'] = OASPrinter._INFO_TITLE + info['description'] = OASPrinter._INFO_DESC + info['license'] = license + license['name'] = OASPrinter._LIC_NAME + license['url'] = OASPrinter._LIC_URL + info['version'] = OASPrinter._MODEL_SCHEMA_VERSION + self._models_obj['paths'] = paths + self._models_obj['components'] = components + components['schemas'] = schemas + + def output_end(self): + jsonutils.dump(self._models_obj, self._output, indent=2) + + def model_start(self, model_name): + self._required = list() + self._model = dict() + self._models_obj['components']['schemas'][model_name] = self._model + self._model['type'] = 'object' + + def model_end(self, model_name): + if len(self._required) > 0: + self._model['required'] = self._required + + def fields_start(self): + self._model['properties'] = dict() + + def fields_end(self): + pass + + def _simple_field(self, field_type, restrictions): + if field_type in self._base_types: + return {'type': field_type} + elif field_type == ENUM_TYPE: + return {field_type: list(restrictions)} + else: + return {'$ref': '{}/{}'.format(OASPrinter._SCHEMA_BASE_PATH, + field_type)} + + def _array_field(self, field_type, restrictions): + return {'items': self._simple_field(field_type, restrictions), + 'type': 'array'} + + def handle_field(self, field_name, field_type, is_required, is_embedded, + is_single, restrictions): + flds = self._model['properties'] + if is_single: + flds[field_name] = self._simple_field(field_type, restrictions) + else: + flds[field_name] = self._array_field(field_type, restrictions) + if is_required: + self._required.append(field_name) + + def handle_index(self, index_name): + pass + + def handle_event(self, event_name): + pass + + +class DfModelsParser(object): + """Parser for the Dragonflow models schema + + This parser iterates over the Dragonflow models by their dependency order + (models that others depend on will be before the dependant ones). + It uses a ModelsPrinter to actually print the information to the supported + formats. + """ def __init__(self, printer): + """Constructor for the DfModelsParser + + It initializes the internal structures before the actual parsing + + :param printer: instance of ModelsPrinter + :type printer: ModelsPrinter + """ self._printer = printer + self._basic_types = BASIC_TYPES + self._processed_models = set() + self._all_models = set() def _stringify_field_type(self, field): if field in six.string_types: - return 'string', None + return STRING_TYPE, None elif isinstance(field, field_types.EnumField): - field_type = 'enum' + field_type = ENUM_TYPE restrictions = list(field._valid_values) return field_type, restrictions elif isinstance(field, field_types.ReferenceField): model = field._model return model.__name__, None elif isinstance(field, fields.StringField): - return 'string', None + return STRING_TYPE, None elif isinstance(field, fields.IntField): - return 'number', None + return NUMBER_TYPE, None elif isinstance(field, fields.FloatField): - return 'float', None + return FLOAT_TYPE, None elif isinstance(field, fields.BoolField): - return 'boolean', None + return BOOL_TYPE, None elif isinstance(field, fields.BaseField): - return type(field).__name__, None + return STRING_TYPE, None else: return field.__name__, None @@ -254,25 +440,33 @@ class DfModelParser(object): is_single = False is_embedded = not isinstance(field, field_types.ReferenceListField) - field_type, restrictions = self._stringify_field_type(field.field) + field_model = field.field elif isinstance(field, fields.ListField): is_single = False is_embedded = False - field_type, restrictions = self._stringify_field_type( - field.items_types[0]) + field_model = field.items_types[0] if isinstance(field, field_types.EnumListField): restrictions = list(field._valid_values) elif isinstance(field, fields.EmbeddedField): is_single = True is_embedded = True - field_type, restrictions = self._stringify_field_type( - field.types[0]) + field_model = field.types[0] else: is_single = True is_embedded = False - field_type, restrictions = self._stringify_field_type(field) + field_model = field + field_type, restrictions = self._stringify_field_type(field_model) - field_type = re.sub('Field$', '', field_type) + if field_type not in self._basic_types: + if isinstance(field_model, field_types.ReferenceField): + model = field_model._model + else: + model = field_model + self._all_models.add(model) + # As we iterate over the models by their dependencies, if we did + # not encounter this model, it is an embedded model (type) + if model not in self._processed_models: + is_embedded = True self._printer.handle_field(key, field_type, field.required, is_embedded, is_single, restrictions) @@ -301,17 +495,31 @@ class DfModelParser(object): def _process_model(self, df_model): model_name = df_model.__name__ self._printer.model_start(model_name) - self._process_fields(df_model) self._process_indexes(df_model) self._process_events(df_model) + self._printer.model_end(model_name) + self._processed_models.add(df_model) + def _process_unvisited_model(self, model): + model_name = model.__name__ + self._printer.model_start(model_name) + self._process_fields(model) self._printer.model_end(model_name) def parse_models(self): + """Iterates over the models and processes them with the printer + + This method iterates over all the models in the schema, sends them + for processing and then, makes sure the unknown models are also handled + """ self._printer.output_start() for model in model_framework.iter_models_by_dependency_order(False): self._process_model(model) + # Handle unvisited models + remaining_models = self._all_models - self._processed_models + for model in remaining_models: + self._process_unvisited_model(model) self._printer.output_end() @@ -336,15 +544,19 @@ def main(): action='store_true') group.add_argument('--uml', help='PlantUML format output', action='store_true') + group.add_argument('--json', help='OpenApiSchema JSON format output', + action='store_true') parser.add_argument('-o', '--outfile', help='Output to file (instead of stdout)') args = parser.parse_args() with smart_open(args.outfile) as fh: if args.uml: printer = UMLPrinter(fh) + elif args.json: + printer = OASPrinter(fh) else: printer = PlaintextPrinter(fh) - parser = DfModelParser(printer) + parser = DfModelsParser(printer) parser.parse_models() diff --git a/dragonflow/db/model_framework.py b/dragonflow/db/model_framework.py index 14bfe2ce1..3fb09dd2a 100644 --- a/dragonflow/db/model_framework.py +++ b/dragonflow/db/model_framework.py @@ -197,7 +197,7 @@ class _CommonBase(models.Base): return "{}({})".format(self.__class__.__name__, ", ".join(fields)) @classmethod - def dependencies(cls): + def dependencies(cls, first_class_only=True): deps = set() for key, field in cls.iterate_over_fields(): if isinstance(field, fields.ListField): @@ -210,11 +210,14 @@ class _CommonBase(models.Base): deps.add(field_type.get_proxied_model()) except AttributeError: if issubclass(field_type, ModelBase): - # If the field is not a reference, and it is a df - # model(derived from ModelBase), it is considered as - # non-first class model. And its dependency - # will be treated as current model's dependency. - deps |= field_type.dependencies() + if first_class_only: + # If the field is not a reference, and it is a df + # model(derived from ModelBase), it is considered + # as non-first class model. And its dependency + # will be treated as current model's dependency. + deps |= field_type.dependencies() + else: + deps.add(field_type) return deps @@ -443,7 +446,7 @@ def iter_models_by_dependency_order(first_class_only=True): unsorted_models = {} # Gather all models and their dependencies for model in iter_models(first_class_only=first_class_only): - dependencies = model.dependencies() + dependencies = model.dependencies(first_class_only) if first_class_only: dependencies = {dep for dep in dependencies if dep.is_first_class()} diff --git a/dragonflow/tests/unit/test_model_framework.py b/dragonflow/tests/unit/test_model_framework.py index 5f10b7b17..4dfad8613 100644 --- a/dragonflow/tests/unit/test_model_framework.py +++ b/dragonflow/tests/unit/test_model_framework.py @@ -474,3 +474,16 @@ class TestModelFramework(tests_base.BaseTestCase): sorted_models.index(ReffingModel3) ) self.assertIn(ReffedModel, ReffingModel3.dependencies()) + + def test_hierarchical_dependency_not_first_class(self): + sorted_models = mf.iter_models_by_dependency_order( + first_class_only=False) + self.assertLess( + sorted_models.index(ReffedModel), + sorted_models.index(ReffingNonFirstClassModel) + ) + self.assertLess( + sorted_models.index(ReffingNonFirstClassModel), + sorted_models.index(ReffingModel3) + ) + self.assertIn(ReffedModel, ReffingModel3.dependencies())