# Copyright 2015 - Mirantis, Inc. # Copyright 2015 - StackStorm, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import copy import json import jsonschema import re import six from mistral import exceptions as exc from mistral import expressions as expr from mistral.expressions.jinja_expression import ANY_JINJA_REGEXP from mistral.expressions.yaql_expression import INLINE_YAQL_REGEXP from mistral.lang import types from mistral import utils ACTION_PATTERNS = { "command": "[\w\.]+[^=\(\s\"]*", "yaql_expression": INLINE_YAQL_REGEXP, "jinja_expression": ANY_JINJA_REGEXP, } CMD_PTRN = re.compile( "^({})".format("|".join(six.itervalues(ACTION_PATTERNS))) ) EXPRESSION = '|'.join([expr.patterns[name] for name in expr.patterns]) _ALL_IN_BRACKETS = "\[.*\]\s*" _ALL_IN_QUOTES = "\"[^\"]*\"\s*" _ALL_IN_APOSTROPHES = "'[^']*'\s*" _DIGITS = "\d+" _TRUE = "true" _FALSE = "false" _NULL = "null" ALL = ( _ALL_IN_QUOTES, _ALL_IN_APOSTROPHES, EXPRESSION, _ALL_IN_BRACKETS, _TRUE, _FALSE, _NULL, _DIGITS ) PARAMS_PTRN = re.compile("([-_\w]+)=(%s)" % "|".join(ALL)) def instantiate_spec(spec_cls, data, validate=False): """Instantiates specification accounting for specification hierarchies. :param spec_cls: Specification concrete or base class. In case if base class or the hierarchy is provided this method relies on attributes _polymorphic_key and _polymorphic_value in order to find a concrete class that needs to be instantiated. :param data: Raw specification data as a dictionary. :type data: dict :param validate: If it equals False then semantics and schema validation will be skipped :type validate: bool """ if issubclass(spec_cls, BaseSpecList): # Ignore polymorphic search for specification lists because # it doesn't make sense for them. return spec_cls(data, validate) if not hasattr(spec_cls, '_polymorphic_key'): spec = spec_cls(data, validate) if validate: spec.validate_semantics() return spec # In order to do polymorphic search we need to make sure that # a spec is backed by a dictionary. Otherwise we can't extract # a polymorphic key. if not isinstance(data, dict): raise exc.InvalidModelException( "A specification with polymorphic key must be backed by" " a dictionary [spec_cls=%s, data=%s]" % (spec_cls, data) ) key = spec_cls._polymorphic_key if not isinstance(key, tuple): key_name = key key_default = None else: key_name = key[0] key_default = key[1] for cls in utils.iter_subclasses(spec_cls): if not hasattr(cls, '_polymorphic_value'): raise exc.DSLParsingException( "Class '%s' is expected to have attribute '_polymorphic_value'" " because it's a part of specification hierarchy inherited " "from class '%s'." % (cls, spec_cls) ) if cls._polymorphic_value == data.get(key_name, key_default): spec = cls(data, validate) if validate: spec.validate_semantics() return spec raise exc.DSLParsingException( 'Failed to find a specification class to instantiate ' '[spec_cls=%s, data=%s]' % (spec_cls, data) ) class BaseSpec(object): """Base class for all DSL specifications. It represents a DSL entity such as workflow or task as a python object providing more convenient API to analyse DSL than just working with raw data in form of a dictionary. Specification classes also implement all required validation logic by overriding instance methods 'validate_schema()' and 'validate_semantics()'. Note that the specification mechanism allows to have polymorphic entities in DSL. For example, if we find it more convenient to have separate specification classes for different types of workflow (i.e. 'direct' and 'reverse') we can do so. In this case, in order to instantiate them correctly method 'instantiate_spec' must always be used where argument 'spec_cls' must be a root class of the specification hierarchy containing class attribute '_polymorhpic_key' pointing to a key in raw data relying on which we can find a concrete class. Concrete classes then must all have attribute '_polymorhpic_value' corresponding to a value in a raw data. Attribute '_polymorhpic_key' can be either a string or a tuple of size two where the first value is a key name itself and the second value is a default polymorphic value that must be used if raw data doesn't contain a configured key at all. An example of this situation is when we don't specify a workflow type in DSL. In this case, we assume it's 'direct'. """ # See http://json-schema.org _schema = { 'type': 'object' } _meta_schema = { 'type': 'object' } _definitions = {} _version = '2.0' @classmethod def get_schema(cls, includes=['meta', 'definitions']): schema = copy.deepcopy(cls._schema) schema['properties'] = utils.merge_dicts( schema.get('properties', {}), cls._meta_schema.get('properties', {}), overwrite=False ) if includes and 'meta' in includes: schema['required'] = list( set(schema.get('required', []) + cls._meta_schema.get('required', [])) ) if includes and 'definitions' in includes: schema['definitions'] = utils.merge_dicts( schema.get('definitions', {}), cls._definitions, overwrite=False ) return schema def __init__(self, data, validate): self._data = data self._validate = validate if validate: self.validate_schema() def validate_schema(self): """Validates DSL entity schema that this specification represents. By default, this method just validate schema of DSL entity that this specification represents using "_schema" class attribute. Additionally, child classes may implement additional logic to validate more specific things like YAQL expressions in their fields. Note that this method is called before construction of specification fields and validation logic should only rely on raw data provided as a dictionary accessible through '_data' instance field. """ try: jsonschema.validate(self._data, self.get_schema()) except jsonschema.ValidationError as e: raise exc.InvalidModelException("Invalid DSL: %s" % e) def validate_semantics(self): """Validates semantics of specification object. Child classes may implement validation logic to check things like integrity of corresponding data structure (e.g. task graph) or other things that can't be expressed in JSON schema. This method is called after specification has been built (i.e. its initializer has finished it's work) so that validation logic can rely on initialized specification fields. """ pass def validate_expr(self, dsl_part): if isinstance(dsl_part, six.string_types): expr.validate(dsl_part) elif isinstance(dsl_part, (list, tuple)): for expression in dsl_part: if isinstance(expression, six.string_types): expr.validate(expression) elif isinstance(dsl_part, dict): for expression in dsl_part.values(): if isinstance(expression, six.string_types): expr.validate(expression) def _spec_property(self, prop_name, spec_cls): prop_val = self._data.get(prop_name) if prop_val is not None: return instantiate_spec(spec_cls, prop_val, self._validate) else: return None def _group_spec(self, spec_cls, *prop_names): if not prop_names: return None data = {} for prop_name in prop_names: prop_val = self._data.get(prop_name) if prop_val: data[prop_name] = prop_val return instantiate_spec(spec_cls, data, self._validate) def _inject_version(self, prop_names): for prop_name in prop_names: prop_data = self._data.get(prop_name) if isinstance(prop_data, dict): prop_data['version'] = self._version def _as_dict(self, prop_name): prop_val = self._data.get(prop_name) if not prop_val: return {} if isinstance(prop_val, dict): return prop_val elif isinstance(prop_val, list): result = {} for t in prop_val: result.update(t if isinstance(t, dict) else {t: ''}) return result elif isinstance(prop_val, six.string_types): return {prop_val: ''} @staticmethod def _parse_cmd_and_input(cmd_str): # TODO(rakhmerov): Try to find a way with one expression. cmd_matcher = CMD_PTRN.search(cmd_str) if not cmd_matcher: msg = "Invalid action/workflow task property: %s" % cmd_str raise exc.InvalidModelException(msg) cmd = cmd_matcher.group() params = {} for match in re.findall(PARAMS_PTRN, cmd_str): k = match[0] # Remove embracing quotes. v = match[1].strip() if v[0] == '"' or v[0] == "'": v = v[1:-1] else: try: v = json.loads(v) except Exception: pass params[k] = v return cmd, params def to_dict(self): return self._data def get_version(self): return self._version def __repr__(self): return "%s %s" % (self.__class__.__name__, self.to_dict()) class BaseListSpec(BaseSpec): item_class = None _schema = { "type": "object", "properties": { "version": types.VERSION }, "additionalProperties": types.NONEMPTY_DICT, "required": ["version"], } def __init__(self, data, validate): super(BaseListSpec, self).__init__(data, validate) self.items = [] for k, v in data.items(): if k != 'version': v['name'] = k self._inject_version([k]) self.items.append( instantiate_spec(self.item_class, v, validate) ) def validate_schema(self): super(BaseListSpec, self).validate_schema() if len(self._data.keys()) < 2: raise exc.InvalidModelException( 'At least one item must be in the list [data=%s].' % self._data ) def get_items(self): return self.items def __getitem__(self, idx): return self.items[idx] def __len__(self): return len(self.items) class BaseSpecList(object): item_class = None _version = '2.0' def __init__(self, data, validate): self.items = {} for k, v in data.items(): if k != 'version': # At this point, we don't know if item schema is valid, # it may not be even a dictionary. So we should check the # type first before manipulating with it. if isinstance(v, dict): v['name'] = k v['version'] = self._version self.items[k] = instantiate_spec(self.item_class, v, validate) def item_keys(self): return self.items.keys() def __iter__(self): return six.itervalues(self.items) def __getitem__(self, name): return self.items.get(name) def __len__(self): return len(self.items) def get(self, name): return self.__getitem__(name)