Introduced YAQL helpers
Added syntax sugar to simplify YAQL expressins. Change-Id: Ifb5ace0302dcf4e041d3962271faec669d494252 Implements: blueprint computable-task-fields-yaql
This commit is contained in:
parent
00188053af
commit
055359b58f
|
@ -77,8 +77,6 @@ def t_error(t):
|
|||
t.lexer.skip(1)
|
||||
|
||||
|
||||
ply.lex.lex()
|
||||
|
||||
expression = None
|
||||
|
||||
precedence = (
|
||||
|
@ -143,10 +141,11 @@ def p_error(p):
|
|||
raise errors.ParseError("Syntax error at '%s'" % getattr(p, 'value', ''))
|
||||
|
||||
|
||||
lexer = ply.lex.lex()
|
||||
parser = ply.yacc.yacc(debug=False, write_tables=False)
|
||||
|
||||
|
||||
def parse(expr):
|
||||
global expression
|
||||
expression = expr
|
||||
return parser.parse(expression.expression_text)
|
||||
return parser.parse(expression.expression_text, lexer=lexer)
|
||||
|
|
|
@ -0,0 +1,162 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from oslo_serialization import jsonutils
|
||||
import six
|
||||
import yaml
|
||||
import yaql
|
||||
from yaql.language import exceptions
|
||||
|
||||
from nailgun.test.base import BaseUnitTest
|
||||
from nailgun import yaql_ext
|
||||
|
||||
|
||||
class TestYaqlExt(BaseUnitTest):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.variables = {
|
||||
'$%new': {
|
||||
'nodes': [
|
||||
{'uid': '1', 'role': 'compute'},
|
||||
{'uid': '2', 'role': 'controller'}
|
||||
],
|
||||
'configs': {
|
||||
'nova': {
|
||||
'value': 1,
|
||||
'value2': None,
|
||||
}
|
||||
},
|
||||
'cluster': {
|
||||
'status': 'operational'
|
||||
},
|
||||
},
|
||||
'$%old': {
|
||||
'nodes': [
|
||||
{'uid': '1', 'role': 'controller'},
|
||||
{'uid': '2', 'role': 'compute'},
|
||||
],
|
||||
'configs': {
|
||||
'nova': {
|
||||
'value': 2,
|
||||
'value2': None
|
||||
}
|
||||
},
|
||||
'cluster': {
|
||||
'status': 'new'
|
||||
},
|
||||
}
|
||||
}
|
||||
cls.variables['$'] = cls.variables['$%new']
|
||||
|
||||
def evaluate(self, expression, variables=None, engine=None):
|
||||
context = yaql_ext.create_context(
|
||||
add_datadiff=True, add_serializers=True
|
||||
)
|
||||
for k, v in six.iteritems(variables or self.variables):
|
||||
context[k] = v
|
||||
|
||||
engine = engine or yaql_ext.create_engine()
|
||||
parsed_exp = engine(expression)
|
||||
return parsed_exp.evaluate(context=context)
|
||||
|
||||
def test_new(self):
|
||||
result = self.evaluate(
|
||||
'new($.nodes.where($.role=compute))'
|
||||
)
|
||||
self.assertEqual([{'uid': '1', 'role': 'compute'}], result)
|
||||
|
||||
def test_old(self):
|
||||
result = self.evaluate(
|
||||
'old($.nodes.where($.role=compute))'
|
||||
)
|
||||
self.assertEqual([{'uid': '2', 'role': 'compute'}], result)
|
||||
|
||||
def test_added(self):
|
||||
self.assertEqual(
|
||||
[{'uid': '1', 'role': 'compute'}],
|
||||
self.evaluate('added($.nodes.where($.role=compute))')
|
||||
)
|
||||
|
||||
def test_deleted(self):
|
||||
self.assertItemsEqual(
|
||||
[{'uid': '2', 'role': 'compute'}],
|
||||
self.evaluate('deleted($.nodes.where($.role=compute))')
|
||||
)
|
||||
|
||||
def test_changed(self):
|
||||
self.assertTrue(self.evaluate('changed($.configs.nova.value)'))
|
||||
self.assertFalse(self.evaluate('changed($.configs.nova.value2)'))
|
||||
|
||||
def test_added_if_no_old(self):
|
||||
variables = self.variables.copy()
|
||||
variables['$%old'] = {}
|
||||
self.assertItemsEqual(
|
||||
[{'uid': '1', 'role': 'compute'}],
|
||||
self.evaluate('added($.nodes.where($.role=compute))', variables)
|
||||
)
|
||||
|
||||
def test_delete_if_no_old(self):
|
||||
variables = self.variables.copy()
|
||||
variables['$%old'] = {}
|
||||
self.assertIsNone(
|
||||
self.evaluate('deleted($.nodes.where($.role=compute))', variables)
|
||||
)
|
||||
|
||||
def test_changed_if_no_old(self):
|
||||
variables = self.variables.copy()
|
||||
variables['$%old'] = {}
|
||||
self.assertTrue(
|
||||
self.evaluate('changed($.configs.nova.value)', variables)
|
||||
)
|
||||
self.assertTrue(
|
||||
self.evaluate('changed($.configs.nova.value2)', variables)
|
||||
)
|
||||
|
||||
def test_undefined(self):
|
||||
variables = self.variables.copy()
|
||||
variables['$%old'] = {}
|
||||
self.assertTrue(
|
||||
self.evaluate('old($.configs.nova.value).isUndef()', variables),
|
||||
)
|
||||
|
||||
def test_to_yaml(self):
|
||||
expected = yaml.safe_dump(self.variables['$%new']['configs'])
|
||||
actual = self.evaluate('$.configs.toYaml()')
|
||||
self.assertEqual(expected, actual)
|
||||
|
||||
def test_to_json(self):
|
||||
expected = jsonutils.dumps(self.variables['$%new']['configs'])
|
||||
actual = self.evaluate('$.configs.toJson()')
|
||||
self.assertEqual(expected, actual)
|
||||
|
||||
def test_limit_iterables(self):
|
||||
engine = yaql.YaqlFactory().create({
|
||||
'yaql.limitIterators': 1,
|
||||
'yaql.convertTuplesToLists': True,
|
||||
'yaql.convertSetsToLists': True
|
||||
})
|
||||
functions = ['added', 'deleted', 'changed']
|
||||
|
||||
expressions = ['$.nodes', '$.configs.nova']
|
||||
for exp in expressions:
|
||||
for func in functions:
|
||||
with self.assertRaises(exceptions.CollectionTooLargeException):
|
||||
self.evaluate('{0}({1})'.format(func, exp), engine=engine)
|
||||
|
||||
expressions = ['$.configs.nova.value', '$.cluster.status']
|
||||
for exp in expressions:
|
||||
for func in functions:
|
||||
self.evaluate('{0}({1})'.format(func, exp), engine=engine)
|
|
@ -0,0 +1,49 @@
|
|||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import yaql
|
||||
|
||||
from nailgun.yaql_ext import datadiff
|
||||
from nailgun.yaql_ext import serializers
|
||||
|
||||
|
||||
LIMIT_ITERATORS = 5000
|
||||
|
||||
MEMORY_QUOTA = 20000
|
||||
|
||||
_global_engine = None
|
||||
|
||||
|
||||
def create_context(add_serializers=False, add_datadiff=False, **kwargs):
|
||||
context = yaql.create_context(**kwargs)
|
||||
if add_serializers:
|
||||
serializers.register(context)
|
||||
if add_datadiff:
|
||||
datadiff.register(context)
|
||||
return context
|
||||
|
||||
|
||||
def create_engine():
|
||||
global _global_engine
|
||||
|
||||
engine_options = {
|
||||
'yaql.limitIterators': LIMIT_ITERATORS,
|
||||
'yaql.memoryQuota': MEMORY_QUOTA,
|
||||
'yaql.convertTuplesToLists': True,
|
||||
'yaql.convertSetsToLists': True
|
||||
}
|
||||
|
||||
if _global_engine is None:
|
||||
_global_engine = yaql.YaqlFactory().create(engine_options)
|
||||
return _global_engine
|
|
@ -0,0 +1,85 @@
|
|||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from yaql.language import specs
|
||||
from yaql.language import utils as yaqlutils
|
||||
from yaql.language import yaqltypes
|
||||
|
||||
from nailgun.logger import logger
|
||||
from nailgun.utils import datadiff
|
||||
|
||||
|
||||
_UNDEFINED = object()
|
||||
|
||||
|
||||
@specs.parameter('expression', yaqltypes.Lambda())
|
||||
def get_new(expression, context):
|
||||
return expression(context['$%new'])
|
||||
|
||||
|
||||
@specs.parameter('expression', yaqltypes.Lambda())
|
||||
def get_old(expression, context):
|
||||
try:
|
||||
return expression(context['$%old'])
|
||||
except Exception as e:
|
||||
# exception in evaluation on old data interprets as data changed
|
||||
logger.debug('Cannot evaluate expression on old data: %s', e)
|
||||
return _UNDEFINED
|
||||
|
||||
|
||||
@specs.parameter('expression', yaqltypes.Lambda())
|
||||
@specs.inject('finalizer', yaqltypes.Delegate('#finalize'))
|
||||
def changed(finalizer, expression, context):
|
||||
new_data = finalizer(get_new(expression, context))
|
||||
old_data = finalizer(get_old(expression, context))
|
||||
return new_data != old_data
|
||||
|
||||
|
||||
def get_limited_if_need(data, engine):
|
||||
if (yaqlutils.is_iterable(data) or yaqlutils.is_sequence(data) or
|
||||
isinstance(data, (yaqlutils.MappingType, yaqlutils.SetType))):
|
||||
return yaqlutils.limit_iterable(data, engine)
|
||||
return data
|
||||
|
||||
|
||||
@specs.parameter('expression', yaqltypes.Lambda())
|
||||
def added(expression, context, engine):
|
||||
new_data = get_limited_if_need(get_new(expression, context), engine)
|
||||
old_data = get_limited_if_need(get_old(expression, context), engine)
|
||||
if old_data is _UNDEFINED:
|
||||
return new_data
|
||||
return datadiff.diff(old_data, new_data).added
|
||||
|
||||
|
||||
@specs.parameter('expression', yaqltypes.Lambda())
|
||||
def deleted(expression, context, engine):
|
||||
new_data = get_limited_if_need(get_new(expression, context), engine)
|
||||
old_data = get_limited_if_need(get_old(expression, context), engine)
|
||||
if old_data is not _UNDEFINED:
|
||||
return datadiff.diff(old_data, new_data).deleted
|
||||
|
||||
|
||||
@specs.method
|
||||
@specs.inject('finalizer', yaqltypes.Delegate('#finalize'))
|
||||
def is_undef(finalizer, receiver):
|
||||
return finalizer(receiver) is _UNDEFINED
|
||||
|
||||
|
||||
def register(context):
|
||||
context.register_function(get_new, name='new')
|
||||
context.register_function(get_old, name='old')
|
||||
context.register_function(changed)
|
||||
context.register_function(added)
|
||||
context.register_function(deleted)
|
||||
context.register_function(is_undef)
|
|
@ -0,0 +1,36 @@
|
|||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
from oslo_serialization import jsonutils
|
||||
import yaml
|
||||
from yaql.language import specs
|
||||
from yaql.language import yaqltypes
|
||||
|
||||
|
||||
@specs.method
|
||||
@specs.inject('finalizer', yaqltypes.Delegate('#finalize'))
|
||||
def to_yaml(finalizer, receiver):
|
||||
return yaml.safe_dump(finalizer(receiver))
|
||||
|
||||
|
||||
@specs.method
|
||||
@specs.inject('finalizer', yaqltypes.Delegate('#finalize'))
|
||||
def to_json(finalizer, receiver):
|
||||
return jsonutils.dumps(finalizer(receiver))
|
||||
|
||||
|
||||
def register(context):
|
||||
context.register_function(to_yaml)
|
||||
context.register_function(to_json)
|
|
@ -46,3 +46,4 @@ stevedore>=1.5.0
|
|||
# the editable mode is broken
|
||||
# See: https://bugs.launchpad.net/fuel/+bug/1519727
|
||||
setuptools<=18.5
|
||||
yaql>=1.0.0
|
||||
|
|
|
@ -55,6 +55,7 @@ Requires: python-networkx-core >= 1.8.0
|
|||
Requires: python-networkx-core < 1.10.0
|
||||
Requires: python-cinderclient >= 1.0.7
|
||||
Requires: pydot-ng >= 1.0.0
|
||||
Requires: python-yaql >= 1.0.0
|
||||
# Workaroud for babel bug
|
||||
Requires: pytz
|
||||
|
||||
|
|
Loading…
Reference in New Issue