openstack
/
congress
Code Issues Proposed changes

Merge "Pad positional args up to required number"

This commit is contained in:
Jenkins 2017-04-13 18:46:08 +00:00 committed by Gerrit Code Review
parent 448dafae06 9f52ebbf68
commit c707d2bd32
3 changed files with 197 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

179
congress/datalog/compile.py
View File

@ -800,84 +800,96 @@ class Literal (object):
self.table.drop_service() self.table.drop_service()
return self return self
def eliminate_column_references(self, theories, default_theory=None, def eliminate_column_references_and_pad_positional(
index=0, prefix=''): self, theories, default_theory=None, index=0, prefix=''):
"""Expand column references to traditional datalog positional args. """Expand column references to positional args and pad positional args.
Returns a new literal, unless no column references. Expand column references to traditional datalog positional args.
Also pad positional args if too few are provided.
Returns a new literal. If no column reference, unless no schema found
for the table.
""" """
# TODO(ekcs): remove unused parameter: index # TODO(ekcs): remove unused parameter: index
# corner cases # corner cases
if len(self.named_arguments) == 0: if len(self.named_arguments) > 0:
return self theory = literal_theory(self, theories, default_theory)
theory = literal_theory(self, theories, default_theory) if theory is None or theory.schema is None:
if theory is None or theory.schema is None: raise exception.IncompleteSchemaException(
raise exception.IncompleteSchemaException( "Literal %s uses named arguments, but the "
"Literal %s uses named arguments, but the " "schema is unknown." % self)
"schema is unknown." % self) if theory.kind != base.DATASOURCE_POLICY_TYPE: # eventually remove
if theory.kind != base.DATASOURCE_POLICY_TYPE: # eventually remove raise exception.PolicyException(
raise exception.PolicyException( "Literal {} uses column references, but '{}' does not "
"Literal {} uses column references, but '{}' does not " "reference a datasource policy.".format(self, theory.name))
"reference a datasource policy.".format(self, theory.name)) schema = theory.schema
schema = theory.schema if self.table.table not in schema:
if self.table.table not in schema: raise exception.IncompleteSchemaException(
raise exception.IncompleteSchemaException( "Literal {} uses unknown table {}.".format(
"Literal {} uses unknown table {}.".format( str(self), str(self.table.table)))
str(self), str(self.table.table)))
# check if named arguments conflict with positional or named arguments # check if named arguments conflict with positional or named args
errors = [] errors = []
term_index = {} term_index = {}
for col, arg in self.named_arguments.items(): for col, arg in self.named_arguments.items():
if isinstance(col, six.string_types): # column name if isinstance(col, six.string_types): # column name
index = schema.column_number(self.table.table, col) index = schema.column_number(self.table.table, col)
if index is None: if index is None:
errors.append(exception.PolicyException( errors.append(exception.PolicyException(
"In literal {} column name {} does not exist".format( "In literal {} column name {} does not "
str(self), col))) "exist".format(str(self), col)))
continue continue
if index < len(self.arguments): if index < len(self.arguments):
errors.append(exception.PolicyException( errors.append(exception.PolicyException(
"In literal {} column name {} references position {}," "In literal {} column name {} references position "
" which is already provided by position.".format( "{}, which is already provided by "
str(self), col, index))) "position.".format(str(self), col, index)))
if index in self.named_arguments: if index in self.named_arguments:
errors.append(exception.PolicyException( errors.append(exception.PolicyException(
"In literal {} column name {} references position {}, " "In literal {} column name {} references position "
"which is also referenced by number.))".format( "{}, which is also referenced by number.))".format(
str(self), col, index))) str(self), col, index)))
if index in term_index: if index in term_index:
# should have already caught this case above # should have already caught this case above
errors.append(exception.PolicyException( errors.append(exception.PolicyException(
"In literal {}, column name {} references position {}," "In literal {}, column name {} references "
" which already has reference {}".format( "position {}, which already has reference "
str(self), col, index, str(term_index[index])))) "{}".format(str(self), col, index,
term_index[index] = arg str(term_index[index]))))
else: # column number term_index[index] = arg
if col >= schema.arity(self.table.table): else: # column number
errors.append(exception.PolicyException( if col >= schema.arity(self.table.table):
"In literal {} column index {} is too large".format( errors.append(exception.PolicyException(
str(self), col))) "In literal {} column index {} is too "
if col < len(self.arguments): "large".format(str(self), col)))
errors.append(exception.PolicyException( if col < len(self.arguments):
"In literal {} column index {} " errors.append(exception.PolicyException(
" is already provided by position.".format( "In literal {} column index {} "
str(self), col))) " is already provided by position.".format(
name = schema.column_name(self.table.table, col) str(self), col)))
if name in self.named_arguments: name = schema.column_name(self.table.table, col)
errors.append(exception.PolicyException( if name in self.named_arguments:
"In literal {} column index {} references column {}, " errors.append(exception.PolicyException(
"which is also referenced by name.))".format( "In literal {} column index {} references column "
str(self), col, name))) "{}, which is also referenced by name.))".format(
if col in term_index: str(self), col, name)))
# should have already caught this case above if col in term_index:
errors.append(exception.PolicyException( # should have already caught this case above
"In literal {} column index {} already has a reference" errors.append(exception.PolicyException(
" {}".format(str(self), col, str(term_index[col])))) "In literal {} column index {} already has a "
term_index[col] = arg "reference {}".format(
if errors: str(self), col, str(term_index[col]))))
raise exception.PolicyException( term_index[col] = arg
" ".join(str(err) for err in errors)) if errors:
raise exception.PolicyException(
" ".join(str(err) for err in errors))
else:
theory = literal_theory(self, theories, default_theory)
if theory is None or theory.schema is None:
return self
schema = theory.schema
if self.table.table not in schema:
return self
term_index = {}
# turn reference args into position args # turn reference args into position args
position_args = list(self.arguments) # copy the original list position_args = list(self.arguments) # copy the original list
@ -1072,17 +1084,21 @@ class Rule(object):
def is_update(self): def is_update(self):
return self.head.is_update() return self.head.is_update()
def eliminate_column_references(self, theories, default_theory=None): def eliminate_column_references_and_pad_positional(
"""Return version of SELF where all column references have been removed. self, theories, default_theory=None):
"""Return version of SELF /w col refs removed and pos args padded.
All column references removed. Positional args padded up to required
length.
Throws exception if RULE is inconsistent with schemas. Throws exception if RULE is inconsistent with schemas.
""" """
pre = self._unused_variable_prefix() pre = self._unused_variable_prefix()
heads = [] heads = []
for i in range(0, len(self.heads)): for i in range(0, len(self.heads)):
heads.append(self.heads[i].eliminate_column_references( heads.append(
theories, default_theory=default_theory, self.heads[i].eliminate_column_references_and_pad_positional(
index=i, prefix='%s%s' % (pre, i))) theories, default_theory=default_theory,
index=i, prefix='%s%s' % (pre, i)))
body = [] body = []
sorted_lits = sorted(self.body) sorted_lits = sorted(self.body)
@ -1091,9 +1107,10 @@ class Rule(object):
lit_rank[sorted_lits[i]] = i lit_rank[sorted_lits[i]] = i
for i in range(0, len(self.body)): for i in range(0, len(self.body)):
body.append(self.body[i].eliminate_column_references( body.append(
theories, default_theory=default_theory, self.body[i].eliminate_column_references_and_pad_positional(
index=i, prefix='%s%s' % (pre, lit_rank[self.body[i]]))) theories, default_theory=default_theory,
index=i, prefix='%s%s' % (pre, lit_rank[self.body[i]])))
return Rule(heads, body, self.location, name=self.name, return Rule(heads, body, self.location, name=self.name,
comment=self.comment, original_str=self.original_str) comment=self.comment, original_str=self.original_str)

16
congress/policy_engines/agnostic.py
View File

@ -1056,18 +1056,20 @@ class Runtime (object):
enabled = [] enabled = []
errors = [] errors = []
for event in events: for event in events:
errs = compile.check_schema_consistency(
event.formula, self.theory, event.target)
if len(errs) > 0:
errors.append((event, errs))
continue
try: try:
oldformula = event.formula oldformula = event.formula
event.formula = oldformula.eliminate_column_references( event.formula = \
self.theory, default_theory=event.target) oldformula.eliminate_column_references_and_pad_positional(
self.theory, default_theory=event.target)
# doesn't copy over ID since it creates a new one # doesn't copy over ID since it creates a new one
event.formula.set_id(oldformula.id) event.formula.set_id(oldformula.id)
enabled.append(event) enabled.append(event)
errs = compile.check_schema_consistency(
event.formula, self.theory, event.target)
if len(errs) > 0:
errors.append((event, errs))
continue
except exception.IncompleteSchemaException as e: except exception.IncompleteSchemaException as e:
if persistent: if persistent:
# FIXME(ekcs): inconsistent behavior? # FIXME(ekcs): inconsistent behavior?

97
congress/tests/datalog/test_compiler.py
View File

@ -264,6 +264,83 @@ class TestColumnReferences(base.TestCase):
'1 is already provided by position arguments', '1 is already provided by position arguments',
'Conflict between name and position') 'Conflict between name and position')
def test_positional_args_padding_atom(self):
"""Test positional args padding on a single atom."""
def check_err(rule, errmsg, msg):
rule = compile.parse1(rule)
try:
rule.eliminate_column_references_and_pad_positional(theories)
self.fail("Failed to throw error {}".format(errmsg))
except (exception.PolicyException,
exception.IncompleteSchemaException) as e:
emsg = "Err messages '{}' should include '{}'".format(
str(e), errmsg)
self.assertIn(errmsg, str(e), msg + ": " + emsg)
def check(code, correct, msg, no_theory=False):
actual = compile.parse1(
code).eliminate_column_references_and_pad_positional(
{} if no_theory else theories)
eq = helper.datalog_same(str(actual), correct)
self.assertTrue(eq, msg)
run = agnostic.Runtime()
run.create_policy('nova')
schema = compile.Schema({'q': ('id', 'name', 'status')})
theories = {'nova': self.SchemaWrapper(schema)}
# Too few positional args
code = ("p(x) :- nova:q(w, y)")
correct = "p(x) :- nova:q(w, y, x3)"
check(code, correct, 'Too few positional args')
code = ("p(x) :- nova:q(w)")
correct = "p(x) :- nova:q(w, y, x3)"
check(code, correct, 'Too few positional args')
code = ("p(x) :- nova:q()")
correct = "p(x) :- nova:q(w, y, x3)"
check(code, correct, 'Too few (no) positional args')
# No schema provided, no change
code = ("p(x) :- nova:q(w, y)")
correct = "p(x) :- nova:q(w, y)"
check(code, correct, 'No schema provided', True)
code = ("p(x) :- nova:q(w, x, y, z)")
correct = "p(x) :- nova:q(w, x, y, z)"
check(code, correct, 'No schema provided', True)
def test_positional_args_padding_multiple_atoms(self):
"""Test positional args padding on a single atom."""
def check(code, correct, msg, no_theory=False):
actual = compile.parse1(
code).eliminate_column_references_and_pad_positional(
{} if no_theory else theories)
eq = helper.datalog_same(str(actual), correct)
self.assertTrue(eq, msg)
run = agnostic.Runtime()
run.create_policy('nova')
schema = compile.Schema({'q': ('id', 'name', 'status'),
'r': ('id', 'age', 'weight')})
theories = {'nova': self.SchemaWrapper(schema)}
# Multiple atoms, no shared variable
code = ("p(x) :- nova:q(x, y), nova:r(w)")
correct = "p(x) :- nova:q(x, y, z0), nova:r(w, y0, y1)"
check(code, correct, 'Multiple atoms')
# Multiple atoms, some shared variable
code = ("p(x) :- nova:q(x, y), nova:r(x)")
correct = "p(x) :- nova:q(x, y, z0), nova:r(x, y0, y1)"
check(code, correct, 'Multiple atoms')
# Multiple atoms, same table
code = ("p(x) :- nova:q(x, y), nova:q(x)")
correct = "p(x) :- nova:q(x, y, z0), nova:q(x, w0, w1)"
check(code, correct, 'Multiple atoms, same table')
def test_column_references_validation_errors(self): def test_column_references_validation_errors(self):
"""Test invalid column references occurring in a single atom.""" """Test invalid column references occurring in a single atom."""
schema = compile.Schema({'q': ('id', 'name', 'status'), schema = compile.Schema({'q': ('id', 'name', 'status'),
@ -274,7 +351,7 @@ class TestColumnReferences(base.TestCase):
def check_err(rule, errmsg, msg): def check_err(rule, errmsg, msg):
rule = compile.parse1(rule) rule = compile.parse1(rule)
try: try:
rule.eliminate_column_references(theories) rule.eliminate_column_references_and_pad_positional(theories)
self.fail("Failed to throw error {}".format(errmsg)) self.fail("Failed to throw error {}".format(errmsg))
except (exception.PolicyException, except (exception.PolicyException,
exception.IncompleteSchemaException) as e: exception.IncompleteSchemaException) as e:
@ -316,7 +393,8 @@ class TestColumnReferences(base.TestCase):
def test_column_references_atom(self): def test_column_references_atom(self):
"""Test column references occurring in a single atom in a rule.""" """Test column references occurring in a single atom in a rule."""
def check(code, correct, msg): def check(code, correct, msg):
actual = compile.parse1(code).eliminate_column_references(theories) actual = compile.parse1(
code).eliminate_column_references_and_pad_positional(theories)
eq = helper.datalog_same(str(actual), correct) eq = helper.datalog_same(str(actual), correct)
self.assertTrue(eq, msg) self.assertTrue(eq, msg)
@ -376,10 +454,13 @@ class TestColumnReferences(base.TestCase):
correct = "p(x) :- nova:q(x, y, z)" correct = "p(x) :- nova:q(x, y, z)"
check(code, correct, 'Pure positional without schema') check(code, correct, 'Pure positional without schema')
# Too few pure positional EKCS
def test_column_references_multiple_atoms(self): def test_column_references_multiple_atoms(self):
"""Test column references occurring in multiple atoms in a rule.""" """Test column references occurring in multiple atoms in a rule."""
def check(code, correct, msg): def check(code, correct, msg):
actual = compile.parse1(code).eliminate_column_references(theories) actual = compile.parse1(
code).eliminate_column_references_and_pad_positional(theories)
eq = helper.datalog_same(str(actual), correct) eq = helper.datalog_same(str(actual), correct)
self.assertTrue(eq, msg) self.assertTrue(eq, msg)
@ -412,10 +493,12 @@ class TestColumnReferences(base.TestCase):
'r': ('id', 'age', 'weight')}) 'r': ('id', 'age', 'weight')})
theories = {'nova': self.SchemaWrapper(schema)} theories = {'nova': self.SchemaWrapper(schema)}
rule1 = compile.parse1("p(x) :- nova:q(id=x, 2=y), nova:r(id=x)" rule1 = compile.parse1(
).eliminate_column_references(theories) "p(x) :- nova:q(id=x, 2=y), nova:r(id=x)"
rule2 = compile.parse1("p(x) :- nova:r(id=x), nova:q(id=x, 2=y)" ).eliminate_column_references_and_pad_positional(theories)
).eliminate_column_references(theories) rule2 = compile.parse1(
"p(x) :- nova:r(id=x), nova:q(id=x, 2=y)"
).eliminate_column_references_and_pad_positional(theories)
self.assertEqual(rule1, rule2, 'eliminate_column_references failed to ' self.assertEqual(rule1, rule2, 'eliminate_column_references failed to '
'preserve order insensitivity') 'preserve order insensitivity')