openstack
/
congress
Code Issues Proposed changes

Merge "Fixes the potential conflictions in rule creation"

This commit is contained in:
Jenkins 2015-09-17 09:26:51 +00:00 committed by Gerrit Code Review
parent 4ace766451 74221cd4be
commit 48a0e22292
6 changed files with 258 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
congress/datalog/compile.py
View File

@ -47,10 +47,13 @@ class Schema(object):
def __init__(self, dictionary=None, complete=False):
if dictionary is None:
self.map = {}
self.count = {}
elif isinstance(dictionary, Schema):
self.map = dict(dictionary.map)
self.count = dictionary.count
else:
self.map = dictionary
self.count = None
# whether to assume there is an entry in this schema for
# every permitted table
self.complete = complete
@ -74,6 +77,71 @@ class Schema(object):
if tablename in self.map:
return len(self.map[tablename])
def update(self, item, is_insert):
"""Returns the schema change of this update.
Return schema change.
"""
if self.count is None:
return None
if isinstance(item, Fact):
tablename, tablelen = item.table, len(item)
th = None
elif isinstance(item, Literal):
tablename, tablelen = item.table.table, len(item.arguments)
th = item.table.service
else:
raise exception.PolicyException(
"Schema cannot update item: %r" % item)
schema_change = None
if is_insert:
if tablename in self:
self.count[tablename] += 1
schema_change = (tablename, None, True, th)
else:
self.count[tablename] = 1
val = ["Col"+str(i) for i in range(0, tablelen)]
self.map[tablename] = val
schema_change = (tablename, val, True, th)
else:
if tablename not in self:
LOG.warn("Attempt to delete a non-existant rule: %s" % item)
elif self.count[tablename] > 1:
self.count[tablename] -= 1
schema_change = (tablename, None, False, th)
else:
schema_change = (tablename, self.map[tablename], False, th)
del self.count[tablename]
del self.map[tablename]
return schema_change
def revert(self, change):
"""Revert change made by update.
Return None
"""
if change is None:
return
inserted = change[2]
tablename = change[0]
val = change[1]
if inserted:
if self.count[tablename] > 1:
self.count[tablename] -= 1
else:
del self.map[tablename]
del self.count[tablename]
else:
if tablename in self.count:
self.count[tablename] += 1
else:
assert val is not None
self.map[tablename] = val
self.count[tablename] = 1
def __str__(self):
return str(self.map)
@ -1209,7 +1277,7 @@ def fact_errors(atom, theories=None, theory=None):
if not atom.is_ground():
errors.append(exception.PolicyException(
"Fact not ground: " + str(atom)))
errors.extend(literal_schema_consistency(atom, theories, theory))
errors.extend(check_schema_consistency(atom, theories, theory))
errors.extend(fact_has_no_theory(atom))
return errors
@ -1304,15 +1372,6 @@ def rule_body_safety(rule):
return [e]
def rule_schema_consistency(rule, theories, theory=None):
"""Returns list of problems with rule's schema."""
assert not rule.is_atom(), "rule_schema_consistency expects a rule"
errors = []
for lit in rule.body:
errors.extend(literal_schema_consistency(lit, theories, theory))
return errors
def literal_schema_consistency(literal, theories, theory=None):
"""Returns list of errors."""
if theories is None:
@ -1336,16 +1395,13 @@ def literal_schema_consistency(literal, theories, theory=None):
return []
# check if known table
if literal.table.table not in schema:
if schema.complete and literal.table.table not in schema:
if schema.complete:
return [exception.PolicyException(
"Literal {} uses unknown table {} "
"from policy {}".format(
str(literal), str(literal.table.table),
str(active_theory)))]
else:
# may not have a declaration for this table's columns
return []
# check width
arity = schema.arity(literal.table.table)
@ -1358,12 +1414,26 @@ def literal_schema_consistency(literal, theories, theory=None):
return []
def check_schema_consistency(item, theories, theory=None):
errors = []
if item.is_rule():
errors.extend(literal_schema_consistency(
item.head, theories, theory))
for lit in item.body:
errors.extend(literal_schema_consistency(
lit, theories, theory))
else:
errors.extend(literal_schema_consistency(
item, theories, theory))
return errors
def rule_errors(rule, theories=None, theory=None):
"""Returns list of errors for RULE."""
errors = []
errors.extend(rule_head_safety(rule))
errors.extend(rule_body_safety(rule))
errors.extend(rule_schema_consistency(rule, theories, theory))
errors.extend(check_schema_consistency(rule, theories, theory))
errors.extend(rule_head_has_no_theory(rule))
errors.extend(rule_modal_safety(rule))
return errors

65
congress/datalog/nonrecursive.py
View File

@ -16,6 +16,7 @@
from oslo_log import log as logging
from congress.datalog import base
from congress.datalog.builtin import congressbuiltin
from congress.datalog import compile
from congress.datalog import ruleset
from congress.datalog import topdown
@ -36,6 +37,8 @@ class NonrecursiveRuleTheory(topdown.TopDownTheory):
# dictionary from table name to list of rules with that table in head
self.rules = ruleset.RuleSet()
self.kind = base.NONRECURSIVE_POLICY_TYPE
if schema is None:
self.schema = compile.Schema()
# External Interface
@ -58,8 +61,11 @@ class NonrecursiveRuleTheory(topdown.TopDownTheory):
if f.table not in cleared_tables:
extra_tables.add(f.table)
ignored_facts += 1
self.rules.add_rule(f.table, f)
count += 1
else:
self.rules.add_rule(f.table, f)
count += 1
if self.schema:
self.schema.update(f, True)
if ignored_facts > 0:
LOG.error("initialize_tables ignored %d facts for tables "
"%s not included in the list of tablenames %s",
@ -75,6 +81,55 @@ class NonrecursiveRuleTheory(topdown.TopDownTheory):
changes = self.update([compile.Event(formula=rule, insert=False)])
return [event.formula for event in changes]
def _update_lit_schema(self, lit, is_insert):
if not self.schema:
raise exception.PolicyException(
"Cannot update schema because theory %s doesn't have"
"schema." % self.name)
if self.schema.complete:
# complete means the schema is pre-built and shouldn't be updated
return None
return self.schema.update(lit, is_insert)
def update_rule_schema(self, rule, is_insert):
schema_changes = []
if not self.schema or not self.theories or self.schema.complete:
# complete means the schema is pre-built like datasoures'
return schema_changes
if isinstance(rule, compile.Fact) or isinstance(rule, compile.Literal):
schema_changes.append(self._update_lit_schema(rule, is_insert))
return schema_changes
schema_changes.append(self._update_lit_schema(rule.head, is_insert))
for lit in rule.body:
if congressbuiltin.builtin_registry.is_builtin(lit.table,
len(lit.arguments)):
continue
active_theory = lit.table.service or self.name
if active_theory not in self.theories:
continue
schema_changes.append(
self.theories[active_theory]._update_lit_schema(lit,
is_insert))
return schema_changes
def revert_schema(self, schema_changes):
if not self.theories:
return
for change in schema_changes:
if not change:
continue
active_theory = change[3]
if not active_theory:
self.schema.revert(change)
else:
self.theories[active_theory].schema.revert(change)
def update(self, events):
"""Apply EVENTS.
@ -86,13 +141,19 @@ class NonrecursiveRuleTheory(topdown.TopDownTheory):
self.log(None, "Update %s", utility.iterstr(events))
try:
for event in events:
schema_changes = self.update_rule_schema(
event.formula, event.insert)
formula = compile.reorder_for_safety(event.formula)
if event.insert:
if self._insert_actual(formula):
changes.append(event)
else:
self.revert_schema(schema_changes)
else:
if self._delete_actual(formula):
changes.append(event)
else:
self.revert_schema(schema_changes)
except Exception as e:
LOG.exception("runtime caught an exception")
raise e

2
congress/policy_engines/agnostic.py
View File

@ -790,6 +790,8 @@ class Runtime (object):
for th, th_events in by_theory.items():
th_obj = self.get_target(th)
errors.extend(th_obj.update_would_cause_errors(th_events))
if len(errors) > 0:
return (False, errors)
# update dependency graph (and undo it if errors)
graph_changes = self.global_dependency_graph.formula_update(
events, include_atoms=False)

22
congress/tests/datalog/test_compiler.py
View File

@ -564,6 +564,28 @@ class TestCompiler(base.TestCase):
'Wrong number of arguments for atom',
f=compile.fact_errors)
# schema update
schema = compile.Schema()
rule1 = compile.parse1('p(x) :- q(x, y)')
change1 = schema.update(rule1.head, True)
rule2 = compile.parse1('p(x) :- r(x, y)')
change2 = schema.update(rule2.head, True)
self.assertEqual(schema.count['p'], 2)
schema.revert(change2)
self.assertEqual(schema.count['p'], 1)
schema.revert(change1)
self.assertEqual('p' in schema.count, False)
schema.update(rule1.head, True)
schema.update(rule2.head, True)
change1 = schema.update(rule1.head, False)
change2 = schema.update(rule2.head, False)
self.assertEqual('p' in schema.count, False)
schema.revert(change2)
self.assertEqual(schema.count['p'], 1)
schema.revert(change1)
self.assertEqual(schema.count['p'], 2)
def test_rule_recursion(self):
rules = compile.parse('p(x) :- q(x), r(x) q(x) :- r(x) r(x) :- t(x)')
self.assertFalse(compile.is_recursive(rules))

57
congress/tests/datalog/test_nonrecur.py
View File

@ -73,10 +73,10 @@ class TestRuntime(base.TestCase):
run = self.prep_runtime('')
run.insert('p(1)', th)
run.insert('p(2)', th)
run.insert('p(3,4)', th)
run.insert('r(3,4)', th)
run.insert('q(1,2,3)', th)
run.insert('q(4,5,6)', th)
ans = 'p(1) p(2) p(3,4) q(1,2,3) q(4,5,6)'
ans = 'p(1) p(2) r(3,4) q(1,2,3) q(4,5,6)'
self.check_equal(run.content(th), ans, 'Multiple atomic insertions')
# insert collection of rules
@ -137,6 +137,55 @@ class TestRuntime(base.TestCase):
self.assertFalse(permitted)
self.assertEqual(run.content(th), '')
# confliction: rule-rule
run = self.prep_runtime("")
run.insert("q(x) :- p(x,y)", th)
permitted, changes = run.insert("q(x,y) :- p(x,y)", th)
self.assertEqual(len(changes), 1)
self.assertFalse(permitted)
# confliction: rule-fact
run = self.prep_runtime("")
run.insert("q(x) :- p(x,y)", th)
permitted, changes = run.insert("q(1,3)", th)
self.assertEqual(len(changes), 1)
self.assertFalse(permitted)
# confliction: fact-rule
run = self.prep_runtime("")
run.insert("q(1,3)", th)
permitted, changes = run.insert("q(x) :- p(x,y)", th)
self.assertEqual(len(changes), 1)
self.assertFalse(permitted)
# confliction: fact-rule
run = self.prep_runtime("")
run.insert("q(1,3)", th)
permitted, changes = run.insert("q(1)", th)
self.assertEqual(len(changes), 1)
self.assertFalse(permitted)
# confliction: body-confliction
run = self.prep_runtime("")
run.insert("q(1,3)", th)
permitted, changes = run.insert("p(x,y) :- q(x,y,z)", th)
self.assertEqual(len(changes), 1)
self.assertFalse(permitted)
# confliction: body-confliction1
run = self.prep_runtime("")
run.insert("p(x,y) :- q(x,y)", th)
permitted, changes = run.insert("q(y) :- r(y)", th)
self.assertEqual(len(changes), 1)
self.assertFalse(permitted)
# confliction: body-confliction2
run = self.prep_runtime("")
run.insert("p(x) :- q(x)", th)
permitted, changes = run.insert("r(y) :- q(x,y)", th)
self.assertEqual(len(changes), 1)
self.assertFalse(permitted)
def test_delete(self):
"""Test ability to delete policy statements."""
th = NREC_THEORY
@ -145,12 +194,12 @@ class TestRuntime(base.TestCase):
run = self.prep_runtime('', 'Data deletion')
run.insert('p(1)', th)
run.insert('p(2)', th)
run.insert('p(3,4)', th)
run.insert('r(3,4)', th)
run.insert('q(1,2,3)', th)
run.insert('q(4,5,6)', th)
run.delete('q(1,2,3)', th)
run.delete('p(2)', th)
ans = ('p(1) p(3,4) q(4,5,6)')
ans = ('p(1) r(3,4) q(4,5,6)')
self.check_equal(run.content(th), ans, 'Multiple atomic deletions')
# Rules and data

34
congress/tests/policy_engines/test_agnostic.py
View File

@ -166,7 +166,10 @@ class TestRuntime(base.TestCase):
self.assertTrue(helper.datalog_equal(run.select('p(x)'), 'p(1)'))
# next insert causes an exceptionsince the thing we indexed on
# doesn't exist
self.assertRaises(IndexError, run.insert, 'q(5)')
permitted, errs = run.insert('q(5)')
self.assertFalse(permitted)
self.assertEqual(len(errs), 1)
self.assertTrue(isinstance(errs[0], exception.PolicyException))
# double-check that the error didn't result in an inconsistent state
self.assertEqual(run.select('q(5)'), '')
@ -886,6 +889,35 @@ class TestMultipolicyRules(base.TestCase):
run.insert('r(3)', 'sigma')
self.assertEqual(run.select('p(x1,x2)', 'alpha'), 'p(1, 3)')
def test_schema_check(self):
"""Test that schema check in multiple policies works."""
run = agnostic.Runtime()
run.debug_mode()
run.create_policy('alpha')
run.create_policy('beta')
run.insert('p(x,y) :- beta:q(x,y)', 'alpha')
permitted, changes = run.insert('q(x) :- r(x)', 'beta')
self.assertFalse(permitted)
self.assertEqual(len(changes), 1)
def test_same_rules(self):
"""Test that same rule insertion can be correctly dealt with."""
run = agnostic.Runtime()
run.debug_mode()
policy = 'alpha'
run.create_policy(policy)
rulestr = 'p(x,y) :- q(x,y)'
rule = compile.parse1(rulestr)
run.insert(rulestr, policy)
self.assertTrue(rule in run.policy_object(policy))
self.assertTrue(
rule.head.table.table in run.policy_object(policy).schema)
run.insert(rulestr, policy)
run.delete(rulestr, policy)
self.assertFalse(rule in run.policy_object(policy))
self.assertFalse(
rule.head.table.table in run.policy_object(policy).schema)
class TestSelect(base.TestCase):
def test_no_dups(self):