diff --git a/README.rst b/README.rst index 83398ede5..37e20203b 100644 --- a/README.rst +++ b/README.rst @@ -185,7 +185,16 @@ Configure Congress (Assume you put config files in /etc/congress) $ sudo cp etc/api-paste.ini /etc/congress $ sudo cp etc/policy.json /etc/congress +Set-up Policy Library [optional] + This step copies the bundled collection Congress policies into the Congress + policy library for easy activation by an administrator. The policies in the + library do not become active until explicitly activated by an administrator. + The step may be skipped if you do not want to load the bundled policies into + the policy library. +.. code-block:: console + + $ sudo cp -r library /etc/congress/. Generate a configuration file as outlined in the Configuration Options section of the :ref:`Deployment ` document. Note: you may have to run the command with sudo. diff --git a/congress/api/library_policy_model.py b/congress/api/library_policy_model.py index 0d8e313ac..73d819ee2 100644 --- a/congress/api/library_policy_model.py +++ b/congress/api/library_policy_model.py @@ -17,9 +17,6 @@ from __future__ import print_function from __future__ import division from __future__ import absolute_import -import json -import jsonschema - from oslo_log import log as logging from congress.api import base @@ -97,7 +94,6 @@ class LibraryPolicyModel(base.APIModel): (num, desc) = error_codes.get('policy_id_must_not_be_provided') raise webservice.DataModelException(num, desc) - self._validate_policy_item(item) try: # Note(thread-safety): blocking call policy_metadata = self.invoke_rpc( @@ -144,8 +140,6 @@ class LibraryPolicyModel(base.APIModel): Raises: KeyError: Item with specified id_ not present. """ - self._validate_policy_item(item) - # Note(thread-safety): blocking call try: return self.invoke_rpc(base.LIBRARY_SERVICE_ID, @@ -154,70 +148,3 @@ class LibraryPolicyModel(base.APIModel): 'policy_dict': item}) except exception.CongressException as e: raise webservice.DataModelException.create(e) - - def _validate_policy_item(self, item): - schema_json = ''' - { - "id": "PolicyProperties", - "title": "Policy Properties", - "type": "object", - "required": ["name", "rules"], - "properties": { - "name": { - "title": "Policy unique name", - "type": "string", - "minLength": 1, - "maxLength": 255 - }, - "description": { - "title": "Policy description", - "type": "string" - }, - "kind": { - "title": "Policy kind", - "type": "string", - "enum": ["database", "nonrecursive", "action", "materialized", - "delta", "datasource"] - }, - "abbreviation": { - "title": "Policy name abbreviation", - "type": "string", - "minLength": 1, - "maxLength": 5 - }, - "rules": { - "title": "collection of rules", - "type": "array", - "items": { - "type": "object", - "properties": { - "PolicyRule": { - "title": "Policy rule", - "type": "object", - "required": ["rule"], - "properties": { - "rule": { - "title": "Rule definition following policy grammar", - "type": "string" - }, - "name": { - "title": "User-friendly name", - "type": "string" - }, - "comment": { - "title": "User-friendly comment", - "type": "string" - } - } - } - } - } - } - } - } - ''' - try: - jsonschema.validate(item, json.loads(schema_json)) - except jsonschema.exceptions.ValidationError as ve: - raise webservice.DataModelException( - 1000, 'Input item violates JSON Schema', data=str(ve)) diff --git a/congress/common/config.py b/congress/common/config.py index c38311e8f..7b0f79c8f 100644 --- a/congress/common/config.py +++ b/congress/common/config.py @@ -75,6 +75,8 @@ core_opts = [ cfg.BoolOpt('replicated_policy_engine', default=False, help='Set the flag to use congress with replicated policy ' 'engines.'), + cfg.StrOpt('policy_library_path', default='/etc/congress/library', + help=_('The directory containing library policy files.')), cfg.BoolOpt('distributed_architecture', deprecated_for_removal=True, deprecated_reason='distributed architecture is now the only ' diff --git a/congress/db/db_library_policies.py b/congress/db/db_library_policies.py index 1cd29a8fc..4f79375d4 100644 --- a/congress/db/db_library_policies.py +++ b/congress/db/db_library_policies.py @@ -18,6 +18,7 @@ from __future__ import absolute_import import json +from oslo_db import exception as oslo_db_exc import sqlalchemy as sa from sqlalchemy.orm import exc as db_exc @@ -26,9 +27,9 @@ from congress.db import model_base class LibraryPolicy(model_base.BASE, model_base.HasId): - __tablename__ = 'librarypolicies' + __tablename__ = 'library_policies' - name = sa.Column(sa.String(255), nullable=False) + name = sa.Column(sa.String(255), nullable=False, unique=True) abbreviation = sa.Column(sa.String(5), nullable=False) description = sa.Column(sa.Text(), nullable=False) kind = sa.Column(sa.Text(), nullable=False) @@ -60,15 +61,19 @@ class LibraryPolicy(model_base.BASE, model_base.HasId): def add_policy(policy_dict, session=None): session = session or db.get_session() - with session.begin(subtransactions=True): - new_row = LibraryPolicy( - name=policy_dict['name'], - abbreviation=policy_dict['abbreviation'], - description=policy_dict['description'], - kind=policy_dict['kind'], - rules=json.dumps(policy_dict['rules'])) - session.add(new_row) - return new_row + try: + with session.begin(subtransactions=True): + new_row = LibraryPolicy( + name=policy_dict['name'], + abbreviation=policy_dict['abbreviation'], + description=policy_dict['description'], + kind=policy_dict['kind'], + rules=json.dumps(policy_dict['rules'])) + session.add(new_row) + return new_row + except oslo_db_exc.DBDuplicateEntry: + raise KeyError( + "Policy with name %s already exists" % policy_dict['name']) def replace_policy(id_, policy_dict, session=None): diff --git a/congress/db/migration/alembic_migrations/versions/c0125080d572_policy_library.py b/congress/db/migration/alembic_migrations/versions/c0125080d572_policy_library.py index e95bb5c47..d6563b69d 100644 --- a/congress/db/migration/alembic_migrations/versions/c0125080d572_policy_library.py +++ b/congress/db/migration/alembic_migrations/versions/c0125080d572_policy_library.py @@ -31,9 +31,9 @@ import sqlalchemy as sa def upgrade(): op.create_table( - 'librarypolicies', + 'library_policies', sa.Column('id', sa.String(length=36), nullable=False), - sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False, unique=True), sa.Column('abbreviation', sa.String(length=5), nullable=False), sa.Column('description', sa.Text(), nullable=False), sa.Column('kind', sa.Text(), nullable=False), @@ -44,4 +44,4 @@ def upgrade(): def downgrade(): - op.drop_table('librarypolicies') + op.drop_table('library_policies') diff --git a/congress/exception.py b/congress/exception.py index e302cea65..90b8d9416 100644 --- a/congress/exception.py +++ b/congress/exception.py @@ -160,6 +160,10 @@ class LazyTable(BadRequest): msg_fmt = _("table %(lazy_table)s is a lazy table and is not subscribed.") +class InvalidPolicyInput(BadRequest): + msg_fmt = _('Input policy item violates schema.') + + # NOTE(thinrichs): The following represent different kinds of # exceptions: the policy compiler and the policy runtime, respectively. class PolicyException(CongressException): diff --git a/congress/harness.py b/congress/harness.py index 60290b189..84775bc17 100644 --- a/congress/harness.py +++ b/congress/harness.py @@ -151,6 +151,9 @@ def initialize_policy_engine(engine): def create_policy_library_service(): """Create policy library service.""" library = library_service.LibraryService(api_base.LIBRARY_SERVICE_ID) + # load library policies from file if none present in DB + if len(library.get_policies(include_rules=False)) == 0: + library.load_policies_from_files() return library diff --git a/congress/library_service/library_service.py b/congress/library_service/library_service.py index 4c65f1bda..93778805f 100644 --- a/congress/library_service/library_service.py +++ b/congress/library_service/library_service.py @@ -18,7 +18,12 @@ from __future__ import division from __future__ import absolute_import import copy +import json +import jsonschema +import os +import yaml +from oslo_config import cfg from oslo_db import exception as db_exc from oslo_log import log as logging @@ -38,6 +43,7 @@ class LibraryService (data_service.DataService): def create_policy(self, policy_dict): policy_dict = copy.deepcopy(policy_dict) + self._validate_policy_item(policy_dict) policy_name = policy_dict['name'] # check name is valid @@ -81,6 +87,7 @@ class LibraryService (data_service.DataService): return db_object.to_dict(include_rules=True) def replace_policy(self, id_, policy_dict): + self._validate_policy_item(policy_dict) policy_name = policy_dict['name'] # check name is valid @@ -101,6 +108,111 @@ class LibraryService (data_service.DataService): id_, policy_dict=policy_dict) return policy.to_dict() + def _validate_policy_item(self, item): + schema_json = ''' + { + "id": "PolicyProperties", + "title": "Policy Properties", + "type": "object", + "required": ["name", "rules"], + "properties": { + "name": { + "title": "Policy unique name", + "type": "string", + "minLength": 1, + "maxLength": 255 + }, + "description": { + "title": "Policy description", + "type": "string" + }, + "kind": { + "title": "Policy kind", + "type": "string", + "enum": ["database", "nonrecursive", "action", "materialized", + "delta", "datasource"] + }, + "abbreviation": { + "title": "Policy name abbreviation", + "type": "string", + "minLength": 1, + "maxLength": 5 + }, + "rules": { + "title": "collection of rules", + "type": "array", + "items": { + "type": "object", + "properties": { + "PolicyRule": { + "title": "Policy rule", + "type": "object", + "required": ["rule"], + "properties": { + "rule": { + "title": "Rule definition following policy grammar", + "type": "string" + }, + "name": { + "title": "User-friendly name", + "type": "string" + }, + "comment": { + "title": "User-friendly comment", + "type": "string" + } + } + } + } + } + } + } + } + ''' + try: + jsonschema.validate(item, json.loads(schema_json)) + except jsonschema.exceptions.ValidationError as ve: + raise exception.InvalidPolicyInput(data=str(ve)) + + def load_policies_from_files(self): + def _load_library_policy_file(full_path): + with open(full_path, "r") as stream: + policies = yaml.load_all(stream) + count = 0 + doc_num_in_file = 0 + for policy in policies: + try: + doc_num_in_file += 1 + self.create_policy(policy) + count += 1 + except db_exc.DBDuplicateEntry: + LOG.debug( + 'Library policy %s (number %s in file %s) already ' + 'exists (likely loaded by another Congress ' + 'instance). Skipping.', + policy.get('name', '[no name]'), + doc_num_in_file, full_path) + except exception.CongressException: + LOG.exception( + 'Library policy %s could not be loaded. Skipped. ' + 'YAML reproduced here %s', + policy.get('name', '[no name]'), + yaml.dumps(policy)) + return count + file_count = 0 + policy_count = 0 + for (dirpath, dirnames, filenames) in os.walk( + cfg.CONF.policy_library_path): + for filename in filenames: + count = _load_library_policy_file( + os.path.join(dirpath, filename)) + if count > 0: + file_count += 1 + policy_count += count + LOG.debug( + '%s library policies from %s files successfully loaded', + policy_count, file_count) + class DseLibraryServiceEndpoints(object): """RPC endpoints exposed by LibraryService.""" @@ -108,10 +220,8 @@ class DseLibraryServiceEndpoints(object): def __init__(self, data_service): self.data_service = data_service - def create_policy( - self, context, policy_dict): - return self.data_service.create_policy( - policy_dict) + def create_policy(self, context, policy_dict): + return self.data_service.create_policy(policy_dict) def get_policies(self, context, include_rules=True): return self.data_service.get_policies(include_rules) diff --git a/congress/tests/api/test_library_policy_model.py b/congress/tests/api/test_library_policy_model.py index 576446ba4..d5d62296b 100644 --- a/congress/tests/api/test_library_policy_model.py +++ b/congress/tests/api/test_library_policy_model.py @@ -20,6 +20,7 @@ from __future__ import absolute_import import copy from congress.api import webservice +from congress.db import db_library_policies from congress.tests.api import base as api_base from congress.tests import base @@ -32,6 +33,10 @@ class TestLibraryPolicyModel(base.SqlTestCase): self.library_policy_model = services['api']['api-library-policy'] self.node = services['node'] self.engine = services['engine'] + + # clear the library policies loaded on startup + db_library_policies.delete_policies() + self._add_test_policy() def _add_test_policy(self): @@ -96,7 +101,6 @@ class TestLibraryPolicyModel(base.SqlTestCase): del expected_ret['rules'] policy_id, policy_obj = self.library_policy_model.add_item(test, {}) - # self.assertEqual(test['id'], policy_id) test['id'] = policy_id self.assertEqual(test, policy_obj) @@ -108,9 +112,11 @@ class TestLibraryPolicyModel(base.SqlTestCase): "abbreviation": "abbr", "rules": [] } - self.library_policy_model.add_item(test, {}) + # duplicate name allowed + self.assertRaises(KeyError, + self.library_policy_model.add_item, test, {}) ret = self.library_policy_model.get_items({}) - self.assertEqual(len(ret['results']), 3) + self.assertEqual(len(ret['results']), 2) def test_add_item_with_id(self): test = { diff --git a/congress/tests/db/test_db_library_policies.py b/congress/tests/db/test_db_library_policies.py index ee71a1df8..571d90dad 100644 --- a/congress/tests/db/test_db_library_policies.py +++ b/congress/tests/db/test_db_library_policies.py @@ -25,6 +25,7 @@ class TestDbLibraryPolicies(base.SqlTestCase): def setUp(self): super(TestDbLibraryPolicies, self).setUp() + db_library_policies.delete_policies() # delete preloaded policies def test_add_policy_no_name(self): self.assertRaises( diff --git a/congress/tests/haht/test.db.clean b/congress/tests/haht/test.db.clean index f7c0a1dfe..8e92f21f9 100644 Binary files a/congress/tests/haht/test.db.clean and b/congress/tests/haht/test.db.clean differ diff --git a/congress/tests/library_service/test_library_service.py b/congress/tests/library_service/test_library_service.py index 340392388..29ccec617 100644 --- a/congress/tests/library_service/test_library_service.py +++ b/congress/tests/library_service/test_library_service.py @@ -29,6 +29,7 @@ class TestLibraryService(base.SqlTestCase): def setUp(self): super(TestLibraryService, self).setUp() self.library = library_service.LibraryService('lib-test') + self.library.delete_all_policies() # clear pre-loaded library policies self.policy1 = {'name': 'policy1', 'abbreviation': 'abbr', 'kind': 'database', 'description': 'descrip', @@ -48,12 +49,12 @@ class TestLibraryService(base.SqlTestCase): del self.policy2_meta['rules'] def test_create_policy_no_name(self): - self.assertRaises( - KeyError, self.library.create_policy, {'rules': []}) + self.assertRaises(exception.InvalidPolicyInput, + self.library.create_policy, {'rules': []}) def test_create_policy_no_rules(self): - self.assertRaises(KeyError, self.library.create_policy, - {'name': 'policy1'}) + self.assertRaises(exception.InvalidPolicyInput, + self.library.create_policy, {'name': 'policy1'}) def test_create_policy_bad_name(self): self.assertRaises(exception.PolicyException, @@ -73,9 +74,10 @@ class TestLibraryService(base.SqlTestCase): def test_create_policy_duplicate(self): self.library.create_policy({'name': 'policy1', 'rules': []}) - self.library.create_policy({'name': 'policy1', 'rules': []}) + self.assertRaises(KeyError, self.library.create_policy, + {'name': 'policy1', 'rules': []}) res = self.library.get_policies() - self.assertEqual(len(res), 2) + self.assertEqual(len(res), 1) def test_get_policy_empty(self): res = self.library.get_policies() @@ -140,15 +142,15 @@ class TestLibraryService(base.SqlTestCase): self.library.create_policy( {'name': 'policy1', 'abbreviation': 'abbr', 'kind': 'database', - 'description': 'descrip', 'rules': [[{'rule': 'p(x) :- q(x)', - 'comment': 'test comment', - 'name': 'testname'}]]}) + 'description': 'descrip', 'rules': [{'rule': 'p(x) :- q(x)', + 'comment': 'test comment', + 'name': 'testname'}]}) self.library.create_policy( {'name': 'policy2', 'abbreviation': 'abbr', 'kind': 'database', - 'description': 'descrip', 'rules': [[{'rule': 'p(x) :- q(x)', - 'comment': 'test comment', - 'name': 'testname'}]]}) + 'description': 'descrip', 'rules': [{'rule': 'p(x) :- q(x)', + 'comment': 'test comment', + 'name': 'testname'}]}) self.library.delete_all_policies() res = self.library.get_policies() @@ -157,15 +159,15 @@ class TestLibraryService(base.SqlTestCase): def test_replace_policy(self): policy1 = self.library.create_policy( {'name': 'policy1', 'abbreviation': 'abbr', 'kind': 'database', - 'description': 'descrip', 'rules': [[{'rule': 'p(x) :- q(x)', - 'comment': 'test comment', - 'name': 'testname'}]]}) + 'description': 'descrip', 'rules': [{'rule': 'p(x) :- q(x)', + 'comment': 'test comment', + 'name': 'testname'}]}) policy2 = self.library.create_policy( {'name': 'policy2', 'abbreviation': 'abbr', 'kind': 'database', - 'description': 'descrip', 'rules': [[{'rule': 'p(x) :- q(x)', - 'comment': 'test comment', - 'name': 'testname'}]]}) + 'description': 'descrip', 'rules': [{'rule': 'p(x) :- q(x)', + 'comment': 'test comment', + 'name': 'testname'}]}) replacement_policy = { "name": "new_name", diff --git a/congress/tests/test_congress.py b/congress/tests/test_congress.py index 2a1d3a6ba..932a49410 100644 --- a/congress/tests/test_congress.py +++ b/congress/tests/test_congress.py @@ -31,6 +31,7 @@ from congress.api import base as api_base from congress.common import config from congress.datasources import neutronv2_driver from congress.datasources import nova_driver +from congress.db import db_library_policies from congress.tests.api import base as tests_api_base from congress.tests import base from congress.tests.datasources import test_neutron_driver as test_neutron @@ -85,6 +86,9 @@ class TestCongress(BaseTestPolicyCongress): """Setup tests that use multiple mock neutron instances.""" super(TestCongress, self).setUp() + # clear the library policies loaded on startup + db_library_policies.delete_policies() + def tearDown(self): super(TestCongress, self).tearDown() diff --git a/congress_tempest_tests/tests/scenario/test_congress_basic_ops.py b/congress_tempest_tests/tests/scenario/test_congress_basic_ops.py index 5c4ddc672..a37da26ff 100644 --- a/congress_tempest_tests/tests/scenario/test_congress_basic_ops.py +++ b/congress_tempest_tests/tests/scenario/test_congress_basic_ops.py @@ -196,6 +196,10 @@ class TestPolicyLibraryBasicOps(manager_congress.ScenarioPolicyBase): response = self.admin_manager.congress_client.list_library_policy() initial_state = response['results'] + self.assertGreater( + len(initial_state), 0, 'library policy shows no policies, ' + 'indicating failed load-on-startup.') + test_policy = { "name": "test_policy", "description": "test policy description", diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 43bac77f4..be75114bf 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -51,6 +51,8 @@ function configure_congress { cp $CONGRESS_DIR/etc/api-paste.ini $CONGRESS_API_PASTE_FILE cp $CONGRESS_DIR/etc/policy.json $CONGRESS_POLICY_FILE + mkdir $CONGRESS_LIBRARY_DIR + cp $CONGRESS_DIR/library/* $CONGRESS_LIBRARY_DIR # Update either configuration file iniset $CONGRESS_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL diff --git a/devstack/settings b/devstack/settings index 5ff8d1b49..3760937ad 100644 --- a/devstack/settings +++ b/devstack/settings @@ -36,6 +36,8 @@ CONGRESS_REPLICATED=${CONGRESS_REPLICATED:-False} CONGRESS_TRANSPORT_URL=${CONGRESS_TRANSPORT_URL:-kombu+memory:////} # Mutli process deployment CONGRESS_MULTIPROCESS_DEPLOYMENT=${CONGRESS_MULTIPROCESS_DEPLOYMENT:-False} +# Directory path to library policy files +CONGRESS_LIBRARY_DIR=$CONGRESS_CONF_DIR/library # File path to predefined policy and rules CONGRESS_PREDEFINED_POLICY_FILE=${CONGRESS_PREDEFINED_POLICY_FILE:-""} diff --git a/library/pause_disallowed_flavors.yaml b/library/pause_disallowed_flavors.yaml index 6a7f32386..023b186f8 100644 --- a/library/pause_disallowed_flavors.yaml +++ b/library/pause_disallowed_flavors.yaml @@ -1,16 +1,17 @@ - - - -id: PauseBadFlavors -description: Pause any server using a flavor that is not permitted +--- +name: PauseBadFlavors +description: "Pause any server using a flavor that is not permitted" rules: - - comment: "User should customize this. Permitted flavors." - rule: permitted_flavor('m1.tiny') - - comment: "User should customize this. Permitted flavors." - rule: permitted_flavor('m1.large') - - rule: > - server_with_bad_flavor(id) :- nova:servers(id=id,flavor_id=flavor_id), nova:flavors(id=flavor_id, name=flavor), - not permitted_flavor(flavor) - - comment: "Remediation: Pause any VM that shows up in the server_with_bad_flavor table" - rule: "execute[nova:servers.pause(id)] :- server_with_bad_flavor(id), nova:servers(id,status='ACTIVE')" - + - + comment: "User should customize this. Permitted flavors." + rule: permitted_flavor('m1.tiny') + - + comment: "User should customize this. Permitted flavors." + rule: permitted_flavor('m1.large') + - + rule: > + "server_with_bad_flavor(id) :- nova:servers(id=id,flavor_id=flavor_id), + nova:flavors(id=flavor_id, name=flavor), not permitted_flavor(flavor)" + - + comment: "Remediation: Pause any VM that shows up in the server_with_bad_flavor table" + rule: "execute[nova:servers.pause(id)] :- server_with_bad_flavor(id), nova:servers(id,status='ACTIVE')" \ No newline at end of file diff --git a/releasenotes/notes/load-lib-policies-a5cca19f58f9030c.yaml b/releasenotes/notes/load-lib-policies-a5cca19f58f9030c.yaml new file mode 100644 index 000000000..aafce99cb --- /dev/null +++ b/releasenotes/notes/load-lib-policies-a5cca19f58f9030c.yaml @@ -0,0 +1,10 @@ +--- +prelude: > +upgrade: + - A new config option `policy_library_path` is added to the [DEFAULT] + section. The string option specifies the directory from which + Congress will load pre-written policies for easy activation later + by an administrator. + This option can be ignored if you do not want + Congress to load pre-written policies from files. Due to MySQL limitations, + the full path to each policy file cannot exceed 760 characters. diff --git a/releasenotes/notes/policy-lib-db-656f809410706e6a.yaml b/releasenotes/notes/policy-lib-db-656f809410706e6a.yaml index 0862bb0f3..1cee0fb29 100644 --- a/releasenotes/notes/policy-lib-db-656f809410706e6a.yaml +++ b/releasenotes/notes/policy-lib-db-656f809410706e6a.yaml @@ -1,5 +1,5 @@ --- prelude: > upgrade: - - A new database table `librarypolicies` is added; + - A new database table `library_policies` is added; alembic migration scripts included.