From b5ee1f6d176c4b77e50c4a1309228f4c8d928474 Mon Sep 17 00:00:00 2001 From: Dmitry Nikishov Date: Tue, 15 Nov 2016 10:31:50 +0000 Subject: [PATCH] Extended whitelist with task name Change-Id: Id832b0c66256b8808ac17c657c7077b6d2164803 --- README.md | 15 ++++- doc/api.md | 8 ++- doc/cli.md | 7 +- fuel_external_git/fuelclient_audit.py | 67 ++++++++++++++++++- fuel_external_git/json_schema.py | 1 + ...a7b6c_extended_whitelist_with_task_name.py | 42 ++++++++++++ fuel_external_git/models.py | 1 + fuel_external_git/objects.py | 3 +- setup.cfg | 1 + 9 files changed, 135 insertions(+), 10 deletions(-) create mode 100644 fuel_external_git/migrations/versions/fc4f164a7b6c_extended_whitelist_with_task_name.py diff --git a/README.md b/README.md index b50e106..b8e5d72 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ This command will run audit, check the changes and will enforce configuration, i Since fuel-library contains non-idempotent tasks, that contain Puppet resources, which will be triggered on each deployment run, this extension provides the operator the ability to filter such changes out. -A whitelist rule is a string, that is included into a Puppet report line for the whitelisted resource change, e.g. for +A whitelist rule is a pair of strings. The first one is a fuel task name to match. The second one is what should be included into a Puppet report line for the whitelisted resource change, e.g. for ``` Openstack_tasks::Swift::Proxy_storage/Package[mc]/ensure ``` @@ -159,14 +159,25 @@ the whitelist rule could be ``` Package[mc]/ensure ``` +A rule with empty fuel_task filter will match to all tasks. + Whitelist rules for an environment can be listed by ``` fuel2 audit whitelist show ``` These rules can be managed by following commands: ``` -fuel2 audit whitelist add +fuel2 audit whitelist add --task --rule fuel2 audit whitelist delete +fuel2 audit whitelist load fromfile +``` + +Example YAML file with whitelist rules: +``` +- fuel_task: netconfig + rule: L23_stored_configs +- fuel_task: top-role-compute + rule: Service[nova-compute]/ensure ``` ### REST API diff --git a/doc/api.md b/doc/api.md index a67f6c2..97756d0 100644 --- a/doc/api.md +++ b/doc/api.md @@ -54,11 +54,12 @@ Input data schema: "type": "object", "properties": { "rule": {"type": "string"}, + "fuel_task": {"type": "string"}, } ``` Example ``` -curl -H "X-Auth-Token: $(fuel token)" -X PUT http://localhost:8000/api/v1/clusters/changes-whitelist/1 -d '{"rule": "new-rule-string"}' +curl -H "X-Auth-Token: $(fuel token)" -X PUT http://localhost:8000/api/v1/clusters/changes-whitelist/1 -d '{"rule": "new-rule-string", "fuel_task": "fuel-task-id"}' ``` #### DELETE /clusters/changes-whitelist/(obj_id) @@ -83,10 +84,11 @@ Input data schema: "description": "Serialized ChangesWhitelistRule collection", "type": "object", "items": { - "rule": {"type": "string"} + "rule": {"type": "string"}, + "fuel_task": {"fuel_task": "string"}, } ``` Example ``` -curl -H "X-Auth-Token: $(fuel token)" -X POST http://localhost:8000/api/v1/clusters/1/changes-whitelist/ -d '[{"rule": "new-rule-string"}, {"rule": "new-rule-2"}]' +curl -H "X-Auth-Token: $(fuel token)" -X POST http://localhost:8000/api/v1/clusters/1/changes-whitelist/ -d '[{"rule": "new-rule-string", "fuel_task": "task1"}, {"rule": "new-rule-2", "fuel_task": ""}]' ``` diff --git a/doc/cli.md b/doc/cli.md index a8504fc..50537ba 100644 --- a/doc/cli.md +++ b/doc/cli.md @@ -77,10 +77,15 @@ fuel2 audit whitelist show To add a rule: ``` -fuel2 audit whitelist add +fuel2 audit whitelist add --task --rule ``` To delete a rule: ``` fuel2 audit whitelist delete ``` + +To add rules from YAML file: +``` +fuel2 audit whitelist load fromfile +``` diff --git a/fuel_external_git/fuelclient_audit.py b/fuel_external_git/fuelclient_audit.py index 1587930..0e38b6c 100644 --- a/fuel_external_git/fuelclient_audit.py +++ b/fuel_external_git/fuelclient_audit.py @@ -17,6 +17,7 @@ import time from cliff import command from cliff import lister +import yaml from fuelclient import client if hasattr(client, 'DefaultAPIClient'): @@ -76,14 +77,27 @@ class Audit(lister.Lister, command.Command): for task in changed_tasks: name = task['task_name'] for item in task['summary']['raw_report']: - if 'Would have triggered' not in item['message'] and \ - 'Finished catalog run' not in item['message']: + if item['source'].startswith('/Stage[main]/'): short_item = item['source'].replace('/Stage[main]/', '') changes.append({'task_id': name, 'resource': short_item, 'node_id': task['node_id']}) return changes + @staticmethod + def filter_changes(changes, env_id): + wl = fc_client.get_request( + '/clusters/{env}/changes-whitelist/'.format(env=env_id) + ) + + changes = filter(lambda c: + len(filter(lambda w: w['rule'] in c['resource'] and + (w['fuel_task'] == c['task_id'] or + w['fuel_task'] == ''), wl) == 0), + changes) + + return changes + def get_parser(self, prog_name): parser = super(Audit, self).get_parser(prog_name) group = parser.add_mutually_exclusive_group(required=True) @@ -209,7 +223,10 @@ class OutOfSyncResources(lister.Lister, command.Command): else: fuel_task = Task(task_id) + env_id = fuel_task.data['cluster'] + changes = Audit.get_outofsync(fuel_task) + changes = Audit.filter_changes(changes, env_id) data = data_utils.get_display_data_multi(self.columns, changes) return (self.columns, data) @@ -218,6 +235,7 @@ class OutOfSyncResources(lister.Lister, command.Command): class WhitelistRulesShow(lister.Lister, command.Command): columns = ( 'id', + 'fuel_task', 'rule' ) @@ -243,6 +261,7 @@ class WhitelistRulesShow(lister.Lister, command.Command): class WhitelistRuleAdd(lister.Lister, command.Command): columns = ( 'id', + 'fuel_task', 'rule' ) @@ -251,15 +270,24 @@ class WhitelistRuleAdd(lister.Lister, command.Command): parser.add_argument('env', type=int, help='Environment to add whitelist rules to') - parser.add_argument('rule', + parser.add_argument('--rule', '-r', type=str, + required=True, help='Rule to add') + parser.add_argument('--task', '-t', + type=str, + required=False, + help=('Fuel task for the rule. Omitting this will' + ' result in matching all Fuel tasks.')) return parser def take_action(self, parsed_args): env_id = parsed_args.env rule = parsed_args.rule + task = parsed_args.task data = {'rule': rule} + if task: + data['fuel_task'] = task ret = fc_client.post_request( '/clusters/{env}/changes-whitelist/'.format(env=env_id), @@ -288,3 +316,36 @@ class WhitelistRuleDelete(command.Command): ) return ((), {}) + + +class WhitelistRuleAddFromFile(lister.Lister, command.Command): + columns = ( + 'id', + 'fuel_task', + 'rule' + ) + + def get_parser(self, prog_name): + parser = super(WhitelistRuleAddFromFile, self).get_parser(prog_name) + parser.add_argument('env', + type=int, + help='Environment to add whitelist rules to') + parser.add_argument('file_name', + type=str, + help='YAML file to load rules from') + return parser + + def take_action(self, parsed_args): + env_id = parsed_args.env + file_name = parsed_args.file_name + + with open(file_name, 'r') as f: + data = yaml.load(f) + + ret = fc_client.post_request( + '/clusters/{env}/changes-whitelist/'.format(env=env_id), + data + ) + ret = data_utils.get_display_data_multi(self.columns, ret) + + return (self.columns, ret) diff --git a/fuel_external_git/json_schema.py b/fuel_external_git/json_schema.py index 8eea7dd..716d63f 100644 --- a/fuel_external_git/json_schema.py +++ b/fuel_external_git/json_schema.py @@ -42,6 +42,7 @@ changeswhitelistrule_single_schema = { "id": {"type": "number"}, "env_id": {"type": "number"}, "rule": {"type": "string"}, + "fuel_task": {"type": "string"}, } } diff --git a/fuel_external_git/migrations/versions/fc4f164a7b6c_extended_whitelist_with_task_name.py b/fuel_external_git/migrations/versions/fc4f164a7b6c_extended_whitelist_with_task_name.py new file mode 100644 index 0000000..659e697 --- /dev/null +++ b/fuel_external_git/migrations/versions/fc4f164a7b6c_extended_whitelist_with_task_name.py @@ -0,0 +1,42 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""extended whitelist with task name + +Revision ID: fc4f164a7b6c +Revises: 8736ad38ca31 +Create Date: 2016-11-15 09:09:46.300987 + +""" + +# revision identifiers, used by Alembic. +revision = 'fc4f164a7b6c' +down_revision = '8736ad38ca31' +branch_labels = None +depends_on = None + +from alembic import context +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + table_prefix = context.config.get_main_option('table_prefix') + op.add_column( + table_prefix + 'changes_whitelist', + sa.Column('fuel_task', sa.String, server_default='', nullable=False) + ) + + +def downgrade(): + table_prefix = context.config.get_main_option('table_prefix') + op.drop_column(table_prefix + 'changes_whitelist', 'fuel_task') diff --git a/fuel_external_git/models.py b/fuel_external_git/models.py index 963bea4..b581e7c 100644 --- a/fuel_external_git/models.py +++ b/fuel_external_git/models.py @@ -37,3 +37,4 @@ class ChangesWhitelistRule(Base): id = Column(Integer, primary_key=True) env_id = Column(Integer, nullable=False) rule = Column(String(255), server_default='', nullable=False) + fuel_task = Column(String(255), server_default='', nullable=False) diff --git a/fuel_external_git/objects.py b/fuel_external_git/objects.py index b6e786c..29da258 100644 --- a/fuel_external_git/objects.py +++ b/fuel_external_git/objects.py @@ -50,7 +50,8 @@ class ChangesWhitelistRuleSerializer(BasicSerializer): fields = ( "id", "env_id", - "rule" + "rule", + "fuel_task" ) diff --git a/setup.cfg b/setup.cfg index 3379ebe..755c673 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,4 +35,5 @@ fuelclient: audit_list_outofsync = fuel_external_git.fuelclient_audit:OutOfSyncResources audit_whitelist_show = fuel_external_git.fuelclient_audit:WhitelistRulesShow audit_whitelist_add = fuel_external_git.fuelclient_audit:WhitelistRuleAdd + audit_whitelist_load_fromfile = fuel_external_git.fuelclient_audit:WhitelistRuleAddFromFile audit_whitelist_delete = fuel_external_git.fuelclient_audit:WhitelistRuleDelete