Extended whitelist with task name

Change-Id: Id832b0c66256b8808ac17c657c7077b6d2164803
This commit is contained in:
Dmitry Nikishov 2016-11-15 10:31:50 +00:00
parent ec68268b3b
commit b5ee1f6d17
9 changed files with 135 additions and 10 deletions

View File

@ -151,7 +151,7 @@ This command will run audit, check the changes and will enforce configuration, i
Since fuel-library contains non-idempotent tasks, that contain Puppet resources, which will be
triggered on each deployment run, this extension provides the operator the ability to filter such changes out.
A whitelist rule is a string, that is included into a Puppet report line for the whitelisted resource change, e.g. for
A whitelist rule is a pair of strings. The first one is a fuel task name to match. The second one is what should be included into a Puppet report line for the whitelisted resource change, e.g. for
```
Openstack_tasks::Swift::Proxy_storage/Package[mc]/ensure
```
@ -159,14 +159,25 @@ the whitelist rule could be
```
Package[mc]/ensure
```
A rule with empty fuel_task filter will match to all tasks.
Whitelist rules for an environment can be listed by
```
fuel2 audit whitelist show <env-id>
```
These rules can be managed by following commands:
```
fuel2 audit whitelist add <env-id> <rule>
fuel2 audit whitelist add <env-id> --task <fuel-task> --rule <rule>
fuel2 audit whitelist delete <rule-id>
fuel2 audit whitelist load fromfile <env-id> <path-to-yaml>
```
Example YAML file with whitelist rules:
```
- fuel_task: netconfig
rule: L23_stored_configs
- fuel_task: top-role-compute
rule: Service[nova-compute]/ensure
```
### REST API

View File

@ -54,11 +54,12 @@ Input data schema:
"type": "object",
"properties": {
"rule": {"type": "string"},
"fuel_task": {"type": "string"},
}
```
Example
```
curl -H "X-Auth-Token: $(fuel token)" -X PUT http://localhost:8000/api/v1/clusters/changes-whitelist/1 -d '{"rule": "new-rule-string"}'
curl -H "X-Auth-Token: $(fuel token)" -X PUT http://localhost:8000/api/v1/clusters/changes-whitelist/1 -d '{"rule": "new-rule-string", "fuel_task": "fuel-task-id"}'
```
#### DELETE /clusters/changes-whitelist/(obj_id)
@ -83,10 +84,11 @@ Input data schema:
"description": "Serialized ChangesWhitelistRule collection",
"type": "object",
"items": {
"rule": {"type": "string"}
"rule": {"type": "string"},
"fuel_task": {"fuel_task": "string"},
}
```
Example
```
curl -H "X-Auth-Token: $(fuel token)" -X POST http://localhost:8000/api/v1/clusters/1/changes-whitelist/ -d '[{"rule": "new-rule-string"}, {"rule": "new-rule-2"}]'
curl -H "X-Auth-Token: $(fuel token)" -X POST http://localhost:8000/api/v1/clusters/1/changes-whitelist/ -d '[{"rule": "new-rule-string", "fuel_task": "task1"}, {"rule": "new-rule-2", "fuel_task": ""}]'
```

View File

@ -77,10 +77,15 @@ fuel2 audit whitelist show <env-id>
To add a rule:
```
fuel2 audit whitelist add <env-id> <rule>
fuel2 audit whitelist add <env-id> --task <fuel-task> --rule <rule>
```
To delete a rule:
```
fuel2 audit whitelist delete <rule-id>
```
To add rules from YAML file:
```
fuel2 audit whitelist load fromfile <env-id> <path-to-yaml>
```

View File

@ -17,6 +17,7 @@ import time
from cliff import command
from cliff import lister
import yaml
from fuelclient import client
if hasattr(client, 'DefaultAPIClient'):
@ -76,14 +77,27 @@ class Audit(lister.Lister, command.Command):
for task in changed_tasks:
name = task['task_name']
for item in task['summary']['raw_report']:
if 'Would have triggered' not in item['message'] and \
'Finished catalog run' not in item['message']:
if item['source'].startswith('/Stage[main]/'):
short_item = item['source'].replace('/Stage[main]/', '')
changes.append({'task_id': name,
'resource': short_item,
'node_id': task['node_id']})
return changes
@staticmethod
def filter_changes(changes, env_id):
wl = fc_client.get_request(
'/clusters/{env}/changes-whitelist/'.format(env=env_id)
)
changes = filter(lambda c:
len(filter(lambda w: w['rule'] in c['resource'] and
(w['fuel_task'] == c['task_id'] or
w['fuel_task'] == ''), wl) == 0),
changes)
return changes
def get_parser(self, prog_name):
parser = super(Audit, self).get_parser(prog_name)
group = parser.add_mutually_exclusive_group(required=True)
@ -209,7 +223,10 @@ class OutOfSyncResources(lister.Lister, command.Command):
else:
fuel_task = Task(task_id)
env_id = fuel_task.data['cluster']
changes = Audit.get_outofsync(fuel_task)
changes = Audit.filter_changes(changes, env_id)
data = data_utils.get_display_data_multi(self.columns, changes)
return (self.columns, data)
@ -218,6 +235,7 @@ class OutOfSyncResources(lister.Lister, command.Command):
class WhitelistRulesShow(lister.Lister, command.Command):
columns = (
'id',
'fuel_task',
'rule'
)
@ -243,6 +261,7 @@ class WhitelistRulesShow(lister.Lister, command.Command):
class WhitelistRuleAdd(lister.Lister, command.Command):
columns = (
'id',
'fuel_task',
'rule'
)
@ -251,15 +270,24 @@ class WhitelistRuleAdd(lister.Lister, command.Command):
parser.add_argument('env',
type=int,
help='Environment to add whitelist rules to')
parser.add_argument('rule',
parser.add_argument('--rule', '-r',
type=str,
required=True,
help='Rule to add')
parser.add_argument('--task', '-t',
type=str,
required=False,
help=('Fuel task for the rule. Omitting this will'
' result in matching all Fuel tasks.'))
return parser
def take_action(self, parsed_args):
env_id = parsed_args.env
rule = parsed_args.rule
task = parsed_args.task
data = {'rule': rule}
if task:
data['fuel_task'] = task
ret = fc_client.post_request(
'/clusters/{env}/changes-whitelist/'.format(env=env_id),
@ -288,3 +316,36 @@ class WhitelistRuleDelete(command.Command):
)
return ((), {})
class WhitelistRuleAddFromFile(lister.Lister, command.Command):
columns = (
'id',
'fuel_task',
'rule'
)
def get_parser(self, prog_name):
parser = super(WhitelistRuleAddFromFile, self).get_parser(prog_name)
parser.add_argument('env',
type=int,
help='Environment to add whitelist rules to')
parser.add_argument('file_name',
type=str,
help='YAML file to load rules from')
return parser
def take_action(self, parsed_args):
env_id = parsed_args.env
file_name = parsed_args.file_name
with open(file_name, 'r') as f:
data = yaml.load(f)
ret = fc_client.post_request(
'/clusters/{env}/changes-whitelist/'.format(env=env_id),
data
)
ret = data_utils.get_display_data_multi(self.columns, ret)
return (self.columns, ret)

View File

@ -42,6 +42,7 @@ changeswhitelistrule_single_schema = {
"id": {"type": "number"},
"env_id": {"type": "number"},
"rule": {"type": "string"},
"fuel_task": {"type": "string"},
}
}

View File

@ -0,0 +1,42 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""extended whitelist with task name
Revision ID: fc4f164a7b6c
Revises: 8736ad38ca31
Create Date: 2016-11-15 09:09:46.300987
"""
# revision identifiers, used by Alembic.
revision = 'fc4f164a7b6c'
down_revision = '8736ad38ca31'
branch_labels = None
depends_on = None
from alembic import context
from alembic import op
import sqlalchemy as sa
def upgrade():
table_prefix = context.config.get_main_option('table_prefix')
op.add_column(
table_prefix + 'changes_whitelist',
sa.Column('fuel_task', sa.String, server_default='', nullable=False)
)
def downgrade():
table_prefix = context.config.get_main_option('table_prefix')
op.drop_column(table_prefix + 'changes_whitelist', 'fuel_task')

View File

@ -37,3 +37,4 @@ class ChangesWhitelistRule(Base):
id = Column(Integer, primary_key=True)
env_id = Column(Integer, nullable=False)
rule = Column(String(255), server_default='', nullable=False)
fuel_task = Column(String(255), server_default='', nullable=False)

View File

@ -50,7 +50,8 @@ class ChangesWhitelistRuleSerializer(BasicSerializer):
fields = (
"id",
"env_id",
"rule"
"rule",
"fuel_task"
)

View File

@ -35,4 +35,5 @@ fuelclient:
audit_list_outofsync = fuel_external_git.fuelclient_audit:OutOfSyncResources
audit_whitelist_show = fuel_external_git.fuelclient_audit:WhitelistRulesShow
audit_whitelist_add = fuel_external_git.fuelclient_audit:WhitelistRuleAdd
audit_whitelist_load_fromfile = fuel_external_git.fuelclient_audit:WhitelistRuleAddFromFile
audit_whitelist_delete = fuel_external_git.fuelclient_audit:WhitelistRuleDelete