From c2d906f5aee6f64bb1074d9df3b405bc181d428d Mon Sep 17 00:00:00 2001 From: Ilya Kutukov Date: Wed, 8 Jun 2016 17:58:59 +0300 Subject: [PATCH] Now FPB tasks schema in sync with Fuel 9.0 with 2.1 support YAQL expressions added. Tasks 2.0 and 2.1 schema coverage improved. Change-Id: If433f29283cb4897e8137ba2b33215af14103bea Closes-Bug: #1590389 --- .../tests/test_validator_v4.py | 405 ++++++++++++++++++ fuel_plugin_builder/validators/schemas/v4.py | 61 ++- 2 files changed, 454 insertions(+), 12 deletions(-) diff --git a/fuel_plugin_builder/tests/test_validator_v4.py b/fuel_plugin_builder/tests/test_validator_v4.py index 4e2a784..e456a5d 100644 --- a/fuel_plugin_builder/tests/test_validator_v4.py +++ b/fuel_plugin_builder/tests/test_validator_v4.py @@ -493,6 +493,411 @@ class TestValidatorV4(TestValidatorV3): def test_check_tasks_schema_validation_passed(self, utils_mock, *args): pass + @mock.patch('fuel_plugin_builder.validators.validator_v4.utils') + def test_check_tasks_schema_1_0_validation_failed(self, utils_mock, *args): + checks = [ + { + 'data': { + 'id': 'task-id', + 'type': 'shell', + 'parameters': { + 'timeout': 3 + }, + 'stage': 'post_deployment', + 'role': '*' + }, + 'errorTextContains': "'cmd' is a required property, " + "value path '0 -> parameters'" + }, + { + 'data': { + 'id': 'task-id', + 'type': 'puppet', + 'parameters': { + 'timeout': 3 + }, + 'stage': 'post_deployment', + 'role': '*' + }, + 'errorTextContains': "'puppet_manifest' is a required property" + ", value path '0 -> parameters'" + }, + { + 'data': { + 'id': 'task-id', + 'type': 'puppet', + 'parameters': { + 'timeout': 3, + 'cmd': 'xx' + }, + 'stage': 'post_deployment', + 'role': '*' + }, + 'errorTextContains': "'puppet_manifest' is a required property" + ", value path '0 -> parameters'" + }, + { + 'data': { + 'id': 'task-id', + 'type': 'shell', + 'parameters': { + 'timeout': 3, + 'puppet_manifest': 'xx', + 'puppet_modules': 'yy', + }, + 'stage': 'post_deployment', + 'role': '*' + }, + 'errorTextContains': "'cmd' is a required property, value path" + " '0 -> parameters'" + }, + { + 'data': { + 'id': 'task-id', + 'type': 'puppet', + 'parameters': { + 'timeout': 3, + 'puppet_manifest': 'xx', + 'puppet_modules': 'yy', + 'retries': 'asd', + }, + 'stage': 'post_deployment', + 'role': '*' + }, + 'errorTextContains': "'asd' is not of type 'integer', value " + "path '0 -> parameters -> retries'" + }, + { + 'data': { + 'id': 'task-id', + 'type': 'puppet', + 'parameters': { + 'timeout': 3, + 'puppet_manifest': 'xx', + 'puppet_modules': '', + 'retries': 1, + }, + 'stage': 'pre_deployment', + 'role': '*' + }, + 'errorTextContains': "'' is too short, value path '0 -> " + "parameters -> puppet_modules'" + }, + { + 'data': { + 'id': 'task-id', + 'type': 'puppet', + 'parameters': { + 'timeout': 3, + 'puppet_manifest': '', + 'puppet_modules': 'yy', + 'retries': 1, + }, + 'stage': 'pre_deployment', + 'role': '*' + }, + 'errorTextContains': "'' is too short, value path '0 -> " + "parameters -> puppet_manifest'" + } + ] + + for check in checks: + utils_mock.parse_yaml.return_value = [check['data']] + self.assertRaisesRegexp( + errors.ValidationError, + check['errorTextContains'], + self.validator.check_deployment_tasks) + + @mock.patch('fuel_plugin_builder.validators.validator_v4.utils') + def test_check_tasks_schema_1_0_validation_passed(self, utils_mock, *args): + data_sets = [ + [ + { + 'id': 'task_id', + 'type': 'shell', + 'parameters': { + 'timeout': 3, + 'cmd': 'xx' + }, + 'stage': 'post_deployment', + 'role': '*' + }, + ], + [ + { + 'id': 'task_id', + 'type': 'shell', + 'parameters': { + 'timeout': 3, + 'cmd': 'xx' + }, + 'stage': 'post_deployment', + 'role': '*' + }, + { + 'id': 'task_id', + 'type': 'puppet', + 'parameters': { + 'timeout': 3, + 'puppet_manifest': 'xx', + 'puppet_modules': 'xxx' + }, + 'stage': 'post_deployment', + 'role': '*' + }, + ], + [ + { + 'id': 'task_id', + 'type': 'shell', + 'parameters': { + 'timeout': 3, + 'cmd': 'reboot' + }, + 'stage': 'post_deployment', + 'role': '*' + }, + { + 'id': 'task_id', + 'type': 'shell', + 'parameters': { + 'timeout': 3, + 'cmd': 'xx' + }, + 'stage': 'post_deployment', + 'role': '*' + }, + { + 'id': 'task_id', + 'type': 'puppet', + 'parameters': { + 'timeout': 3, + 'puppet_manifest': 'xx', + 'puppet_modules': 'xxx' + }, + 'stage': 'post_deployment', + 'role': '*' + } + ], + [ + { + 'id': 'task_id', + 'type': 'shell', + 'parameters': { + 'timeout': 3, + 'cmd': 'reboot' + }, + 'stage': 'post_deployment', + 'role': '*' + }, + { + 'id': 'task_id', + 'type': 'shell', + 'parameters': { + 'timeout': 3, + 'puppet_manifest': 'xx', + 'puppet_modules': 'yy', + 'cmd': 'reboot' + }, + 'stage': 'post_deployment', + 'role': '*' + }, + { + 'id': 'task_id', + 'type': 'puppet', + 'parameters': { + 'timeout': 3, + 'retries': 10, + 'puppet_manifest': 'xx', + 'puppet_modules': 'xxx' + }, + 'stage': 'post_deployment', + 'role': '*' + }, + { + 'id': 'task_id', + 'type': 'puppet', + 'parameters': { + 'timeout': 3, + 'retries': 10, + 'puppet_manifest': 'xx', + 'puppet_modules': 'xxx' + }, + 'stage': 'post_deployment', + 'role': 'master' + }, + ] + ] + + for data in data_sets: + utils_mock.parse_yaml.return_value = data + self.validator.check_deployment_tasks() + + @mock.patch('fuel_plugin_builder.validators.validator_v4.utils') + def test_check_tasks_schema_2_0_validation_failed(self, utils_mock, *args): + tasks_data = [ + { + 'id': 'test', + 'type': 'shell', + 'version': '2' + }, + { + 'id': 'test', + 'type': 'shell', + 'cross-depends': [ + { + 'role': 'role_without_name' + } + ] + }, + { + 'id': 'test', + 'type': 'shell', + 'parameters': { + 'strategy': 'NOSUCHSTRATEGY' + } + }, + { + 'id': 'test', + 'type': 'shell', + 'parameters': { + 'strategy': { + 'type': 'NOSUCHSTRATEGY' + } + } + } + ] + + utils_mock.parse_yaml.return_value = tasks_data + self.assertRaises(errors.ValidationError, + self.validator.check_deployment_tasks) + + @mock.patch('fuel_plugin_builder.validators.validator_v4.utils') + def test_check_tasks_schema_2_0_validation_passed(self, utils_mock, *args): + tasks_data = [ + { + 'id': 'task_id', + 'type': 'puppet', + 'version': '2.0.0', + 'parameters': { + 'timeout': 3, + 'retries': 10, + 'puppet_manifest': 'xx', + 'puppet_modules': 'xxx' + }, + 'stage': 'post_deployment', + 'roles': ['test_role'], + 'cross-depends': [ + { + 'name': 'task_id2', + }, + { + 'name': 'task_id2', + 'role': ['some_role'] + }, + { + 'name': 'task_id2', + 'role': 'some_role' + }, + { + 'name': 'task_id2', + 'policy': 'all' + }, + { + 'name': 'task_id2', + 'policy': 'any' + } + ], + 'cross-depended-by': [ + { + 'name': 'task_id2', + }, + { + 'name': 'task_id2', + 'role': ['some_role'] + }, + { + 'name': 'task_id2', + 'role': 'some_role' + }, + { + 'name': 'task_id2', + 'policy': 'all' + }, + { + 'name': 'task_id2', + 'policy': 'any' + } + ], + 'strategy': { + 'type': 'parallel', + 'amount': 10 + } + } + ] + + utils_mock.parse_yaml.return_value = tasks_data + self.validator.check_deployment_tasks() + + @mock.patch('fuel_plugin_builder.validators.validator_v4.utils') + def test_check_tasks_schema_2_1_validation_passed(self, utils_mock, *args): + # this is a slightly modified task from netconfig.yaml + tasks_data = [ + { + "id": "netconfig", + "type": "puppet", + "version": "2.1.0", + "groups": [ + "primary-controller", + "controller", + ], + "required_for": [ + "deploy_end" + ], + "requires": [ + "tools" + ], + "condition": { + "yaql_exp": "changedAny($.network_scheme, $.dpdk, $.get('" + "use_ovs'), $.get('set_rps'), $.get('set_rps')" + ", $.get('run_ping_checker'), $.network_scheme" + ".endpoints.values().where(\n $.get('gateway'" + ") != null).gateway)\n" + }, + "parameters": { + "puppet_manifest": "/etc/puppet/modules/osnailyfacter/" + "modular/netconfig/netconfig.pp", + "puppet_modules": "/etc/puppet/modules", + "timeout": 300, + "strategy": { + "type": "parallel", + "amount": { + "yaql_exp": "switch($.get('deployed_before', {})." + "get('value') => 1, true => 3)\n" + } + } + }, + "test_pre": { + "cmd": "ruby /etc/puppet/modules/osnailyfacter/modular/" + "netconfig/netconfig_pre.rb" + }, + "test_post": { + "cmd": "ruby /etc/puppet/modules/osnailyfacter/modular/" + "netconfig/netconfig_post.rb" + }, + "cross-depends": { + "yaql_exp": "switch( (\n $.roles.any($.matches('(" + "primary-)?(controller|mongo)'))\n " + "or ($.network_metadata.get('vips',{}).get" + "('management') = null)\n ) => [],\n " + "true => [{name =>'virtual_ips'}]\n)\n" + } + } + ] + + utils_mock.parse_yaml.return_value = tasks_data + self.validator.check_deployment_tasks() + @mock.patch('fuel_plugin_builder.validators.base.utils.exists') def test_check_tasks_schema_validation_no_file(self, exists_mock, *args): mocked_methods = ['validate_schema'] diff --git a/fuel_plugin_builder/validators/schemas/v4.py b/fuel_plugin_builder/validators/schemas/v4.py index dbb370f..f6781d3 100644 --- a/fuel_plugin_builder/validators/schemas/v4.py +++ b/fuel_plugin_builder/validators/schemas/v4.py @@ -47,18 +47,43 @@ class SchemaV4(SchemaV3): self.roleless_tasks = ROLELESS_TASKS self.role_aliases = ROLE_ALIASES + @property + def _node_resolve_policy(self): + return { + 'type': 'string', + 'enum': ['all', 'any'] + } + + @property + def _yaql_expression(self): + return { + '$schema': 'http://json-schema.org/draft-04/schema#', + 'type': 'object', + 'required': ['yaql_exp'], + 'properties': { + 'yaql_exp': {'type': 'string'}, + } + } + @property def _task_relation(self): return { + '$schema': 'http://json-schema.org/draft-04/schema#', 'type': 'object', 'required': ['name'], 'properties': { - 'name': {'type': 'string'}, - 'role': self._task_role, - 'policy': { - 'type': 'string', - 'enum': ['all', 'any'] - } + 'name': { + 'oneOf': [ + {'type': 'string'}, + self._yaql_expression], + }, + 'role': { + 'oneOf': [ + {'type': 'string'}, + {'type': 'array'}, + self._yaql_expression] + }, + 'policy': self._node_resolve_policy, } } @@ -83,10 +108,18 @@ class SchemaV4(SchemaV3): @property def _task_strategy(self): return { + '$schema': 'http://json-schema.org/draft-04/schema#', 'type': 'object', + 'required': ['type'], 'properties': { 'type': { - 'enum': ['parallel', 'one_by_one'] + 'type': 'string', + 'enum': ['parallel', 'one_by_one']}, + 'amount': { + 'oneOf': [ + {'type': 'integer'}, + self._yaql_expression + ] } } } @@ -149,11 +182,15 @@ class SchemaV4(SchemaV3): 'required_for': self.task_group, 'requires': self.task_group, 'cross-depends': { - 'type': 'array', - 'items': self._task_relation}, + 'oneOf': [ + {'type': 'array', 'items': self._task_relation}, + self._yaql_expression] + }, 'cross-depended-by': { - 'type': 'array', - 'items': self._task_relation}, + 'oneOf': [ + {'type': 'array', 'items': self._task_relation}, + self._yaql_expression] + }, 'stage': self._task_stage, 'tasks': { # used only for 'group' tasks 'type': 'array', @@ -161,7 +198,7 @@ class SchemaV4(SchemaV3): 'type': 'string', 'pattern': TASK_ROLE_PATTERN}}, 'reexecute_on': self._task_reexecute, - 'parameters': parameters or {}, + 'parameters': parameters, }, }