openstack
/
rally
Code Issues Proposed changes

Split validation at 3 layers 

We can group validation by 3 types: syntax (check that config is valid
for plugin in terms of jsonschema, required arguments, etc),
platform (check that deployment has credentials for launching
the plugin on specific environment), semantic(all others which operates
users: for example validate that specific service is enabled).

Splitting validation by these groups is required to support "validators
for validators". I mean that if we check that syntax of config is valid
and that we have credentials for all required platforms, we able to
execute more complex validators and ensure that they will not failed on
"random" failures like "AttributeError", "KeyError" and etc.

Change-Id: I1ea38403af6cfbd6b7765ae3c0caca12759bc571
This commit is contained in:
Andrey Kurilin 2017-04-11 17:50:17 +03:00
parent 3d113b5a23
commit d12632579f
9 changed files with 304 additions and 269 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
rally/cli/cliutils.py
View File

@ -664,7 +664,8 @@ def run(argv, categories):
exceptions.RallyException, jsonschema.ValidationError) as e:
if logging.is_debug():
LOG.exception(e)
print(e)
else:
print(e)
return 1
except sqlalchemy.exc.OperationalError as e:
if logging.is_debug():

55
rally/common/validation.py
View File

@ -18,12 +18,16 @@ import traceback
import six
from rally.common import logging
from rally.common.plugin import plugin
from rally import exceptions
configure = plugin.configure
LOG = logging.getLogger(__name__)
@plugin.base()
@six.add_metaclass(abc.ABCMeta)
class Validator(plugin.Plugin):
@ -76,7 +80,12 @@ class RequiredPlatformValidator(Validator):
if self.admin and credentials.get("admin") is None:
return self.fail("No admin credential for %s" % self.platform)
if self.users and len(credentials.get("users", ())) == 0:
return self.fail("No user credentials for %s" % self.platform)
if credentials.get("admin") is not None:
LOG.debug("Plugin %s requires 'users' for launching. There "
"are no specified users, assumes that 'users' "
"context can create them via admin user.")
else:
return self.fail("No user credentials for %s" % self.platform)
def add(name, **kwargs):
@ -153,7 +162,7 @@ class ValidatablePluginMixin(object):
@classmethod
def validate(cls, name, credentials, config, plugin_cfg,
namespace=None, allow_hidden=False):
namespace=None, allow_hidden=False, vtype=None):
"""Execute all validators stored in meta of plugin.
Iterate during all validators stored in the meta of Validator
@ -165,6 +174,10 @@ class ValidatablePluginMixin(object):
:param credentials: credentials dict for all platforms
:param config: dict with configuration of specified workload
:param plugin_cfg: dict, with exact configuration of the plugin
:param allow_hidden: do not ignore hidden plugins
:param vtype: Type of validation. Allowed types: syntax, platform,
semantic. HINT: To specify several types use tuple or list with
types
:returns: list of ValidationResult(is_valid=False) instances
"""
try:
@ -175,21 +188,45 @@ class ValidatablePluginMixin(object):
cls.__name__, name)
return [ValidationResult(is_valid=False, msg=msg)]
if vtype is None:
semantic = True
syntax = True
platform = True
else:
if not isinstance(vtype, (list, tuple)):
vtype = [vtype]
wrong_types = set(vtype) - {"semantic", "syntax", "platform"}
if wrong_types:
raise ValueError("Wrong type of validation: %s" %
", ".join(wrong_types))
semantic = "semantic" in vtype
syntax = "syntax" in vtype
platform = "platform" in vtype
syntax_validators = []
platform_validators = []
regular_validators = []
plugin_validators = cls._load_validators(plugin)
for validator, args, kwargs in plugin_validators:
if issubclass(validator, RequiredPlatformValidator):
platform_validators.append((validator, args, kwargs))
if platform:
platform_validators.append((validator, args, kwargs))
else:
regular_validators.append((validator, args, kwargs))
# Load platform validators from each validator
platform_validators.extend(cls._load_validators(validator))
validators_of_validators = cls._load_validators(validator)
if validators_of_validators:
if semantic:
regular_validators.append((validator, args, kwargs))
if platform:
# Load platform validators from each validator
platform_validators.extend(validators_of_validators)
else:
if syntax:
syntax_validators.append((validator, args, kwargs))
results = []
for validators in (platform_validators, regular_validators):
for validators in (syntax_validators, platform_validators,
regular_validators):
for validator_cls, args, kwargs in validators:
try:
validator = validator_cls(*args, **kwargs)
@ -207,6 +244,8 @@ class ValidatablePluginMixin(object):
etype=type(exc).__name__,
etraceback=traceback.format_exc())
if not result.is_valid:
LOG.debug("Result of validator '%s' is not successful for "
"plugin %s.", validator_cls.get_name(), name)
results.append(result)
if results:

2
rally/plugins/openstack/context/keystone/users.py
View File

@ -30,6 +30,7 @@ from rally.plugins.openstack.services.identity import identity
from rally.plugins.openstack.wrappers import network
from rally.task import context
from rally.task import utils
from rally.task import validation
from rally.common import opts
opts.register()
@ -45,6 +46,7 @@ PROJECT_DOMAIN_DESCR = "ID of domain in which projects will be created."
USER_DOMAIN_DESCR = "ID of domain in which users will be created."
@validation.add("required_platform", platform="openstack", admin=True)
@context.configure(name="users", namespace="openstack", order=100)
class UserGenerator(context.Context):
"""Context class for generating temporary users/tenants for benchmarks."""

172
rally/task/engine.py
View File

@ -261,80 +261,92 @@ class TaskEngine(object):
self.deployment = deployment
self.abort_on_sla_failure = abort_on_sla_failure
@logging.log_task_wrapper(LOG.info,
_("Task validation of scenarios names."))
def _validate_config_scenarios_name(self, config):
available = set(s.get_name() for s in scenario.Scenario.get_all())
def _validate_workload(self, workload, credentials=None, vtype=None):
scenario_cls = scenario.Scenario.get(workload.name)
namespace = scenario_cls.get_namespace()
scenario_context = copy.deepcopy(scenario_cls.get_default_context())
specified = set()
for subtask in config.subtasks:
for s in subtask.workloads:
specified.add(s.name)
results = []
if not specified.issubset(available):
names = ", ".join(specified - available)
raise exceptions.NotFoundScenarios(names=names)
results.extend(scenario.Scenario.validate(
name=workload.name,
credentials=credentials,
config=workload.to_dict(),
plugin_cfg=None,
vtype=vtype))
if workload.runner:
results.extend(runner.ScenarioRunner.validate(
name=workload.runner["type"],
credentials=credentials,
config=None,
plugin_cfg=workload.runner,
namespace=namespace,
vtype=vtype))
for context_name, context_conf in workload.context.items():
results.extend(context.Context.validate(
name=context_name,
credentials=credentials,
config=None,
plugin_cfg=context_conf,
namespace=namespace,
vtype=vtype))
for context_name, context_conf in scenario_context.items():
results.extend(context.Context.validate(
name=context_name,
credentials=credentials,
config=None,
plugin_cfg=context_conf,
namespace=namespace,
allow_hidden=True,
vtype=vtype))
for sla_name, sla_conf in workload.sla.items():
results.extend(sla.SLA.validate(
name=sla_name,
credentials=credentials,
config=None,
plugin_cfg=sla_conf,
vtype=vtype))
for hook_conf in workload.hooks:
results.extend(hook.Hook.validate(
name=hook_conf["name"],
credentials=credentials,
config=None,
plugin_cfg=hook_conf["args"],
vtype=vtype))
trigger_conf = hook_conf["trigger"]
results.extend(trigger.Trigger.validate(
name=trigger_conf["name"],
credentials=credentials,
config=None,
plugin_cfg=trigger_conf["args"],
vtype=vtype))
if results:
msg = "\n ".join([str(r) for r in results])
kw = workload.make_exception_args(msg)
raise exceptions.InvalidTaskConfig(**kw)
@logging.log_task_wrapper(LOG.info, _("Task validation of syntax."))
def _validate_config_syntax(self, config):
for subtask in config.subtasks:
for workload in subtask.workloads:
scenario_cls = scenario.Scenario.get(workload.name)
namespace = scenario_cls.get_namespace()
scenario_context = copy.deepcopy(
scenario_cls.get_default_context())
self._validate_workload(workload, vtype="syntax")
results = []
if workload.runner:
results.extend(runner.ScenarioRunner.validate(
name=workload.runner["type"],
credentials=None,
config=None,
plugin_cfg=workload.runner,
namespace=namespace))
for context_name, context_conf in workload.context.items():
results.extend(context.Context.validate(
name=context_name,
credentials=None,
config=None,
plugin_cfg=context_conf,
namespace=namespace))
for context_name, context_conf in scenario_context.items():
results.extend(context.Context.validate(
name=context_name,
credentials=None,
config=None,
plugin_cfg=context_conf,
namespace=namespace,
allow_hidden=True))
for sla_name, sla_conf in workload.sla.items():
results.extend(sla.SLA.validate(
name=sla_name,
credentials=None,
config=None,
plugin_cfg=sla_conf))
for hook_conf in workload.hooks:
results.extend(hook.Hook.validate(
name=hook_conf["name"],
credentials=None,
config=None,
plugin_cfg=hook_conf["args"]))
trigger_conf = hook_conf["trigger"]
results.extend(trigger.Trigger.validate(
name=trigger_conf["name"],
credentials=None,
config=None,
plugin_cfg=trigger_conf["args"]))
if results:
msg = "\n ".join([str(r) for r in results])
kw = workload.make_exception_args(msg)
raise exceptions.InvalidTaskConfig(**kw)
@logging.log_task_wrapper(LOG.info, _("Task validation of required "
"platforms."))
def _validate_config_platforms(self, config):
credentials = self.deployment.get_all_credentials()
credentials = dict((p, creds[0]) for p, creds in credentials.items())
for subtask in config.subtasks:
for workload in subtask.workloads:
self._validate_workload(workload, vtype="platform",
credentials=credentials)
def _validate_config_semantic_helper(self, admin, user_context,
workloads, platform):
@ -342,15 +354,9 @@ class TaskEngine(object):
ctx.setup()
users = ctx.context["users"]
for workload in workloads:
results = scenario.Scenario.validate(
name=workload.name,
credentials={platform: {"admin": admin, "users": users}},
config=workload.to_dict(),
plugin_cfg=None)
if results:
msg = "\n ".join([str(r) for r in results])
kw = workload.make_exception_args(msg)
raise exceptions.InvalidTaskConfig(**kw)
credentials = {platform: {"admin": admin, "users": users}}
self._validate_workload(workload, credentials=credentials,
vtype="semantic")
@logging.log_task_wrapper(LOG.info, _("Task validation of semantic."))
def _validate_config_semantic(self, config):
@ -405,19 +411,25 @@ class TaskEngine(object):
platform)
@logging.log_task_wrapper(LOG.info, _("Task validation."))
def validate(self):
"""Perform full task configuration validation."""
def validate(self, only_syntax=False):
"""Perform full task configuration validation.
:param only_syntax: Check only syntax of task configuration
"""
self.task.update_status(consts.TaskStatus.VALIDATING)
try:
self._validate_config_scenarios_name(self.config)
self._validate_config_syntax(self.config)
if only_syntax:
return
self._validate_config_platforms(self.config)
self._validate_config_semantic(self.config)
except Exception as e:
exception_info = json.dumps(traceback.format_exc(), indent=2,
separators=(",", ": "))
self.task.set_failed(type(e).__name__,
str(e), exception_info)
if logging.is_debug():
if (logging.is_debug() and
not isinstance(e, exceptions.InvalidTaskConfig)):
LOG.exception(e)
raise exceptions.InvalidTaskException(str(e))
@ -782,5 +794,5 @@ class Workload(object):
def make_exception_args(self, reason):
return {"name": self.name,
"pos": self.pos,
"config": self.to_dict(),
"config": json.dumps(self.to_dict()),
"reason": reason}

2
rally/task/validation.py
View File

@ -39,6 +39,8 @@ ValidationResult = validation.ValidationResult
add = validation.add
@validation.add("required_platform", platform="openstack", admin=True,
users=True)
@validation.configure(name="old_validator", namespace="openstack")
class OldValidator(validation.Validator):
"""Legacy validator for OpenStack scenarios"""

7
tests/unit/common/test_validation.py
View File

@ -118,7 +118,12 @@ class RequiredPlatformValidatorTestCase(test.TestCase):
{"kwargs": {"platform": "foo", "admin": True},
"credentials": {"foo": {"admin": "fake_admin"}}},
{"kwargs": {"platform": "foo", "admin": True, "users": True},
"credentials": {"foo": {"admin": "fake_admin"}},
"credentials": {"foo": {"admin": "fake_admin"}}},
{"kwargs": {"platform": "foo", "admin": True, "users": True},
"credentials": {"foo": {"users": ["fake_user"]}},
"error_msg": "No admin credential for foo"},
{"kwargs": {"platform": "foo", "users": True},
"credentials": {"foo": {}},
"error_msg": "No user credentials for foo"},
{"kwargs": {"platform": "foo", "admin": True, "users": True},
"credentials": {"foo": {"admin": "fake_admin",

7
tests/unit/doc/test_task_samples.py
View File

@ -40,10 +40,7 @@ class TaskSampleTestCase(test.TestCase):
if os.environ.get("TOX_ENV_NAME") == "cover":
self.skipTest("There is no need to check samples in coverage job.")
@mock.patch("rally.task.engine.TaskEngine"
"._validate_config_semantic")
def test_schema_is_valid(self,
mock_task_engine__validate_config_semantic):
def test_schema_is_valid(self):
scenarios = set()
for dirname, dirnames, filenames in os.walk(self.samples_path):
@ -61,7 +58,7 @@ class TaskSampleTestCase(test.TestCase):
(task_file.read()))
eng = engine.TaskEngine(task_config,
mock.MagicMock(), mock.Mock())
eng.validate()
eng.validate(only_syntax=True)
except Exception:
print(traceback.format_exc())
self.fail("Invalid task file: %s" % full_path)

7
tests/unit/rally_jobs/test_jobs.py
View File

@ -29,10 +29,7 @@ class RallyJobsTestCase(test.TestCase):
rally_jobs_path = os.path.join(
os.path.dirname(rally.__file__), "..", "rally-jobs")
@mock.patch("rally.task.engine.TaskEngine"
"._validate_config_semantic")
def test_schema_is_valid(
self, mock_task_engine__validate_config_semantic):
def test_schema_is_valid(self):
discover.load_plugins(os.path.join(self.rally_jobs_path, "plugins"))
files = {f for f in os.listdir(self.rally_jobs_path)
@ -64,7 +61,7 @@ class RallyJobsTestCase(test.TestCase):
eng = engine.TaskEngine(task, mock.MagicMock(),
mock.Mock())
eng.validate()
eng.validate(only_syntax=True)
except Exception:
print(traceback.format_exc())
self.fail("Wrong task input file: %s" % full_path)

318
tests/unit/task/test_engine.py
View File

@ -16,6 +16,7 @@
"""Tests for the Test engine."""
import collections
import json
import threading
import mock
@ -62,18 +63,15 @@ class TaskEngineTestCase(test.TestCase):
mock.Mock())
mock_validate = mock.MagicMock()
eng._validate_config_scenarios_name = mock_validate.names
eng._validate_config_syntax = mock_validate.syntax
eng._validate_config_platforms = mock_validate.platforms
eng._validate_config_semantic = mock_validate.semantic
eng.validate()
expected_calls = [
mock.call.names(config),
mock.call.syntax(config),
mock.call.semantic(config)
]
mock_validate.assert_has_calls(expected_calls)
mock_validate.syntax.assert_called_once_with(config)
mock_validate.platforms.assert_called_once_with(config)
mock_validate.semantic.assert_called_once_with(config)
def test_validate__wrong_schema(self):
config = {
@ -84,85 +82,36 @@ class TaskEngineTestCase(test.TestCase):
engine.TaskEngine, config, task, mock.Mock())
self.assertTrue(task.set_failed.called)
@mock.patch("rally.task.engine.TaskConfig")
def test_validate__wrong_scenarios_name(self, mock_task_config):
task = mock.MagicMock()
eng = engine.TaskEngine(mock.MagicMock(), task, mock.Mock())
eng._validate_config_scenarios_name = mock.MagicMock(
side_effect=exceptions.NotFoundScenarios)
self.assertRaises(exceptions.InvalidTaskException, eng.validate)
self.assertTrue(task.set_failed.called)
@mock.patch("rally.task.engine.TaskConfig")
def test_validate__wrong_syntax(self, mock_task_config):
task = mock.MagicMock()
eng = engine.TaskEngine(mock.MagicMock(), task, mock.Mock())
eng._validate_config_scenarios_name = mock.MagicMock()
eng._validate_config_syntax = mock.MagicMock(
side_effect=exceptions.InvalidTaskConfig)
eng._validate_config_platforms = mock.Mock()
self.assertRaises(exceptions.InvalidTaskException, eng.validate)
self.assertTrue(task.set_failed.called)
# the next validation step should not be processed
self.assertFalse(eng._validate_config_platforms.called)
@mock.patch("rally.task.engine.TaskConfig")
def test_validate__wrong_semantic(self, mock_task_config):
task = mock.MagicMock()
eng = engine.TaskEngine(mock.MagicMock(), task, mock.Mock())
eng._validate_config_scenarios_name = mock.MagicMock()
eng._validate_config_syntax = mock.MagicMock()
eng._validate_config_platforms = mock.MagicMock()
eng._validate_config_semantic = mock.MagicMock(
side_effect=exceptions.InvalidTaskConfig)
self.assertRaises(exceptions.InvalidTaskException, eng.validate)
self.assertTrue(task.set_failed.called)
@mock.patch("rally.task.engine.TaskConfig")
@mock.patch("rally.task.engine.scenario.Scenario.get_all")
def test__validate_config_scenarios_name(
self, mock_scenario_get_all, mock_task_config):
mock_task_instance = mock.MagicMock()
mock_subtask = mock.MagicMock()
mock_subtask.workloads = [
engine.Workload({"name": "a"}, 0),
engine.Workload({"name": "b"}, 1)
]
mock_task_instance.subtasks = [mock_subtask]
mock_scenario_get_all.return_value = [
mock.MagicMock(get_name=lambda: "e"),
mock.MagicMock(get_name=lambda: "b"),
mock.MagicMock(get_name=lambda: "a")
]
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
mock.Mock())
eng._validate_config_scenarios_name(mock_task_instance)
@mock.patch("rally.task.engine.TaskConfig")
@mock.patch("rally.task.engine.scenario.Scenario")
def test__validate_config_scenarios_name_non_exsisting(
self, mock_scenario, mock_task_config):
mock_task_instance = mock.MagicMock()
mock_subtask = mock.MagicMock()
mock_subtask.workloads = [
engine.Workload({"name": "exist"}, 0),
engine.Workload({"name": "nonexist1"}, 1),
engine.Workload({"name": "nonexist2"}, 2)
]
mock_task_instance.subtasks = [mock_subtask]
mock_scenario.get_all.return_value = [
mock.Mock(get_name=lambda: "exist"),
mock.Mock(get_name=lambda: "aaa")]
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
mock.Mock())
exc = self.assertRaises(exceptions.NotFoundScenarios,
eng._validate_config_scenarios_name,
mock_task_instance)
self.assertEqual("There are no benchmark scenarios with names: "
"`nonexist2, nonexist1`.", str(exc))
# all steps of validation are called, which means that the last one is
# failed
self.assertTrue(eng._validate_config_syntax)
self.assertTrue(eng._validate_config_platforms)
self.assertTrue(eng._validate_config_semantic)
@mock.patch("rally.task.engine.scenario.Scenario.get")
@mock.patch("rally.task.sla.SLA.validate")
@ -171,15 +120,15 @@ class TaskEngineTestCase(test.TestCase):
@mock.patch("rally.task.engine.TaskConfig")
@mock.patch("rally.task.engine.runner.ScenarioRunner.validate")
@mock.patch("rally.task.engine.context.Context.validate")
def test__validate_config_syntax(
def test__validate_workload(
self, mock_context_validate,
mock_scenario_runner_validate,
mock_task_config,
mock_hook_validate,
mock_trigger_validate,
mock_sla_validate,
mock_scenario_get
):
mock_scenario_get):
mock_context_validate.return_value = []
mock_sla_validate.return_value = []
mock_hook_validate.return_value = []
@ -188,87 +137,81 @@ class TaskEngineTestCase(test.TestCase):
scenario_cls = mock_scenario_get.return_value
scenario_cls.get_namespace.return_value = "default"
scenario_cls.get_default_context.return_value = default_context
mock_task_instance = mock.MagicMock()
mock_subtask = mock.MagicMock()
scenario_name = "Foo.bar"
runner_type = "MegaRunner"
hook_conf = {"name": "c",
"args": "c_args",
"trigger": {"name": "d", "args": "d_args"}}
mock_subtask.workloads = [
engine.Workload({"name": "sca", "context": {"a": "a_conf"}}, 0),
engine.Workload({"name": "sca", "runner": {"type": "b"},
"sla": {"foo_sla": "sla_conf"}}, 1),
engine.Workload({"name": "sca", "hooks": [hook_conf]}, 2),
]
mock_task_instance.subtasks = [mock_subtask]
workload = engine.Workload({"name": scenario_name,
"runner": {"type": runner_type},
"context": {"a": "a_conf"},
"hooks": [hook_conf],
"sla": {"foo_sla": "sla_conf"}}, 2)
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
mock.Mock())
eng._validate_config_syntax(mock_task_instance)
eng._validate_workload(workload)
mock_scenario_runner_validate.assert_called_once_with(
name="b", credentials=None, config=None,
plugin_cfg={"type": "b"}, namespace="default")
mock_context_validate.assert_has_calls(
[mock.call(name="a",
credentials=None,
config=None,
plugin_cfg="a_conf",
namespace="default"),
mock.call(name="foo",
credentials=None,
config=None,
plugin_cfg="foo_conf",
namespace="default",
allow_hidden=True),
mock.call(name="foo",
credentials=None,
config=None,
plugin_cfg="foo_conf",
namespace="default",
allow_hidden=True),
mock.call(name="foo",
credentials=None,
config=None,
plugin_cfg="foo_conf",
namespace="default",
allow_hidden=True)],
any_order=True
)
name=runner_type, credentials=None, config=None,
plugin_cfg={"type": runner_type}, namespace="default", vtype=None)
self.assertEqual([mock.call(name="a",
credentials=None,
config=None,
plugin_cfg="a_conf",
namespace="default",
vtype=None),
mock.call(name="foo",
credentials=None,
config=None,
plugin_cfg="foo_conf",
namespace="default",
allow_hidden=True,
vtype=None)],
mock_context_validate.call_args_list)
mock_sla_validate.assert_called_once_with(
config=None, credentials=None,
name="foo_sla", plugin_cfg="sla_conf")
name="foo_sla", plugin_cfg="sla_conf", vtype=None)
mock_hook_validate.assert_called_once_with(
config=None, credentials=None, name="c", plugin_cfg="c_args")
config=None, credentials=None, name="c", plugin_cfg="c_args",
vtype=None)
mock_trigger_validate.assert_called_once_with(
config=None, credentials=None, name="d", plugin_cfg="d_args")
config=None, credentials=None, name="d", plugin_cfg="d_args",
vtype=None)
@mock.patch("rally.task.engine.json.dumps")
@mock.patch("rally.task.engine.scenario.Scenario.get")
@mock.patch("rally.task.engine.TaskConfig")
@mock.patch("rally.task.engine.runner.ScenarioRunner.validate")
def test__validate_config_syntax__wrong_runner(
self, mock_scenario_runner_validate,
mock_task_config, mock_scenario_get):
result = validation.ValidationResult(False, "context_error")
def test___validate_workload__wrong_runner(
self, mock_scenario_runner_validate, mock_task_config,
mock_scenario_get, mock_dumps):
mock_dumps.return_value = "<JSON>"
result = validation.ValidationResult(False, "There is no such runner")
mock_scenario_runner_validate.return_value = [result]
scenario_cls = mock_scenario_get.return_value
scenario_cls.get_default_context.return_value = {}
mock_task_instance = mock.MagicMock()
mock_subtask = mock.MagicMock()
mock_subtask.workloads = [
engine.Workload({"name": "sca"}, 0),
engine.Workload({"name": "sca", "runner": {"type": "b"}}, 1)
]
mock_task_instance.subtasks = [mock_subtask]
workload = engine.Workload({"name": "sca", "runner": {"type": "b"}}, 0)
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
mock.Mock())
self.assertRaises(exceptions.InvalidTaskConfig,
eng._validate_config_syntax, mock_task_instance)
e = self.assertRaises(exceptions.InvalidTaskConfig,
eng._validate_workload, workload)
self.assertEqual("Input task is invalid!\n\nSubtask sca[0] has wrong "
"configuration\nSubtask configuration:\n"
"<JSON>\n\nReason(s):\n"
" There is no such runner", e.format_message())
@mock.patch("rally.task.engine.json.dumps")
@mock.patch("rally.task.engine.scenario.Scenario.get")
@mock.patch("rally.task.engine.TaskConfig")
@mock.patch("rally.task.engine.context.Context.validate")
def test__validate_config_syntax__wrong_context(
self, mock_context_validate,
mock_task_config, mock_scenario_get):
self, mock_context_validate, mock_task_config, mock_scenario_get,
mock_dumps):
mock_dumps.return_value = "<JSON>"
result = validation.ValidationResult(False, "context_error")
mock_context_validate.return_value = [result]
scenario_cls = mock_scenario_get.return_value
@ -276,21 +219,27 @@ class TaskEngineTestCase(test.TestCase):
mock_task_instance = mock.MagicMock()
mock_subtask = mock.MagicMock()
mock_subtask.workloads = [
engine.Workload({"name": "sca", "context": {"a": "a_conf"}}, 0),
engine.Workload({"name": "sca"}, 1)
engine.Workload({"name": "sca"}, 0),
engine.Workload({"name": "sca", "context": {"a": "a_conf"}}, 1)
]
mock_task_instance.subtasks = [mock_subtask]
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
mock.Mock())
self.assertRaises(exceptions.InvalidTaskConfig,
eng._validate_config_syntax, mock_task_instance)
e = self.assertRaises(exceptions.InvalidTaskConfig,
eng._validate_config_syntax, mock_task_instance)
self.assertEqual("Input task is invalid!\n\nSubtask sca[1] has wrong "
"configuration\nSubtask configuration:\n<JSON>\n\n"
"Reason(s):\n context_error", e.format_message())
@mock.patch("rally.task.engine.json.dumps")
@mock.patch("rally.task.engine.scenario.Scenario.get")
@mock.patch("rally.task.sla.SLA.validate")
@mock.patch("rally.task.engine.TaskConfig")
def test__validate_config_syntax__wrong_sla(
self, mock_task_config, mock_sla_validate, mock_scenario_get):
self, mock_task_config, mock_sla_validate, mock_scenario_get,
mock_dumps):
mock_dumps.return_value = "<JSON>"
result = validation.ValidationResult(False, "sla_error")
mock_sla_validate.return_value = [result]
scenario_cls = mock_scenario_get.return_value
@ -304,14 +253,23 @@ class TaskEngineTestCase(test.TestCase):
mock_task_instance.subtasks = [mock_subtask]
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
mock.Mock())
self.assertRaises(exceptions.InvalidTaskConfig,
eng._validate_config_syntax, mock_task_instance)
e = self.assertRaises(exceptions.InvalidTaskConfig,
eng._validate_config_syntax, mock_task_instance)
self.assertEqual("Input task is invalid!\n\n"
"Subtask sca[1] has wrong configuration\n"
"Subtask configuration:\n<JSON>\n\n"
"Reason(s):\n sla_error", e.format_message())
@mock.patch("rally.task.engine.json.dumps")
@mock.patch("rally.task.engine.scenario.Scenario.get")
@mock.patch("rally.task.hook.Hook.validate")
@mock.patch("rally.task.trigger.Trigger.validate")
@mock.patch("rally.task.engine.TaskConfig")
def test__validate_config_syntax__wrong_hook(
self, mock_task_config, mock_hook_validate, mock_scenario_get):
self, mock_task_config, mock_trigger_validate, mock_hook_validate,
mock_scenario_get, mock_dumps):
mock_dumps.return_value = "<JSON>"
mock_trigger_validate.return_value = []
result = validation.ValidationResult(False, "hook_error")
mock_hook_validate.return_value = [result]
scenario_cls = mock_scenario_get.return_value
@ -329,16 +287,23 @@ class TaskEngineTestCase(test.TestCase):
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
mock.Mock())
self.assertRaises(exceptions.InvalidTaskConfig,
eng._validate_config_syntax, mock_task_instance)
e = self.assertRaises(exceptions.InvalidTaskConfig,
eng._validate_config_syntax, mock_task_instance)
self.assertEqual("Input task is invalid!\n\n"
"Subtask sca[1] has wrong configuration\n"
"Subtask configuration:\n<JSON>\n\n"
"Reason(s):\n hook_error", e.format_message())
@mock.patch("rally.task.engine.json.dumps")
@mock.patch("rally.task.engine.scenario.Scenario.get")
@mock.patch("rally.task.trigger.Trigger.validate")
@mock.patch("rally.task.hook.Hook.validate")
@mock.patch("rally.task.engine.TaskConfig")
def test__validate_config_syntax__wrong_trigger(
self, mock_task_config, mock_hook_validate, mock_trigger_validate,
mock_scenario_get):
mock_scenario_get, mock_dumps):
mock_dumps.return_value = "<JSON>"
result = validation.ValidationResult(False, "trigger_error")
mock_trigger_validate.return_value = [result]
mock_hook_validate.return_value = []
@ -357,42 +322,32 @@ class TaskEngineTestCase(test.TestCase):
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
mock.Mock())
self.assertRaises(exceptions.InvalidTaskConfig,
eng._validate_config_syntax, mock_task_instance)
e = self.assertRaises(exceptions.InvalidTaskConfig,
eng._validate_config_syntax, mock_task_instance)
self.assertEqual("Input task is invalid!\n\n"
"Subtask sca[1] has wrong configuration\n"
"Subtask configuration:\n<JSON>\n\n"
"Reason(s):\n trigger_error", e.format_message())
@mock.patch("rally.task.engine.scenario.Scenario.validate")
@mock.patch("rally.task.engine.TaskConfig")
def test__validate_config_semantic_helper(self, mock_task_config,
mock_scenario_validate):
mock_scenario_validate.return_value = []
def test__validate_config_semantic_helper(self, mock_task_config):
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
mock.Mock())
eng._validate_workload = mock.Mock()
workloads = [engine.Workload(
{"name": "name", "runner": "runner", "args": "args"}, 0)]
users = [{"foo": "user1"}]
user_context = mock.MagicMock()
user_context.__enter__.return_value.context = {
"users": [{"foo": "user1"}]}
user_context.__enter__.return_value.context = {"users": users}
eng._validate_config_semantic_helper(
"admin", user_context, workloads, "foo")
mock_scenario_validate.assert_called_once_with(
name="name", config={"runner": "runner", "args": "args"},
credentials={"foo": {"admin": "admin",
"users": [{"foo": "user1"}]}},
plugin_cfg=None)
@mock.patch("rally.task.engine.scenario.Scenario.validate")
@mock.patch("rally.task.engine.TaskConfig")
def test__validate_config_semanitc_helper_invalid_arg(
self, mock_task_config, mock_scenario_validate):
eng = engine.TaskEngine(mock.MagicMock(), mock.MagicMock(),
mock.Mock())
mock_scenario_validate.return_value = [
validation.ValidationResult(False, msg="foo")]
user_context = mock.MagicMock()
workloads = [engine.Workload({"name": "name"}, 0)]
self.assertRaises(exceptions.InvalidTaskConfig,
eng._validate_config_semantic_helper, "a",
user_context, workloads, "foo")
eng._validate_workload.assert_called_once_with(
workloads[0], credentials={"foo": {"admin": "admin",
"users": users}},
vtype="semantic")
@mock.patch("rally.task.engine.scenario.Scenario.get")
@mock.patch("rally.task.engine.context.Context")
@ -438,6 +393,36 @@ class TaskEngineTestCase(test.TestCase):
mock.call(admin, user_context, [wconf2, wconf3], "openstack"),
], any_order=True)
@mock.patch("rally.task.engine.TaskConfig")
@mock.patch("rally.task.engine.TaskEngine._validate_workload")
def test__validate_config_platforms(
self, mock__validate_workload, mock_task_config):
class FakeDeployment(object):
def __init__(self, credentials):
self._creds = credentials
self.get_all_credentials = mock.Mock()
self.get_all_credentials.return_value = self._creds
foo_cred1 = {"admin": "admin", "users": ["user1"]}
foo_cred2 = {"admin": "admin", "users": ["user1"]}
deployment = FakeDeployment({"foo": [foo_cred1, foo_cred2]})
workload1 = "workload1"
workload2 = "workload2"
subtasks = [mock.Mock(workloads=[workload1]),
mock.Mock(workloads=[workload2])]
config = mock.Mock(subtasks=subtasks)
eng = engine.TaskEngine({}, mock.MagicMock(), deployment)
eng._validate_config_platforms(config)
self.assertEqual(
[mock.call(w, vtype="platform", credentials={"foo": foo_cred1})
for w in (workload1, workload2)],
mock__validate_workload.call_args_list)
deployment.get_all_credentials.assert_called_once_with()
@mock.patch("rally.common.objects.Task.get_status")
@mock.patch("rally.task.engine.TaskConfig")
@mock.patch("rally.task.engine.ResultConsumer")
@ -1147,13 +1132,8 @@ class WorkloadTestCase(test.TestCase):
"name": "n",
"pos": 0,
"reason": "r",
"config": {
"runner": "r",
"context": "c",
"sla": "s",
"hooks": "h",
"args": "a"
}
"config": json.dumps({"runner": "r", "context": "c", "sla": "s",
"hooks": "h", "args": "a"})
}
self.assertEqual(expected_args,