Change to DataDrivenClass generator to fail when DSL does not generate data.
If a DataSetList is passed into a test that does not generate or contain data the current behavior is that the test suites passes. This proposal is to change that behavior. If a DSL cannot generate data the test suite as a whole should still run but it should fail. Otherwise teams using tooling to track and manage test results will have false positives with mysteriously missing tests. Added Metatests to check basic functionality. Added Engine config section for unittest driver. Change-Id: I3430e9573c3e99b80f25e7577709144d49c8eae2
This commit is contained in:
parent
d14fc7dd0d
commit
799656da85
|
@ -0,0 +1,46 @@
|
|||
# Copyright 2016 Rackspace
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from cafe.engine.models.data_interfaces import (
|
||||
ConfigSectionInterface, _get_path_from_env)
|
||||
|
||||
|
||||
class DriverConfig(ConfigSectionInterface):
|
||||
"""
|
||||
Unittest driver configuration values.
|
||||
|
||||
This config section is intended to supply values and configuration that can
|
||||
not be programatically identified to the unittest driver.
|
||||
"""
|
||||
|
||||
SECTION_NAME = 'drivers.unittest'
|
||||
|
||||
def __init__(self, config_file_path=None):
|
||||
config_file_path = config_file_path or _get_path_from_env(
|
||||
'CAFE_ENGINE_CONFIG_FILE_PATH')
|
||||
super(DriverConfig, self).__init__(config_file_path=config_file_path)
|
||||
|
||||
@property
|
||||
def ignore_empty_datasets(self):
|
||||
"""
|
||||
Identify whether empty datasets should change suite results.
|
||||
|
||||
A dataset provided to a suite should result in the suite failing. This
|
||||
value provides a mechanism to modify that behavior in the case of
|
||||
suites with intensionally included empty datasets. If this is set to
|
||||
'True' empty datasets will not cause suite failures. This defaults
|
||||
to 'False'.
|
||||
"""
|
||||
return self.get_boolean(
|
||||
item_name="ignore_empty_datasets",
|
||||
default=False)
|
|
@ -18,6 +18,9 @@ import re
|
|||
|
||||
from cafe.common.reporting import cclogging
|
||||
from cafe.drivers.unittest.datasets import DatasetList
|
||||
from cafe.drivers.unittest.fixtures import BaseTestFixture
|
||||
from cafe.drivers.unittest.config import DriverConfig
|
||||
|
||||
|
||||
DATA_DRIVEN_TEST_ATTR = "__data_driven_test_data__"
|
||||
DATA_DRIVEN_TEST_PREFIX = "ddtest_"
|
||||
|
@ -85,8 +88,55 @@ def data_driven_test(*dataset_sources, **kwargs):
|
|||
return decorator
|
||||
|
||||
|
||||
class EmptyDSLError(Exception):
|
||||
"""Custom exception to allow errors in Datadriven classes with no data."""
|
||||
def __init__(self, dsl_namespace, original_test_list):
|
||||
general_message = (
|
||||
"The Dataset list used to generate this Data Driven Class was "
|
||||
"empty. No Fixtures or Tests were generated. Review the Dataset "
|
||||
"used to run this test.")
|
||||
DSL_information = "Dataset List location: {dsl_namespace}".format(
|
||||
dsl_namespace=dsl_namespace)
|
||||
pretty_test_list_header = (
|
||||
"The following {n} tests were not run".format(
|
||||
n=len(original_test_list)))
|
||||
pretty_test_list = "\t" + "\n\t".join(original_test_list)
|
||||
self.message = (
|
||||
"{general_message}\n{DSL_information}\n\n"
|
||||
"{pretty_test_list_header}\n{pretty_test_list}").format(
|
||||
general_message=general_message,
|
||||
DSL_information=DSL_information,
|
||||
pretty_test_list_header=pretty_test_list_header,
|
||||
pretty_test_list=pretty_test_list)
|
||||
super(EmptyDSLError, self).__init__(self.message)
|
||||
|
||||
|
||||
class _FauxDSLFixture(BaseTestFixture):
|
||||
"""Faux Test Fixture and Test class to inject into DDC that lack data."""
|
||||
|
||||
dsl_namespace = None
|
||||
original_test_list = []
|
||||
|
||||
# This is so we don't have to call super as there will never be anything
|
||||
# here.
|
||||
_class_cleanup_tasks = []
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""setUpClass to force a fixture error for DDCs which lack data."""
|
||||
raise EmptyDSLError(
|
||||
dsl_namespace=cls.dsl_namespace,
|
||||
original_test_list=cls.original_test_list)
|
||||
|
||||
# A test method is required in order to allow this to be injected without
|
||||
# making changes to Unittest itself.
|
||||
def test_data_failed_to_generate(self):
|
||||
"""Faux test method to allow injection."""
|
||||
pass
|
||||
|
||||
|
||||
def DataDrivenClass(*dataset_lists):
|
||||
"""Use data driven class decorator. designed to be used on a fixture"""
|
||||
"""Use data driven class decorator. designed to be used on a fixture."""
|
||||
def decorator(cls):
|
||||
"""Creates classes with variables named after datasets.
|
||||
Names of classes are equal to (class_name with out fixture) + ds_name
|
||||
|
@ -96,7 +146,35 @@ def DataDrivenClass(*dataset_lists):
|
|||
class_name = re.sub("fixture", "", cls.__name__, flags=re.IGNORECASE)
|
||||
if not re.match(".*fixture", cls.__name__, flags=re.IGNORECASE):
|
||||
cls.__name__ = "{0}Fixture".format(cls.__name__)
|
||||
for dataset_list in dataset_lists:
|
||||
|
||||
unittest_driver_config = DriverConfig()
|
||||
|
||||
for i, dataset_list in enumerate(dataset_lists):
|
||||
if (not dataset_list and
|
||||
not unittest_driver_config.ignore_empty_datasets):
|
||||
# The DSL did not generate anything
|
||||
class_name_new = "{class_name}_{exception}_{index}".format(
|
||||
class_name=class_name,
|
||||
exception="DSL_EXCEPTION",
|
||||
index=i)
|
||||
# We are creating a new, special class here that willd allow us
|
||||
# to force an error during test set up that contains
|
||||
# information useful for triaging the DSL failure.
|
||||
# Additionally this should surface any tests that did not run
|
||||
# due to the DSL issue.
|
||||
new_cls = DataDrivenFixture(_FauxDSLFixture)
|
||||
new_class = type(
|
||||
class_name_new,
|
||||
(new_cls,),
|
||||
{})
|
||||
dsl_namespace = cclogging.get_object_namespace(
|
||||
dataset_list.__class__)
|
||||
test_ls = [test for test in dir(cls) if test.startswith(
|
||||
'test_') or test.startswith(DATA_DRIVEN_TEST_PREFIX)]
|
||||
new_class.dsl_namespace = dsl_namespace
|
||||
new_class.original_test_list = test_ls
|
||||
new_class.__module__ = cls.__module__
|
||||
setattr(module, class_name_new, new_class)
|
||||
for dataset in dataset_list:
|
||||
class_name_new = "{0}_{1}".format(class_name, dataset.name)
|
||||
new_class = type(class_name_new, (cls,), dataset.data)
|
||||
|
|
|
@ -0,0 +1,43 @@
|
|||
# Copyright 2016 Rackspace
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import unittest
|
||||
|
||||
from cafe.drivers.unittest import decorators
|
||||
|
||||
|
||||
class DSLSuiteBuilderTests(unittest.TestCase):
|
||||
"""Metatests for the DSL Suite Builder."""
|
||||
|
||||
def test_FauxDSLFixture_raises_Exception(self):
|
||||
"""Check that the _FauxDSLFixture raises an exception as expected."""
|
||||
|
||||
faux_fixture = type(
|
||||
'_FauxDSLFixture',
|
||||
(object,),
|
||||
dict(decorators._FauxDSLFixture.__dict__))
|
||||
|
||||
# Check that the fixture raises an exception
|
||||
with self.assertRaises(decorators.EmptyDSLError) as e:
|
||||
faux_fixture().setUpClass()
|
||||
|
||||
# If it does, let's make sure the exception generates the correct
|
||||
# message.
|
||||
msg = (
|
||||
"The Dataset list used to generate this Data Driven Class was "
|
||||
"empty. No Fixtures or Tests were generated. Review the Dataset "
|
||||
"used to run this test.\n"
|
||||
"Dataset List location: None\n\n"
|
||||
"The following 0 tests were not run\n\t")
|
||||
|
||||
self.assertEquals(msg, e.exception.message)
|
Loading…
Reference in New Issue