diff --git a/cafe/drivers/unittest/config.py b/cafe/drivers/unittest/config.py new file mode 100644 index 0000000..9e511e5 --- /dev/null +++ b/cafe/drivers/unittest/config.py @@ -0,0 +1,46 @@ +# Copyright 2016 Rackspace +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from cafe.engine.models.data_interfaces import ( + ConfigSectionInterface, _get_path_from_env) + + +class DriverConfig(ConfigSectionInterface): + """ + Unittest driver configuration values. + + This config section is intended to supply values and configuration that can + not be programatically identified to the unittest driver. + """ + + SECTION_NAME = 'drivers.unittest' + + def __init__(self, config_file_path=None): + config_file_path = config_file_path or _get_path_from_env( + 'CAFE_ENGINE_CONFIG_FILE_PATH') + super(DriverConfig, self).__init__(config_file_path=config_file_path) + + @property + def ignore_empty_datasets(self): + """ + Identify whether empty datasets should change suite results. + + A dataset provided to a suite should result in the suite failing. This + value provides a mechanism to modify that behavior in the case of + suites with intensionally included empty datasets. If this is set to + 'True' empty datasets will not cause suite failures. This defaults + to 'False'. + """ + return self.get_boolean( + item_name="ignore_empty_datasets", + default=False) diff --git a/cafe/drivers/unittest/decorators.py b/cafe/drivers/unittest/decorators.py index 520e9e3..5c6d020 100644 --- a/cafe/drivers/unittest/decorators.py +++ b/cafe/drivers/unittest/decorators.py @@ -18,6 +18,9 @@ import re from cafe.common.reporting import cclogging from cafe.drivers.unittest.datasets import DatasetList +from cafe.drivers.unittest.fixtures import BaseTestFixture +from cafe.drivers.unittest.config import DriverConfig + DATA_DRIVEN_TEST_ATTR = "__data_driven_test_data__" DATA_DRIVEN_TEST_PREFIX = "ddtest_" @@ -85,8 +88,55 @@ def data_driven_test(*dataset_sources, **kwargs): return decorator +class EmptyDSLError(Exception): + """Custom exception to allow errors in Datadriven classes with no data.""" + def __init__(self, dsl_namespace, original_test_list): + general_message = ( + "The Dataset list used to generate this Data Driven Class was " + "empty. No Fixtures or Tests were generated. Review the Dataset " + "used to run this test.") + DSL_information = "Dataset List location: {dsl_namespace}".format( + dsl_namespace=dsl_namespace) + pretty_test_list_header = ( + "The following {n} tests were not run".format( + n=len(original_test_list))) + pretty_test_list = "\t" + "\n\t".join(original_test_list) + self.message = ( + "{general_message}\n{DSL_information}\n\n" + "{pretty_test_list_header}\n{pretty_test_list}").format( + general_message=general_message, + DSL_information=DSL_information, + pretty_test_list_header=pretty_test_list_header, + pretty_test_list=pretty_test_list) + super(EmptyDSLError, self).__init__(self.message) + + +class _FauxDSLFixture(BaseTestFixture): + """Faux Test Fixture and Test class to inject into DDC that lack data.""" + + dsl_namespace = None + original_test_list = [] + + # This is so we don't have to call super as there will never be anything + # here. + _class_cleanup_tasks = [] + + @classmethod + def setUpClass(cls): + """setUpClass to force a fixture error for DDCs which lack data.""" + raise EmptyDSLError( + dsl_namespace=cls.dsl_namespace, + original_test_list=cls.original_test_list) + + # A test method is required in order to allow this to be injected without + # making changes to Unittest itself. + def test_data_failed_to_generate(self): + """Faux test method to allow injection.""" + pass + + def DataDrivenClass(*dataset_lists): - """Use data driven class decorator. designed to be used on a fixture""" + """Use data driven class decorator. designed to be used on a fixture.""" def decorator(cls): """Creates classes with variables named after datasets. Names of classes are equal to (class_name with out fixture) + ds_name @@ -96,7 +146,35 @@ def DataDrivenClass(*dataset_lists): class_name = re.sub("fixture", "", cls.__name__, flags=re.IGNORECASE) if not re.match(".*fixture", cls.__name__, flags=re.IGNORECASE): cls.__name__ = "{0}Fixture".format(cls.__name__) - for dataset_list in dataset_lists: + + unittest_driver_config = DriverConfig() + + for i, dataset_list in enumerate(dataset_lists): + if (not dataset_list and + not unittest_driver_config.ignore_empty_datasets): + # The DSL did not generate anything + class_name_new = "{class_name}_{exception}_{index}".format( + class_name=class_name, + exception="DSL_EXCEPTION", + index=i) + # We are creating a new, special class here that willd allow us + # to force an error during test set up that contains + # information useful for triaging the DSL failure. + # Additionally this should surface any tests that did not run + # due to the DSL issue. + new_cls = DataDrivenFixture(_FauxDSLFixture) + new_class = type( + class_name_new, + (new_cls,), + {}) + dsl_namespace = cclogging.get_object_namespace( + dataset_list.__class__) + test_ls = [test for test in dir(cls) if test.startswith( + 'test_') or test.startswith(DATA_DRIVEN_TEST_PREFIX)] + new_class.dsl_namespace = dsl_namespace + new_class.original_test_list = test_ls + new_class.__module__ = cls.__module__ + setattr(module, class_name_new, new_class) for dataset in dataset_list: class_name_new = "{0}_{1}".format(class_name, dataset.name) new_class = type(class_name_new, (cls,), dataset.data) diff --git a/tests/drivers/__init__.py b/tests/drivers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/drivers/unittest/__init__.py b/tests/drivers/unittest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/drivers/unittest/test_data_driven_class_suite_generation.py b/tests/drivers/unittest/test_data_driven_class_suite_generation.py new file mode 100644 index 0000000..cb9905c --- /dev/null +++ b/tests/drivers/unittest/test_data_driven_class_suite_generation.py @@ -0,0 +1,43 @@ +# Copyright 2016 Rackspace +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest + +from cafe.drivers.unittest import decorators + + +class DSLSuiteBuilderTests(unittest.TestCase): + """Metatests for the DSL Suite Builder.""" + + def test_FauxDSLFixture_raises_Exception(self): + """Check that the _FauxDSLFixture raises an exception as expected.""" + + faux_fixture = type( + '_FauxDSLFixture', + (object,), + dict(decorators._FauxDSLFixture.__dict__)) + + # Check that the fixture raises an exception + with self.assertRaises(decorators.EmptyDSLError) as e: + faux_fixture().setUpClass() + + # If it does, let's make sure the exception generates the correct + # message. + msg = ( + "The Dataset list used to generate this Data Driven Class was " + "empty. No Fixtures or Tests were generated. Review the Dataset " + "used to run this test.\n" + "Dataset List location: None\n\n" + "The following 0 tests were not run\n\t") + + self.assertEquals(msg, e.exception.message)