x
/
opencafe
Code Issues Proposed changes

Change to DataDrivenClass generator to fail when DSL does not generate data. 

If a DataSetList is passed into a test that does not generate or
contain data the current behavior is that the test suites passes.
This proposal is to change that behavior. If a DSL cannot generate
data the test suite as a whole should still run but it should
fail. Otherwise teams using tooling to track and manage test results
will have false positives with mysteriously missing tests.

Added Metatests to check basic functionality.
Added Engine config section for unittest driver.

Change-Id: I3430e9573c3e99b80f25e7577709144d49c8eae2
This commit is contained in:
Anna Eilering 2016-03-31 14:30:53 -05:00
parent d14fc7dd0d
commit 799656da85
5 changed files with 169 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
cafe/drivers/unittest/config.py Normal file
View File

@ -0,0 +1,46 @@
# Copyright 2016 Rackspace
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from cafe.engine.models.data_interfaces import (
ConfigSectionInterface, _get_path_from_env)
class DriverConfig(ConfigSectionInterface):
"""
Unittest driver configuration values.
This config section is intended to supply values and configuration that can
not be programatically identified to the unittest driver.
"""
SECTION_NAME = 'drivers.unittest'
def __init__(self, config_file_path=None):
config_file_path = config_file_path or _get_path_from_env(
'CAFE_ENGINE_CONFIG_FILE_PATH')
super(DriverConfig, self).__init__(config_file_path=config_file_path)
@property
def ignore_empty_datasets(self):
"""
Identify whether empty datasets should change suite results.
A dataset provided to a suite should result in the suite failing. This
value provides a mechanism to modify that behavior in the case of
suites with intensionally included empty datasets. If this is set to
'True' empty datasets will not cause suite failures. This defaults
to 'False'.
"""
return self.get_boolean(
item_name="ignore_empty_datasets",
default=False)

82
cafe/drivers/unittest/decorators.py
View File

@ -18,6 +18,9 @@ import re
from cafe.common.reporting import cclogging
from cafe.drivers.unittest.datasets import DatasetList
from cafe.drivers.unittest.fixtures import BaseTestFixture
from cafe.drivers.unittest.config import DriverConfig
DATA_DRIVEN_TEST_ATTR = "__data_driven_test_data__"
DATA_DRIVEN_TEST_PREFIX = "ddtest_"
@ -85,8 +88,55 @@ def data_driven_test(*dataset_sources, **kwargs):
return decorator
class EmptyDSLError(Exception):
"""Custom exception to allow errors in Datadriven classes with no data."""
def __init__(self, dsl_namespace, original_test_list):
general_message = (
"The Dataset list used to generate this Data Driven Class was "
"empty. No Fixtures or Tests were generated. Review the Dataset "
"used to run this test.")
DSL_information = "Dataset List location: {dsl_namespace}".format(
dsl_namespace=dsl_namespace)
pretty_test_list_header = (
"The following {n} tests were not run".format(
n=len(original_test_list)))
pretty_test_list = "\t" + "\n\t".join(original_test_list)
self.message = (
"{general_message}\n{DSL_information}\n\n"
"{pretty_test_list_header}\n{pretty_test_list}").format(
general_message=general_message,
DSL_information=DSL_information,
pretty_test_list_header=pretty_test_list_header,
pretty_test_list=pretty_test_list)
super(EmptyDSLError, self).__init__(self.message)
class _FauxDSLFixture(BaseTestFixture):
"""Faux Test Fixture and Test class to inject into DDC that lack data."""
dsl_namespace = None
original_test_list = []
# This is so we don't have to call super as there will never be anything
# here.
_class_cleanup_tasks = []
@classmethod
def setUpClass(cls):
"""setUpClass to force a fixture error for DDCs which lack data."""
raise EmptyDSLError(
dsl_namespace=cls.dsl_namespace,
original_test_list=cls.original_test_list)
# A test method is required in order to allow this to be injected without
# making changes to Unittest itself.
def test_data_failed_to_generate(self):
"""Faux test method to allow injection."""
pass
def DataDrivenClass(*dataset_lists):
"""Use data driven class decorator. designed to be used on a fixture"""
"""Use data driven class decorator. designed to be used on a fixture."""
def decorator(cls):
"""Creates classes with variables named after datasets.
Names of classes are equal to (class_name with out fixture) + ds_name
@ -96,7 +146,35 @@ def DataDrivenClass(*dataset_lists):
class_name = re.sub("fixture", "", cls.__name__, flags=re.IGNORECASE)
if not re.match(".*fixture", cls.__name__, flags=re.IGNORECASE):
cls.__name__ = "{0}Fixture".format(cls.__name__)
for dataset_list in dataset_lists:
unittest_driver_config = DriverConfig()
for i, dataset_list in enumerate(dataset_lists):
if (not dataset_list and
not unittest_driver_config.ignore_empty_datasets):
# The DSL did not generate anything
class_name_new = "{class_name}_{exception}_{index}".format(
class_name=class_name,
exception="DSL_EXCEPTION",
index=i)
# We are creating a new, special class here that willd allow us
# to force an error during test set up that contains
# information useful for triaging the DSL failure.
# Additionally this should surface any tests that did not run
# due to the DSL issue.
new_cls = DataDrivenFixture(_FauxDSLFixture)
new_class = type(
class_name_new,
(new_cls,),
{})
dsl_namespace = cclogging.get_object_namespace(
dataset_list.__class__)
test_ls = [test for test in dir(cls) if test.startswith(
'test_') or test.startswith(DATA_DRIVEN_TEST_PREFIX)]
new_class.dsl_namespace = dsl_namespace
new_class.original_test_list = test_ls
new_class.__module__ = cls.__module__
setattr(module, class_name_new, new_class)
for dataset in dataset_list:
class_name_new = "{0}_{1}".format(class_name, dataset.name)
new_class = type(class_name_new, (cls,), dataset.data)

0
tests/drivers/__init__.py Normal file
View File

0
tests/drivers/unittest/__init__.py Normal file
View File

43
tests/drivers/unittest/test_data_driven_class_suite_generation.py Normal file
View File

@ -0,0 +1,43 @@
# Copyright 2016 Rackspace
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
from cafe.drivers.unittest import decorators
class DSLSuiteBuilderTests(unittest.TestCase):
"""Metatests for the DSL Suite Builder."""
def test_FauxDSLFixture_raises_Exception(self):
"""Check that the _FauxDSLFixture raises an exception as expected."""
faux_fixture = type(
'_FauxDSLFixture',
(object,),
dict(decorators._FauxDSLFixture.__dict__))
# Check that the fixture raises an exception
with self.assertRaises(decorators.EmptyDSLError) as e:
faux_fixture().setUpClass()
# If it does, let's make sure the exception generates the correct
# message.
msg = (
"The Dataset list used to generate this Data Driven Class was "
"empty. No Fixtures or Tests were generated. Review the Dataset "
"used to run this test.\n"
"Dataset List location: None\n\n"
"The following 0 tests were not run\n\t")
self.assertEquals(msg, e.exception.message)