From f0bacce53909a904c92ea77e812fdd20e29977b3 Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Thu, 2 Feb 2017 17:49:27 +0000 Subject: [PATCH 1/2] Adjust runner to track test suite failures Change the reporter used by gabbi-run so that it provides a more complete test name in its output, using the id() of the test. This will include the name of the yaml file. Also track failures per input file and provide a summary of which files contained failures (if any) at the end. Both of these changes were done in the simplest way possible so there is probably considerably room for improvement now that the concept has been proven. Fixes #181 --- gabbi/reporter.py | 3 ++- gabbi/runner.py | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/gabbi/reporter.py b/gabbi/reporter.py index f7dcf6a..0efef2e 100644 --- a/gabbi/reporter.py +++ b/gabbi/reporter.py @@ -80,7 +80,8 @@ class ConciseTestResult(TextTestResult): self.stream.writeln('\t[unexpected success]') def getDescription(self, test): - name = test.test_data['name'] + # Chop the test method (test_request) off the test.id(). + name = test.id().rsplit('.', 1)[0] desc = test.test_data.get('desc', None) return ': '.join((name, desc)) if desc else name diff --git a/gabbi/runner.py b/gabbi/runner.py index febb23f..ea232be 100644 --- a/gabbi/runner.py +++ b/gabbi/runner.py @@ -12,6 +12,8 @@ # under the License. """Implementation of a command-line runner for gabbi files (AKA suites).""" +from __future__ import print_function + import argparse from importlib import import_module import os @@ -81,6 +83,8 @@ def run(): verbosity = args.verbosity failfast = args.failfast failure = False + # Keep track of file names that have failures. + failures = [] if not input_files: success = run_suite(sys.stdin, handler_objects, host, port, @@ -89,22 +93,28 @@ def run(): failure = not success else: for input_file in input_files: + name = os.path.splitext(os.path.basename(input_file))[0] with open(input_file, 'r') as fh: data_dir = os.path.dirname(input_file) success = run_suite(fh, handler_objects, host, port, prefix, force_ssl, failfast, data_dir=data_dir, - verbosity=verbosity) + verbosity=verbosity, name=name) + if not success: + failures.append(input_file) if not failure: # once failed, this is considered immutable failure = not success if failure and failfast: break + if failures: + print("There were failures in the following files:", file=sys.stderr) + print('\n'.join(failures), file=sys.stderr) sys.exit(failure) def run_suite(handle, handler_objects, host, port, prefix, force_ssl=False, - failfast=False, data_dir='.', verbosity=False): + failfast=False, data_dir='.', verbosity=False, name='input'): """Run the tests from the YAML in handle.""" data = utils.load_yaml(handle) if force_ssl: @@ -120,8 +130,8 @@ def run_suite(handle, handler_objects, host, port, prefix, force_ssl=False, loader = unittest.defaultTestLoader test_suite = suitemaker.test_suite_from_dict( - loader, 'input', data, data_dir, host, port, None, None, prefix=prefix, - handlers=handler_objects) + loader, name, data, data_dir, host, port, None, None, prefix=prefix, + handlers=handler_objects, test_loader_name='gabbi-runner') result = ConciseTestRunner( verbosity=2, failfast=failfast).run(test_suite) From cade01d2991080944fdcd10e9c486ed4c0a1cab2 Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Mon, 6 Feb 2017 12:52:16 +0000 Subject: [PATCH 2/2] Minor comment clarification Use quote marks to indicate literal string. --- gabbi/reporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gabbi/reporter.py b/gabbi/reporter.py index 0efef2e..c9c88ab 100644 --- a/gabbi/reporter.py +++ b/gabbi/reporter.py @@ -80,7 +80,7 @@ class ConciseTestResult(TextTestResult): self.stream.writeln('\t[unexpected success]') def getDescription(self, test): - # Chop the test method (test_request) off the test.id(). + # Chop the test method ('test_request') off the test.id(). name = test.id().rsplit('.', 1)[0] desc = test.test_data.get('desc', None) return ': '.join((name, desc)) if desc else name