From f95f1d444841f7d1f414176e83064ec23a4692f4 Mon Sep 17 00:00:00 2001 From: melanie witt Date: Thu, 26 May 2016 20:31:30 +0000 Subject: [PATCH] Sort failed tests at the top on HTML result page Currently tests are sorted alphabetically by test class name regardless of the pass/fail status. When troubleshooting failed unit tests in jenkins jobs with many unit tests (> 11242 in nova), one must scroll potentially a lot or wait as the page loads to search for the word 'fail' to get to the details of the failed test. This sorts failed (and error) tests at the top of the HTML page for easier troubleshooting (sorted failed tests + sorted passed tests). Change-Id: I0b575e77c4a3e1cc73e60ea48ca5b9bf2e84d76f --- os_testr/subunit2html.py | 10 +++++-- os_testr/tests/test_subunit2html.py | 45 +++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/os_testr/subunit2html.py b/os_testr/subunit2html.py index 096a91b..a710b9f 100755 --- a/os_testr/subunit2html.py +++ b/os_testr/subunit2html.py @@ -617,8 +617,12 @@ class HtmlOutput(testtools.TestResult): # unittest does not seems to run in any particular order. # Here at least we want to group them together by class. rmap = {} - classes = [] + # Differentiate between classes that have test failures so we can sort + # them at the top of the html page for easier troubleshooting + failclasses = [] + passclasses = [] for n, t, o, e in result_list: + classes = failclasses if n == 1 or n == 2 else passclasses if hasattr(t, '_tests'): for inner_test in t._tests: self._add_cls(rmap, classes, inner_test, @@ -626,7 +630,9 @@ class HtmlOutput(testtools.TestResult): else: self._add_cls(rmap, classes, t, (n, t, o, e)) classort = lambda s: str(s) - sortedclasses = sorted(classes, key=classort) + sortedfailclasses = sorted(failclasses, key=classort) + sortedpassclasses = sorted(passclasses, key=classort) + sortedclasses = sortedfailclasses + sortedpassclasses r = [(cls, rmap[str(cls)]) for cls in sortedclasses] return r diff --git a/os_testr/tests/test_subunit2html.py b/os_testr/tests/test_subunit2html.py index f3196a5..994eed6 100644 --- a/os_testr/tests/test_subunit2html.py +++ b/os_testr/tests/test_subunit2html.py @@ -10,6 +10,8 @@ # License for the specific language governing permissions and limitations # under the License. +import sys + from ddt import data from ddt import ddt from subunit import RemotedTestCase @@ -29,3 +31,46 @@ class TestSubunit2html(base.TestCase): cls_ = [] obj_._add_cls({}, cls_, test_, ()) self.assertEqual("example.path.to.test", cls_[0].name) + + @data(RemotedTestCase, PlaceHolder) + def test_result_sorting(self, test_cls): + tests = [] + for i in range(9): + tests.append(test_cls('example.path.to.test%d.method' % i)) + # addFailure, addError, and addSkip need the real exc_info + try: + raise Exception('fake') + except Exception: + err = sys.exc_info() + obj = subunit2html.HtmlOutput() + obj.addSuccess(tests[3]) + obj.addSuccess(tests[1]) + # example.path.to.test2 has a failure + obj.addFailure(tests[2], err) + obj.addSkip(tests[0], err) + obj.addSuccess(tests[8]) + # example.path.to.test5 has a failure (error) + obj.addError(tests[5], err) + # example.path.to.test4 has a failure + obj.addFailure(tests[4], err) + obj.addSuccess(tests[7]) + # example.path.to.test6 has a failure + obj.addFailure(tests[6], err) + sorted_result = obj._sortResult(obj.result) + # _sortResult returns a list of results of format: + # [(class, [test_result_tuple, ...]), ...] + # sorted by str(class) + # + # Classes with failures (2, 4, 5, and 6) should be sorted separately + # at the top. The rest of the classes should be in sorted order after. + expected_class_order = ['example.path.to.test2', + 'example.path.to.test4', + 'example.path.to.test5', + 'example.path.to.test6', + 'example.path.to.test0', + 'example.path.to.test1', + 'example.path.to.test3', + 'example.path.to.test7', + 'example.path.to.test8'] + for i, r in enumerate(sorted_result): + self.assertEqual(expected_class_order[i], str(r[0]))