support TRACE_FAILONLY env variable

In order to get similar behavior to the way testr out of the box
works, only displaying failing results, all the user to define
TRACE_FAILONLY environment variable (any value will do). After this
point only failures, and things leaking out of stderr will end up on
the screen.

We also need to remove the suppression of the inline fail dumps for
this to have the impact that you'd expect.

Change-Id: I72911adc8a03450522429c52db661ec0a1bc4678

tools/pretty_tox.sh

@@ -3,4 +3,4 @@
 set -o pipefail
 
 TESTRARGS=$1
-python -m nova.openstack.common.lockutils python setup.py testr --slowest --testr-args="--subunit $TESTRARGS" | $(dirname $0)/subunit-trace.py --no-failure-debug -f
+python -m nova.openstack.common.lockutils python setup.py testr --slowest --testr-args="--subunit $TESTRARGS" | $(dirname $0)/subunit-trace.py -f

tools/subunit-trace.py

@@ -20,6 +20,7 @@
 
 import argparse
 import functools
+import os
 import re
 import sys
 
@@ -152,7 +153,7 @@ def print_attachments(stream, test, all_channels=False):
                 stream.write("    %s\n" % line)
 
 
-def show_outcome(stream, test, print_failures=False):
+def show_outcome(stream, test, print_failures=False, failonly=False):
     global RESULTS
     status = test['status']
     # TODO(sdague): ask lifeless why on this?
@@ -171,24 +172,25 @@ def show_outcome(stream, test, print_failures=False):
     if name == 'process-returncode':
         return
 
-    if status == 'success':
-        stream.write('{%s} %s [%s] ... ok\n' % (
-            worker, name, duration))
-        print_attachments(stream, test)
-    elif status == 'fail':
+    if status == 'fail':
         FAILS.append(test)
         stream.write('{%s} %s [%s] ... FAILED\n' % (
             worker, name, duration))
         if not print_failures:
             print_attachments(stream, test, all_channels=True)
-    elif status == 'skip':
-        stream.write('{%s} %s ... SKIPPED: %s\n' % (
-            worker, name, test['details']['reason'].as_text()))
-    else:
-        stream.write('{%s} %s [%s] ... %s\n' % (
-            worker, name, duration, test['status']))
-        if not print_failures:
-            print_attachments(stream, test, all_channels=True)
+    elif not failonly:
+        if status == 'success':
+            stream.write('{%s} %s [%s] ... ok\n' % (
+                worker, name, duration))
+            print_attachments(stream, test)
+        elif status == 'skip':
+            stream.write('{%s} %s ... SKIPPED: %s\n' % (
+                worker, name, test['details']['reason'].as_text()))
+        else:
+            stream.write('{%s} %s [%s] ... %s\n' % (
+                worker, name, duration, test['status']))
+            if not print_failures:
+                print_attachments(stream, test, all_channels=True)
 
     stream.flush()
 
@@ -267,6 +269,11 @@ def parse_args():
     parser.add_argument('--fails', '-f', action='store_true',
                         dest='post_fails', help='Print failure debug '
                         'information after the stream is proccesed')
+    parser.add_argument('--failonly', action='store_true',
+                        dest='failonly', help="Don't print success items",
+                        default=(
+                            os.environ.get('TRACE_FAILONLY', False)
+                            is not False))
     return parser.parse_args()
 
 
@@ -277,7 +284,9 @@ def main():
     starts = Starts(sys.stdout)
     outcomes = testtools.StreamToDict(
         functools.partial(show_outcome, sys.stdout,
-                          print_failures=args.print_failures))
+                          print_failures=args.print_failures,
+                          failonly=args.failonly
+                      ))
     summary = testtools.StreamSummary()
     result = testtools.CopyStreamResult([starts, outcomes, summary])
     result.startTestRun()

tox.ini

@@ -15,6 +15,8 @@ deps = -r{toxinidir}/requirements.txt
 commands =
   find . -type f -name "*.pyc" -delete
   bash tools/pretty_tox.sh '{posargs}'
+# there is also secret magic in pretty_tox.sh which lets you run in a fail only
+# mode. To do this define the TRACE_FAILONLY environmental variable.
 
 [tox:jenkins]
 downloadcache = ~/cache/pip