diff --git a/.gitignore b/.gitignore index 26415aface..57eedbbd2a 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ keystone/tests/tmp/ .project .pydevproject keystone/locale/*/LC_MESSAGES/*.mo +.testrepository/ diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 0000000000..33297a1ca2 --- /dev/null +++ b/.testr.conf @@ -0,0 +1,12 @@ +[DEFAULT] +test_command=${PYTHON:-python} -m subunit.run discover \ + -t ./ ./keystone/tests \ + $LISTOPT $IDOPTION + +test_id_option=--load-list $IDFILE +test_list_option=--list + +# NOTE(dstanek): Ensures that Keystone test never run in parallel. +# Please remove once the issues have been worked out. +# Bug: #1240052 +test_run_concurrency=echo 1 diff --git a/keystone/tests/backend_sql_disk.conf b/keystone/tests/backend_sql_disk.conf index 0f8dfea7df..bb60879861 100644 --- a/keystone/tests/backend_sql_disk.conf +++ b/keystone/tests/backend_sql_disk.conf @@ -1,2 +1,2 @@ [sql] -connection = sqlite:///tmp/test.db +connection = sqlite:///keystone/tests/tmp/test.db diff --git a/keystone/tests/core.py b/keystone/tests/core.py index 612d196d4b..f5ff65fc52 100644 --- a/keystone/tests/core.py +++ b/keystone/tests/core.py @@ -25,7 +25,6 @@ import time from lxml import etree import mox -import nose.exc from paste import deploy import stubout import testtools @@ -74,9 +73,9 @@ from keystone.openstack.common import policy as common_policy # noqa LOG = logging.getLogger(__name__) -ROOTDIR = os.path.dirname(os.path.abspath('..')) +TESTSDIR = os.path.dirname(os.path.abspath(__file__)) +ROOTDIR = os.path.normpath(os.path.join(TESTSDIR, '..', '..')) VENDOR = os.path.join(ROOTDIR, 'vendor') -TESTSDIR = os.path.join(ROOTDIR, 'keystone', 'tests') ETCDIR = os.path.join(ROOTDIR, 'etc') TMPDIR = os.path.join(TESTSDIR, 'tmp') @@ -490,7 +489,7 @@ class TestCase(NoModule, testtools.TestCase): def assertEqualXML(self, a, b): """Parses two XML documents from strings and compares the results. - This provides easy-to-read failures from nose. + This provides easy-to-read failures. """ parser = etree.XMLParser(remove_blank_text=True) @@ -510,13 +509,12 @@ class TestCase(NoModule, testtools.TestCase): b = canonical_xml(b) self.assertEqual(a.split('\n'), b.split('\n')) - @staticmethod - def skip_if_no_ipv6(): + def skip_if_no_ipv6(self): try: s = socket.socket(socket.AF_INET6) except socket.error as e: if e.errno == errno.EAFNOSUPPORT: - raise nose.exc.SkipTest("IPv6 is not enabled in the system") + raise self.skipTest("IPv6 is not enabled in the system") else: raise else: diff --git a/keystone/tests/test_ipv6.py b/keystone/tests/test_ipv6.py index e55847da2d..aa2b4e7519 100644 --- a/keystone/tests/test_ipv6.py +++ b/keystone/tests/test_ipv6.py @@ -24,11 +24,9 @@ CONF = config.CONF class IPv6TestCase(tests.TestCase): - @classmethod - def setUpClass(cls): - cls.skip_if_no_ipv6() def setUp(self): + self.skip_if_no_ipv6() super(IPv6TestCase, self).setUp() self.load_backends() diff --git a/keystone/tests/test_overrides.conf b/keystone/tests/test_overrides.conf index 801b0d2f76..4c3abbc249 100644 --- a/keystone/tests/test_overrides.conf +++ b/keystone/tests/test_overrides.conf @@ -21,6 +21,6 @@ debug_cache_backend = True proxies = keystone.tests.test_cache.CacheIsolatingProxy [signing] -certfile = ../../examples/pki/certs/signing_cert.pem -keyfile = ../../examples/pki/private/signing_key.pem -ca_certs = ../../examples/pki/certs/cacert.pem +certfile = examples/pki/certs/signing_cert.pem +keyfile = examples/pki/private/signing_key.pem +ca_certs = examples/pki/certs/cacert.pem diff --git a/keystone/tests/test_v3_auth.py b/keystone/tests/test_v3_auth.py index 245ca68db2..d30d5cc5a3 100644 --- a/keystone/tests/test_v3_auth.py +++ b/keystone/tests/test_v3_auth.py @@ -1071,7 +1071,7 @@ class TestTokenRevoking(test_v3.RestfulTestCase): class TestAuthExternalDisabled(test_v3.RestfulTestCase): def config_files(self): list = self._config_file_list[:] - list.append('auth_plugin_external_disabled.conf') + list.append(tests.testsdir('auth_plugin_external_disabled.conf')) return list def test_remote_user_disabled(self): @@ -1093,7 +1093,7 @@ class TestAuthExternalDomain(test_v3.RestfulTestCase): def config_files(self): list = self._config_file_list[:] - list.append('auth_plugin_external_domain.conf') + list.append(tests.testsdir('auth_plugin_external_domain.conf')) return list def test_remote_user_with_realm(self): diff --git a/openstack-common.conf b/openstack-common.conf index e36745c7b0..85e33ea6b3 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -3,6 +3,7 @@ # The list of modules to copy from openstack-common module=db module=db.sqlalchemy +module=colorizer module=crypto module=importutils module=install_venv_common diff --git a/run_tests.sh b/run_tests.sh index 198d2564aa..8a48ffc5eb 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -32,9 +32,6 @@ function usage { echo " -P, --no-pep8 Don't run flake8" echo " -c, --coverage Generate coverage report" echo " -h, --help Print this usage message" - echo " -xintegration Ignore all keystoneclient test cases (integration tests)" - echo " --hide-elapsed Don't print the elapsed time for each test along with slow test list" - echo " --standard-threads Don't do the eventlet threading monkeypatch." echo "" echo "Note: with no options specified, the script will try to run the tests in a virtual environment," echo " If no virtualenv is found, the script will ask if you would like to create one. If you " @@ -55,12 +52,8 @@ function process_option { -8|--8) short_flake8=1;; -P|--no-pep8) no_flake8=1;; -c|--coverage) coverage=1;; - -xintegration) nokeystoneclient=1;; - --standard-threads) - export STANDARD_THREADS=1 - ;; - -*) noseopts="$noseopts $1";; - *) noseargs="$noseargs $1" + -*) testropts="$testropts $1";; + *) testrargs="$testrargs $1" esac } @@ -69,14 +62,13 @@ with_venv=tools/with_venv.sh always_venv=0 never_venv=0 force=0 -noseargs= -noseopts="--with-openstack --openstack-color" +testrargs= +testropts=--subunit wrapper="" just_flake8=0 short_flake8=0 no_flake8=0 coverage=0 -nokeystoneclient=0 recreate_db=1 update=0 @@ -84,14 +76,11 @@ for arg in "$@"; do process_option $arg done +TESTRTESTS="python setup.py testr" + # If enabled, tell nose to collect coverage data if [ $coverage -eq 1 ]; then - noseopts="$noseopts --with-coverage --cover-package=keystone" -fi - -if [ $nokeystoneclient -eq 1 ]; then - # disable the integration tests - noseopts="$noseopts -I test_keystoneclient* -I _test_import_auth_token.py" + TESTRTESTS="$TESTRTESTS --coverage" fi function cleanup_test_db { @@ -103,19 +92,11 @@ function cleanup_test_db { } function run_tests { - # Just run the test suites in current environment - ${wrapper} $NOSETESTS - # If we get some short import error right away, print the error log directly - RESULT=$? - if [ "$RESULT" -ne "0" ]; - then - ERRSIZE=`wc -l run_tests.log | awk '{print \$1}'` - if [ "$ERRSIZE" -lt "40" ]; - then - cat run_tests.log - fi - fi - return $RESULT + set -e + echo ${wrapper} + ${wrapper} $TESTRTESTS --testr-args="$testropts $testrargs" | \ + ${wrapper} subunit-2to1 | \ + ${wrapper} tools/colorizer.py } function run_flake8 { @@ -125,14 +106,16 @@ function run_flake8 { FLAGS='' fi - echo "Running flake8 ..." # Just run flake8 in current environment echo ${wrapper} flake8 $FLAGS | tee pep8.txt ${wrapper} flake8 $FLAGS | tee pep8.txt } -NOSETESTS="nosetests $noseopts $noseargs" +echo "This script is now deprecated. Please use tox instead." +echo "Checkout http://tox.readthedocs.org/en/latest/ for information on tox." +echo "[press enter to continue]" +read if [ $never_venv -eq 0 ] then @@ -188,15 +171,10 @@ run_tests # NOTE(sirp): we only want to run flake8 when we're running the full-test # suite, not when we're running tests individually. To handle this, we need to -# distinguish between options (noseopts), which begin with a '-', and arguments -# (noseargs). -if [ -z "$noseargs" ]; then +# distinguish between options (testropts), which begin with a '-', and arguments +# (testrargs). +if [ -z "$testrargs" ]; then if [ $no_flake8 -eq 0 ]; then run_flake8 fi fi - -if [ $coverage -eq 1 ]; then - echo "Generating coverage report in covhtml/" - ${wrapper} coverage html -d covhtml -i -fi diff --git a/setup.cfg b/setup.cfg index 36496277f8..6654aabd06 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,15 +55,3 @@ mapping_file = babel.cfg output_file = keystone/locale/keystone.pot copyright_holder = OpenStack Foundation msgid_bugs_address = https://bugs.launchpad.net/keystone - -[nosetests] -# NOTE(jkoelker) To run the test suite under nose install the following -# coverage http://pypi.python.org/pypi/coverage -# tissue http://pypi.python.org/pypi/tissue (pep8 checker) -# openstack-nose https://github.com/jkoelker/openstack-nose -verbosity=2 -detailed-errors=1 -cover-package = keystone -cover-html = true -cover-erase = true -where=keystone/tests diff --git a/test-requirements.txt b/test-requirements.txt index cb4635d1ad..703984c09d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -15,17 +15,17 @@ python-ldap==2.3.13 coverage # mock object framework mox -# for test discovery and console feedback -nose -nosexcover -openstack.nose_plugin -nosehtmloutput # required to build documentation Sphinx>=1.1.2 -testtools>=0.9.32 # test wsgi apps without starting an http server webtest +extras +discover +python-subunit +testrepository>=0.0.17 +testtools>=0.9.32 + # for python-keystoneclient # keystoneclient <0.2.1 httplib2 diff --git a/tools/colorizer.py b/tools/colorizer.py new file mode 100755 index 0000000000..13364badb0 --- /dev/null +++ b/tools/colorizer.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2013, Nebula, Inc. +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Colorizer Code is borrowed from Twisted: +# Copyright (c) 2001-2010 Twisted Matrix Laboratories. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Display a subunit stream through a colorized unittest test runner.""" + +import heapq +import subunit +import sys +import unittest + +import testtools + + +class _AnsiColorizer(object): + """Colorizer allows callers to write text in a particular color. + + A colorizer is an object that loosely wraps around a stream, allowing + callers to write text to the stream in a particular color. + + Colorizer classes must implement C{supported()} and C{write(text, color)}. + """ + _colors = dict(black=30, red=31, green=32, yellow=33, + blue=34, magenta=35, cyan=36, white=37) + + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + """Check is the current platform supports coloring terminal output. + + A class method that returns True if the current platform supports + coloring terminal output using this method. Returns False otherwise. + """ + if not stream.isatty(): + return False # auto color only on TTYs + try: + import curses + except ImportError: + return False + else: + try: + try: + return curses.tigetnum("colors") > 2 + except curses.error: + curses.setupterm() + return curses.tigetnum("colors") > 2 + except Exception: + # guess false in case of error + return False + supported = classmethod(supported) + + def write(self, text, color): + """Write the given text to the stream in the given color. + + @param text: Text to be written to the stream. + + @param color: A string label for a color. e.g. 'red', 'white'. + """ + color = self._colors[color] + self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text)) + + +class _Win32Colorizer(object): + """See _AnsiColorizer docstring.""" + def __init__(self, stream): + import win32console + red, green, blue, bold = (win32console.FOREGROUND_RED, + win32console.FOREGROUND_GREEN, + win32console.FOREGROUND_BLUE, + win32console.FOREGROUND_INTENSITY) + self.stream = stream + self.screenBuffer = win32console.GetStdHandle( + win32console.STD_OUT_HANDLE) + self._colors = { + 'normal': red | green | blue, + 'red': red | bold, + 'green': green | bold, + 'blue': blue | bold, + 'yellow': red | green | bold, + 'magenta': red | blue | bold, + 'cyan': green | blue | bold, + 'white': red | green | blue | bold, + } + + def supported(cls, stream=sys.stdout): + try: + import win32console + screenBuffer = win32console.GetStdHandle( + win32console.STD_OUT_HANDLE) + except ImportError: + return False + import pywintypes + try: + screenBuffer.SetConsoleTextAttribute( + win32console.FOREGROUND_RED | + win32console.FOREGROUND_GREEN | + win32console.FOREGROUND_BLUE) + except pywintypes.error: + return False + else: + return True + supported = classmethod(supported) + + def write(self, text, color): + color = self._colors[color] + self.screenBuffer.SetConsoleTextAttribute(color) + self.stream.write(text) + self.screenBuffer.SetConsoleTextAttribute(self._colors['normal']) + + +class _NullColorizer(object): + """See _AnsiColorizer docstring.""" + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + return True + supported = classmethod(supported) + + def write(self, text, color): + self.stream.write(text) + + +def get_elapsed_time_color(elapsed_time): + if elapsed_time > 1.0: + return 'red' + elif elapsed_time > 0.25: + return 'yellow' + else: + return 'green' + + +class OpenStackTestResult(testtools.TestResult): + def __init__(self, stream, descriptions, verbosity): + super(OpenStackTestResult, self).__init__() + self.stream = stream + self.showAll = verbosity > 1 + self.num_slow_tests = 10 + self.slow_tests = [] # this is a fixed-sized heap + self.colorizer = None + # NOTE(vish): reset stdout for the terminal check + stdout = sys.stdout + sys.stdout = sys.__stdout__ + for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]: + if colorizer.supported(): + self.colorizer = colorizer(self.stream) + break + sys.stdout = stdout + self.start_time = None + self.last_time = {} + self.results = {} + self.last_written = None + + def _writeElapsedTime(self, elapsed): + color = get_elapsed_time_color(elapsed) + self.colorizer.write(" %.2f" % elapsed, color) + + def _addResult(self, test, *args): + try: + name = test.id() + except AttributeError: + name = 'Unknown.unknown' + test_class, test_name = name.rsplit('.', 1) + + elapsed = (self._now() - self.start_time).total_seconds() + item = (elapsed, test_class, test_name) + if len(self.slow_tests) >= self.num_slow_tests: + heapq.heappushpop(self.slow_tests, item) + else: + heapq.heappush(self.slow_tests, item) + + self.results.setdefault(test_class, []) + self.results[test_class].append((test_name, elapsed) + args) + self.last_time[test_class] = self._now() + self.writeTests() + + def _writeResult(self, test_name, elapsed, long_result, color, + short_result, success): + if self.showAll: + self.stream.write(' %s' % str(test_name).ljust(66)) + self.colorizer.write(long_result, color) + if success: + self._writeElapsedTime(elapsed) + self.stream.writeln() + else: + self.colorizer.write(short_result, color) + + def addSuccess(self, test): + super(OpenStackTestResult, self).addSuccess(test) + self._addResult(test, 'OK', 'green', '.', True) + + def addFailure(self, test, err): + if test.id() == 'process-returncode': + return + super(OpenStackTestResult, self).addFailure(test, err) + self._addResult(test, 'FAIL', 'red', 'F', False) + + def addError(self, test, err): + super(OpenStackTestResult, self).addFailure(test, err) + self._addResult(test, 'ERROR', 'red', 'E', False) + + def addSkip(self, test, reason=None, details=None): + super(OpenStackTestResult, self).addSkip(test, reason, details) + self._addResult(test, 'SKIP', 'blue', 'S', True) + + def startTest(self, test): + self.start_time = self._now() + super(OpenStackTestResult, self).startTest(test) + + def writeTestCase(self, cls): + if not self.results.get(cls): + return + if cls != self.last_written: + self.colorizer.write(cls, 'white') + self.stream.writeln() + for result in self.results[cls]: + self._writeResult(*result) + del self.results[cls] + self.stream.flush() + self.last_written = cls + + def writeTests(self): + time = self.last_time.get(self.last_written, self._now()) + if not self.last_written or (self._now() - time).total_seconds() > 2.0: + diff = 3.0 + while diff > 2.0: + classes = self.results.keys() + oldest = min(classes, key=lambda x: self.last_time[x]) + diff = (self._now() - self.last_time[oldest]).total_seconds() + self.writeTestCase(oldest) + else: + self.writeTestCase(self.last_written) + + def done(self): + self.stopTestRun() + + def stopTestRun(self): + for cls in list(self.results.iterkeys()): + self.writeTestCase(cls) + self.stream.writeln() + self.writeSlowTests() + + def writeSlowTests(self): + # Pare out 'fast' tests + slow_tests = [item for item in self.slow_tests + if get_elapsed_time_color(item[0]) != 'green'] + if slow_tests: + slow_total_time = sum(item[0] for item in slow_tests) + slow = ("Slowest %i tests took %.2f secs:" + % (len(slow_tests), slow_total_time)) + self.colorizer.write(slow, 'yellow') + self.stream.writeln() + last_cls = None + # sort by name + for elapsed, cls, name in sorted(slow_tests, + key=lambda x: x[1] + x[2]): + if cls != last_cls: + self.colorizer.write(cls, 'white') + self.stream.writeln() + last_cls = cls + self.stream.write(' %s' % str(name).ljust(68)) + self._writeElapsedTime(elapsed) + self.stream.writeln() + + def printErrors(self): + if self.showAll: + self.stream.writeln() + self.printErrorList('ERROR', self.errors) + self.printErrorList('FAIL', self.failures) + + def printErrorList(self, flavor, errors): + for test, err in errors: + self.colorizer.write("=" * 70, 'red') + self.stream.writeln() + self.colorizer.write(flavor, 'red') + self.stream.writeln(": %s" % test.id()) + self.colorizer.write("-" * 70, 'red') + self.stream.writeln() + self.stream.writeln("%s" % err) + + +test = subunit.ProtocolTestCase(sys.stdin, passthrough=None) + +if sys.version_info[0:2] <= (2, 6): + runner = unittest.TextTestRunner(verbosity=2) +else: + runner = unittest.TextTestRunner(verbosity=2, + resultclass=OpenStackTestResult) + +if runner.run(test).wasSuccessful(): + exit_code = 0 +else: + exit_code = 1 +sys.exit(exit_code) diff --git a/tox.ini b/tox.ini index c2c9cb749e..c8a7860acc 100644 --- a/tox.ini +++ b/tox.ini @@ -5,18 +5,14 @@ envlist = py26,py27,py33,pep8 [testenv] usedevelop = True -install_command = pip install {opts} {packages} +install_command = pip install -U {opts} {packages} setenv = VIRTUAL_ENV={envdir} - NOSE_WITH_OPENSTACK=1 - NOSE_OPENSTACK_COLOR=1 - NOSE_OPENSTACK_RED=0.05 - NOSE_OPENSTACK_YELLOW=0.025 - NOSE_OPENSTACK_SHOW_ELAPSED=1 - NOSE_OPENSTACK_STDOUT=1 + LANG=en_US.UTF-8 + LANGUAGE=en_US:en + LC_ALL=C deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -commands = python tools/patch_tox_venv.py - nosetests {posargs} +commands = python setup.py testr --testr-args='{posargs}' [testenv:pep8] commands = flake8 @@ -26,9 +22,7 @@ downloadcache = ~/cache/pip [testenv:cover] setenv = VIRTUAL_ENV={envdir} - NOSE_WITH_COVERAGE=1 - NOSE_COVER_HTML=1 - NOSE_COVER_HTML_DIR={toxinidir}/cover +commands = python setup.py testr --testr-args='--coverage {posargs}' [testenv:venv] commands = {posargs}