From c14aaee50ba0e9bda26e5e24cf28f2301c389180 Mon Sep 17 00:00:00 2001 From: Luigi Toscano Date: Mon, 25 Jun 2018 23:06:58 +0200 Subject: [PATCH] Port away from ostestr/testr and use stestr The changes impacts different components: - unit tests uses stestr directly instead of ostestr; - the cover target uses stestr+coverage instead of testr; - sahara-scenario (scenario tests runner) uses stestr internally instead of stestr; - remove the references to testr and ostestr from the documentation. Change-Id: I5eddc4a0f9b245863b4de9056df4c27c876f7123 --- doc/source/tempest-plugin.rst | 35 ++++++------------- .../switch-to-stestr-6de59dc9e46768f6.yaml | 4 +++ requirements.txt | 3 +- sahara_tests/scenario/stestr.conf | 3 ++ sahara_tests/scenario/testr.conf | 8 ----- sahara_tests/scenario/utils.py | 12 +++---- tools/cover.sh | 15 ++++++-- tox.ini | 4 +-- 8 files changed, 38 insertions(+), 46 deletions(-) create mode 100644 releasenotes/notes/switch-to-stestr-6de59dc9e46768f6.yaml create mode 100644 sahara_tests/scenario/stestr.conf delete mode 100644 sahara_tests/scenario/testr.conf diff --git a/doc/source/tempest-plugin.rst b/doc/source/tempest-plugin.rst index 3a27817a..be375478 100644 --- a/doc/source/tempest-plugin.rst +++ b/doc/source/tempest-plugin.rst @@ -24,39 +24,24 @@ same environment where Tempest is installed. .. -After that you can run Tempest tests. There you can specify the name of -test (and even run a single test from directory). One way to run Tempest -tests by using ``ostestr`` command, for instance: +After that you can run Tempest tests. You can specify the name of +test or a more complex regular expression. While any ``testr``-based +test runner can be used, the official command for executing Tempest +tests is ``tempest run``. + +For example, the following command will run a specific subset of tests: .. sourcecode:: console - $ ostestr sahara_tempest_plugin.tests.cli.test_scenario.Scenario.test_plugin_cli + $ tempest run --regex '^sahara_tempest_plugin.tests.cli.test_scenario.Scenario.' .. -The command ``tempest run`` is another way to run tests. See the additional -information about using this `command `_. +The full syntax of ``tempest run`` is described on `the relavant section of +the Tempest documentation `_. -For example, the following command will run all the Tempest tests. +Other useful links: -.. sourcecode:: console - - $ tempest run - -.. - -Finally, you can use ``testr`` directly to run the tests. For example, -the following command will run all the cli-tests: - -.. sourcecode:: console - - $ testr run sahara_tempest_plugin.tests.cli - -.. - -Useful links: - -* `Running Tempest tests with testr `_. * `Using Tempest plugins `_. * `Tempest Quickstart `_. diff --git a/releasenotes/notes/switch-to-stestr-6de59dc9e46768f6.yaml b/releasenotes/notes/switch-to-stestr-6de59dc9e46768f6.yaml new file mode 100644 index 00000000..31a7f9ec --- /dev/null +++ b/releasenotes/notes/switch-to-stestr-6de59dc9e46768f6.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + sahara-scenario now requires stestr. diff --git a/requirements.txt b/requirements.txt index 0bb22e75..c0f8a5db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,6 @@ oslo.serialization>=1.10.0 # Apache-2.0 oslo.utils>=3.5.0 # Apache-2.0 oslotest>=1.10.0 # Apache-2.0 os-client-config>=1.13.1 # Apache-2.0 -os-testr>=0.8.0 # Apache-2.0 paramiko>=1.16.0 # LGPL python-glanceclient>=2.0.0 # Apache-2.0 python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0 @@ -22,6 +21,6 @@ python-swiftclient>=2.2.0 # Apache-2.0 python-neutronclient>=4.2.0 # Apache-2.0 rfc3986>=0.2.0 # Apache-2.0 six>=1.9.0 # MIT +stestr>=1.0.0 # Apache-2.0 tempest>=16.0.0 # Apache-2.0 -testrepository>=0.0.18 # Apache-2.0/BSD testtools>=1.4.0 # MIT diff --git a/sahara_tests/scenario/stestr.conf b/sahara_tests/scenario/stestr.conf new file mode 100644 index 00000000..d177f2b2 --- /dev/null +++ b/sahara_tests/scenario/stestr.conf @@ -0,0 +1,3 @@ +[DEFAULT] +test_path=. +group_regex=([^\.]+\.)+ diff --git a/sahara_tests/scenario/testr.conf b/sahara_tests/scenario/testr.conf deleted file mode 100644 index 077512e8..00000000 --- a/sahara_tests/scenario/testr.conf +++ /dev/null @@ -1,8 +0,0 @@ -[DEFAULT] -test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ - OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ - OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ - ${PYTHON:-python} -m subunit.run discover $DISCOVER_DIRECTORY $LISTOPT $IDOPTION -test_id_option=--load-list $IDFILE -test_list_option=--list -group_regex=([^\.]+\.)+ diff --git a/sahara_tests/scenario/utils.py b/sahara_tests/scenario/utils.py index 0355f20d..eff6adce 100644 --- a/sahara_tests/scenario/utils.py +++ b/sahara_tests/scenario/utils.py @@ -44,7 +44,7 @@ DEFAULT_TEMPLATE_VARS = [os.path.join(TEST_TEMPLATE_DIR, 'edp.yaml.mako')] TEST_TEMPLATE_PATH = os.path.join(SCENARIO_RESOURCES_DIR, 'testcase.py.mako') -DEFAULT_TESTR_CONF = os.path.join(SCENARIO_RESOURCES_DIR, 'testr.conf') +DEFAULT_STESTR_CONF = os.path.join(SCENARIO_RESOURCES_DIR, 'stestr.conf') def rand_name(name=''): @@ -56,11 +56,10 @@ def rand_name(name=''): def run_tests(concurrency, test_dir_path): - command = ['ostestr'] + command = ['stestr', 'run'] if concurrency: - command.extend(['--concurrency', '%d' % concurrency]) + command.extend(['--concurrency=%d' % concurrency]) new_env = os.environ.copy() - new_env['DISCOVER_DIRECTORY'] = '.' tester_runner = subprocess.Popen(command, env=new_env, cwd=test_dir_path) tester_runner.communicate() return tester_runner.returncode @@ -83,8 +82,9 @@ def create_testcase_file(testcases, credentials, network, report): print("The generated test file located at: %s" % test_dir_path) fileutils.write_to_tempfile(testcase_data.encode("ASCII"), prefix='test_', suffix='.py', path=test_dir_path) - shutil.copyfile(DEFAULT_TESTR_CONF, os.path.join(test_dir_path, - '.testr.conf')) + # Copy both files as long as the old runner is supported + shutil.copyfile(DEFAULT_STESTR_CONF, os.path.join(test_dir_path, + '.stestr.conf')) return test_dir_path diff --git a/tools/cover.sh b/tools/cover.sh index 36ff3332..c658fc7a 100755 --- a/tools/cover.sh +++ b/tools/cover.sh @@ -23,13 +23,20 @@ show_diff () { } package_name=${PACKAGE_NAME:-sahara_tests} +export PYTHON="coverage run --source ${package_name} --parallel-mode" + +run_coverage () { + find . -type f -name "*.pyc" -delete && coverage erase && \ + stestr run "$*" && coverage combine +} + # Stash uncommitted changes, checkout master and save coverage report uncommitted=$(git status --porcelain | grep -v "^??") [[ -n $uncommitted ]] && git stash > /dev/null git checkout HEAD^ baseline_report=$(mktemp -t sahara-scenario_coverageXXXXXXX) -find . -type f -name "*.pyc" -delete && python setup.py test --coverage --coverage-package-name=${package_name} --testr-args="$*" +run_coverage "$*" coverage report > $baseline_report baseline_missing=$(awk '/^TOTAL/ { print $3 }' $baseline_report) @@ -39,10 +46,13 @@ git checkout - # Generate and save coverage report current_report=$(mktemp -t sahara-scenario_coverageXXXXXXX) -find . -type f -name "*.pyc" -delete && python setup.py test --coverage --coverage-package-name=${package_name} --testr-args="$*" +run_coverage "$*" coverage report > $current_report current_missing=$(awk '/^TOTAL/ { print $3 }' $current_report) +coverage html -d cover +coverage xml -o cover/coverage.xml + # Show coverage details allowed_missing=$((baseline_missing+ALLOWED_EXTRA_MISSING)) @@ -70,4 +80,5 @@ else fi rm $baseline_report $current_report + exit $exit_code diff --git a/tox.ini b/tox.ini index 31507c06..86a28b97 100644 --- a/tox.ini +++ b/tox.ini @@ -8,11 +8,10 @@ usedevelop = True install_command = pip install -U {opts} {packages} setenv = VIRTUAL_ENV={envdir} - DISCOVER_DIRECTORY=sahara_tests/unit/scenario deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -commands = ostestr {posargs} +commands = stestr run {posargs} passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY OS_* [testenv:venv] @@ -22,7 +21,6 @@ passenv = OS_* [testenv:cover] basepython = python3 setenv = - DISCOVER_DIRECTORY=sahara_tests PACKAGE_NAME=sahara_tests commands = {toxinidir}/tools/cover.sh {posargs}