From 497074a1a792d3dfc9ccae45a6c142b9e7597ce9 Mon Sep 17 00:00:00 2001 From: Chris Hoge Date: Tue, 11 Sep 2018 10:28:35 -0600 Subject: [PATCH] Make consistency job gating. This patch updates some of the consistency checking tools to make them more generally usable by allowing both automatic checkout of tempest and manual checkout. It builds on this to add a Zuul job that gates against the consistency check. Change-Id: Id0d9148af39c2d2ccf8f11502cc9aff6699f0ead --- tools/checktests.py | 50 +++++++++++++++--------------- tools/consistency.sh | 35 ++++++++++++++++----- tox.ini | 2 +- zuul.d/jobs.yaml | 12 +++++++ .zuul.yaml => zuul.d/projects.yaml | 2 ++ 5 files changed, 68 insertions(+), 33 deletions(-) create mode 100644 zuul.d/jobs.yaml rename .zuul.yaml => zuul.d/projects.yaml (68%) diff --git a/tools/checktests.py b/tools/checktests.py index c82d3459..c686b804 100644 --- a/tools/checktests.py +++ b/tools/checktests.py @@ -152,41 +152,41 @@ def run(): exit_code = 0 if len(missing_uuids) > 0: exit_code = 1 - print "### Idempotent ID Errors Detected. To resolve these errors, " \ - "fix the uuid name (format id-) in the guideline to " \ - "match the id in the test suite:" + print("### Idempotent ID Errors Detected. To resolve these errors, " + "fix the uuid name (format id-) in the guideline to " + "match the id in the test suite:") for test in missing_uuids: - print "Idempotent ID in guideline '%s' does not appear in test " \ - "library '%s'\n" \ - " idempotent_id:\n" \ - " %s\n" \ - " names: " % (args.guideline_file, args.testlib, test[0]) + print("Idempotent ID in guideline '%s' does not appear in test " + "library '%s'\n" + " idempotent_id:\n" + " %s\n" + " names: " % (args.guideline_file, args.testlib, test[0])) for testname in test[1]: - print " %s" % (testname) - print "" + print(" %s" % (testname)) + print("") if len(missing_tests) > 0: exit_code = 1 - print "### Test Name Errors Detected. " \ - "To resolve these errors, update " \ - "the Interop guideline with the missing " \ - "test names:" + print("### Test Name Errors Detected. " + "To resolve these errors, update " + "the Interop guideline with the missing " + "test names:") for uuid in missing_tests: - print "Test found in test library '%s'\n" \ - " idempotent_id:\n" \ - " %s\n" \ - " name:\n" \ - " %s\n" \ - "Entry in guideline '%s'\n" \ - " idempotent_id:\n" \ - " %s\n" \ + print("Test found in test library '%s'\n" + " idempotent_id:\n" + " %s\n" + " name:\n" + " %s\n" + "Entry in guideline '%s'\n" + " idempotent_id:\n" + " %s\n" " names: " % (args.testlib, uuid, tests[uuid], args.guideline_file, - missing_tests[uuid][0]) + missing_tests[uuid][0])) for testname in missing_tests[uuid][1]: - print " %s" % (testname) - print "" + print(" %s" % (testname)) + print("") sys.exit(exit_code) diff --git a/tools/consistency.sh b/tools/consistency.sh index c623e4fa..bb32545f 100755 --- a/tools/consistency.sh +++ b/tools/consistency.sh @@ -1,19 +1,40 @@ #!/bin/bash -tempestdir=$(mktemp -d) -echo $tempestdir -git clone git://git.openstack.org/openstack/tempest $tempestdir +# This script will run consistency checks for Tempest tests against +# the three latest interoperability guidelines. It can run in two +# modes. +# +# * If no arguments are specified, the script will check out Tempest +# into a temporary directory, run the consistency checks, then delete +# temporary checkout. +# +# * If an argument is given, this script will assume that it is a +# user checked-out repository and run the consistency checks against +# that, and leave the directory unchanged on exit. This mode is useful +# for gate jobs and Tempest development. +set -x -PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline next.json +if [ ! $@ ]; then + TEMPESTDIR=$(mktemp -d) + git clone git://git.openstack.org/openstack/tempest $TEMPESTDIR + CLEANTEMPEST=cleantempest +else + TEMPESTDIR=${1} +fi + +PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline next.json exit_1=$? -PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline 2018.02.json +PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2018.02.json exit_2=$? -PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline 2017.09.json +PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2017.09.json exit_3=$? -rm -rf $tempestdir +if [[ ! -z "${CLEANTEMPEST}" ]]; then + rm -rf $TEMPESTDIR +fi + ! (( $exit_1 || $exit_2 || $exit_3 )) diff --git a/tox.ini b/tox.ini index 9f552ead..29230a28 100644 --- a/tox.ini +++ b/tox.ini @@ -76,7 +76,7 @@ commands= [testenv:consistency] commands= - bash tools/consistency.sh + bash tools/consistency.sh {posargs} whitelist_externals = sh bash diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml new file mode 100644 index 00000000..6c737a0d --- /dev/null +++ b/zuul.d/jobs.yaml @@ -0,0 +1,12 @@ +- job: + name: interop-tempest-consistency + parent: tox + description: | + Ensure consistency between Tempest source and current interop guidelines. + vars: + tox_envlist: consistency + tox_extra_args: "-vv -- {{ ansible_user_dir }}/{{ zuul.projects['git.openstack.org/openstack/tempest'].src_dir }}" + zuul_work_dir: src/git.openstack.org/openstack/interop + required-projects: + - name: openstack/tempest + - name: openstack/interop diff --git a/.zuul.yaml b/zuul.d/projects.yaml similarity index 68% rename from .zuul.yaml rename to zuul.d/projects.yaml index 315a2eaf..4b419196 100644 --- a/.zuul.yaml +++ b/zuul.d/projects.yaml @@ -4,6 +4,8 @@ check: jobs: - openstack-tox-pep8 + - interop-tempest-consistency gate: jobs: - openstack-tox-pep8 + - interop-tempest-consistency