Make consistency job gating.

This patch updates some of the consistency checking tools
to make them more generally usable by allowing both
automatic checkout of tempest and manual checkout. It builds
on this to add a Zuul job that gates against the consistency
check.

Change-Id: Id0d9148af39c2d2ccf8f11502cc9aff6699f0ead
This commit is contained in:
Chris Hoge 2018-09-11 10:28:35 -06:00
parent fa75d41c79
commit 497074a1a7
5 changed files with 68 additions and 33 deletions

View File

@ -152,41 +152,41 @@ def run():
exit_code = 0
if len(missing_uuids) > 0:
exit_code = 1
print "### Idempotent ID Errors Detected. To resolve these errors, " \
"fix the uuid name (format id-<uuid>) in the guideline to " \
"match the id in the test suite:"
print("### Idempotent ID Errors Detected. To resolve these errors, "
"fix the uuid name (format id-<uuid>) in the guideline to "
"match the id in the test suite:")
for test in missing_uuids:
print "Idempotent ID in guideline '%s' does not appear in test " \
"library '%s'\n" \
" idempotent_id:\n" \
" %s\n" \
" names: " % (args.guideline_file, args.testlib, test[0])
print("Idempotent ID in guideline '%s' does not appear in test "
"library '%s'\n"
" idempotent_id:\n"
" %s\n"
" names: " % (args.guideline_file, args.testlib, test[0]))
for testname in test[1]:
print " %s" % (testname)
print ""
print(" %s" % (testname))
print("")
if len(missing_tests) > 0:
exit_code = 1
print "### Test Name Errors Detected. " \
"To resolve these errors, update " \
"the Interop guideline with the missing " \
"test names:"
print("### Test Name Errors Detected. "
"To resolve these errors, update "
"the Interop guideline with the missing "
"test names:")
for uuid in missing_tests:
print "Test found in test library '%s'\n" \
" idempotent_id:\n" \
" %s\n" \
" name:\n" \
" %s\n" \
"Entry in guideline '%s'\n" \
" idempotent_id:\n" \
" %s\n" \
print("Test found in test library '%s'\n"
" idempotent_id:\n"
" %s\n"
" name:\n"
" %s\n"
"Entry in guideline '%s'\n"
" idempotent_id:\n"
" %s\n"
" names: " % (args.testlib,
uuid, tests[uuid],
args.guideline_file,
missing_tests[uuid][0])
missing_tests[uuid][0]))
for testname in missing_tests[uuid][1]:
print " %s" % (testname)
print ""
print(" %s" % (testname))
print("")
sys.exit(exit_code)

View File

@ -1,19 +1,40 @@
#!/bin/bash
tempestdir=$(mktemp -d)
echo $tempestdir
git clone git://git.openstack.org/openstack/tempest $tempestdir
# This script will run consistency checks for Tempest tests against
# the three latest interoperability guidelines. It can run in two
# modes.
#
# * If no arguments are specified, the script will check out Tempest
# into a temporary directory, run the consistency checks, then delete
# temporary checkout.
#
# * If an argument is given, this script will assume that it is a
# user checked-out repository and run the consistency checks against
# that, and leave the directory unchanged on exit. This mode is useful
# for gate jobs and Tempest development.
set -x
PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline next.json
if [ ! $@ ]; then
TEMPESTDIR=$(mktemp -d)
git clone git://git.openstack.org/openstack/tempest $TEMPESTDIR
CLEANTEMPEST=cleantempest
else
TEMPESTDIR=${1}
fi
PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline next.json
exit_1=$?
PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline 2018.02.json
PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2018.02.json
exit_2=$?
PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline 2017.09.json
PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2017.09.json
exit_3=$?
rm -rf $tempestdir
if [[ ! -z "${CLEANTEMPEST}" ]]; then
rm -rf $TEMPESTDIR
fi
! (( $exit_1 || $exit_2 || $exit_3 ))

View File

@ -76,7 +76,7 @@ commands=
[testenv:consistency]
commands=
bash tools/consistency.sh
bash tools/consistency.sh {posargs}
whitelist_externals =
sh
bash

12
zuul.d/jobs.yaml Normal file
View File

@ -0,0 +1,12 @@
- job:
name: interop-tempest-consistency
parent: tox
description: |
Ensure consistency between Tempest source and current interop guidelines.
vars:
tox_envlist: consistency
tox_extra_args: "-vv -- {{ ansible_user_dir }}/{{ zuul.projects['git.openstack.org/openstack/tempest'].src_dir }}"
zuul_work_dir: src/git.openstack.org/openstack/interop
required-projects:
- name: openstack/tempest
- name: openstack/interop

View File

@ -4,6 +4,8 @@
check:
jobs:
- openstack-tox-pep8
- interop-tempest-consistency
gate:
jobs:
- openstack-tox-pep8
- interop-tempest-consistency