diff --git a/tools/checktests.py b/tools/checktests.py
index c82d3459..c686b804 100644
--- a/tools/checktests.py
+++ b/tools/checktests.py
@@ -152,41 +152,41 @@ def run():
     exit_code = 0
     if len(missing_uuids) > 0:
         exit_code = 1
-        print "### Idempotent ID Errors Detected. To resolve these errors, " \
-              "fix the uuid name (format id-<uuid>) in the guideline to " \
-              "match the id in the test suite:"
+        print("### Idempotent ID Errors Detected. To resolve these errors, "
+              "fix the uuid name (format id-<uuid>) in the guideline to "
+              "match the id in the test suite:")
         for test in missing_uuids:
-            print "Idempotent ID in guideline '%s' does not appear in test " \
-                  "library '%s'\n" \
-                  "  idempotent_id:\n" \
-                  "    %s\n" \
-                  "  names: " % (args.guideline_file, args.testlib, test[0])
+            print("Idempotent ID in guideline '%s' does not appear in test "
+                  "library '%s'\n"
+                  "  idempotent_id:\n"
+                  "    %s\n"
+                  "  names: " % (args.guideline_file, args.testlib, test[0]))
             for testname in test[1]:
-                print "    %s" % (testname)
-            print ""
+                print("    %s" % (testname))
+            print("")
 
     if len(missing_tests) > 0:
         exit_code = 1
-        print "### Test Name Errors Detected. " \
-              "To resolve these errors, update " \
-              "the Interop guideline with the missing " \
-              "test names:"
+        print("### Test Name Errors Detected. "
+              "To resolve these errors, update "
+              "the Interop guideline with the missing "
+              "test names:")
         for uuid in missing_tests:
-            print "Test found in test library '%s'\n" \
-                  "  idempotent_id:\n" \
-                  "    %s\n" \
-                  "  name:\n" \
-                  "    %s\n" \
-                  "Entry in guideline '%s'\n" \
-                  "  idempotent_id:\n" \
-                  "    %s\n" \
+            print("Test found in test library '%s'\n"
+                  "  idempotent_id:\n"
+                  "    %s\n"
+                  "  name:\n"
+                  "    %s\n"
+                  "Entry in guideline '%s'\n"
+                  "  idempotent_id:\n"
+                  "    %s\n"
                   "  names: " % (args.testlib,
                                  uuid, tests[uuid],
                                  args.guideline_file,
-                                 missing_tests[uuid][0])
+                                 missing_tests[uuid][0]))
             for testname in missing_tests[uuid][1]:
-                print "    %s" % (testname)
-            print ""
+                print("    %s" % (testname))
+            print("")
 
     sys.exit(exit_code)
 
diff --git a/tools/consistency.sh b/tools/consistency.sh
index c623e4fa..bb32545f 100755
--- a/tools/consistency.sh
+++ b/tools/consistency.sh
@@ -1,19 +1,40 @@
 #!/bin/bash
 
-tempestdir=$(mktemp -d)
-echo $tempestdir
-git clone git://git.openstack.org/openstack/tempest $tempestdir
+# This script will run consistency checks for Tempest tests against
+# the three latest interoperability guidelines. It can run in two
+# modes.
+#
+# * If no arguments are specified, the script will check out Tempest
+#   into a temporary directory, run the consistency checks, then delete
+#   temporary checkout.
+#
+# * If an argument is given, this script will assume that it is a
+#   user checked-out repository and run the consistency checks against
+#   that, and leave the directory unchanged on exit. This mode is useful
+#   for gate jobs and Tempest development.
 
+set -x
 
-PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline next.json
+if [ ! $@ ]; then
+  TEMPESTDIR=$(mktemp -d)
+  git clone git://git.openstack.org/openstack/tempest $TEMPESTDIR
+  CLEANTEMPEST=cleantempest
+else
+  TEMPESTDIR=${1}
+fi
+
+PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline next.json
 exit_1=$?
 
-PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline 2018.02.json
+PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2018.02.json
 exit_2=$?
 
-PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline 2017.09.json
+PYTHONPATH=$TEMPESTDIR python ./tools/checktests.py --guideline 2017.09.json
 exit_3=$?
 
-rm -rf $tempestdir
+if [[ ! -z "${CLEANTEMPEST}" ]]; then
+  rm -rf $TEMPESTDIR
+fi
+
 
 ! (( $exit_1 || $exit_2 || $exit_3 ))
diff --git a/tox.ini b/tox.ini
index 9f552ead..29230a28 100644
--- a/tox.ini
+++ b/tox.ini
@@ -76,7 +76,7 @@ commands=
 
 [testenv:consistency]
 commands=
-  bash tools/consistency.sh
+  bash tools/consistency.sh {posargs}
 whitelist_externals =
   sh
   bash
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
new file mode 100644
index 00000000..6c737a0d
--- /dev/null
+++ b/zuul.d/jobs.yaml
@@ -0,0 +1,12 @@
+- job:
+    name: interop-tempest-consistency
+    parent: tox
+    description: |
+      Ensure consistency between Tempest source and current interop guidelines.
+    vars:
+      tox_envlist: consistency
+      tox_extra_args: "-vv -- {{ ansible_user_dir }}/{{ zuul.projects['git.openstack.org/openstack/tempest'].src_dir }}"
+      zuul_work_dir: src/git.openstack.org/openstack/interop
+    required-projects:
+      - name: openstack/tempest
+      - name: openstack/interop
diff --git a/.zuul.yaml b/zuul.d/projects.yaml
similarity index 68%
rename from .zuul.yaml
rename to zuul.d/projects.yaml
index 315a2eaf..4b419196 100644
--- a/.zuul.yaml
+++ b/zuul.d/projects.yaml
@@ -4,6 +4,8 @@
     check:
       jobs:
         - openstack-tox-pep8
+        - interop-tempest-consistency
     gate:
       jobs:
         - openstack-tox-pep8
+        - interop-tempest-consistency