From a5953d4a095dd76a83a5ea38abc37b43103e4694 Mon Sep 17 00:00:00 2001 From: Chris Hoge Date: Thu, 23 Aug 2018 08:09:53 -0700 Subject: [PATCH] Add tool and tests to check guideline against test suite This tool checks a guideline against a Tempest test suite to make sure that: * all idempotent IDs in the guideline exist in the tests * test names in the test suite match names in the guideline By gating against this tool in both the Interop project and in Tempest, we can help prevent typos and catch renaming of tests. Change-Id: I8b6778c2063eeac8b0d7c0d64127eefe420c617f --- tools/checktests.py | 195 +++++++++++++++++++++++++++++++++++++++++++ tools/consistency.sh | 19 +++++ tox.ini | 10 ++- 3 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 tools/checktests.py create mode 100755 tools/consistency.sh diff --git a/tools/checktests.py b/tools/checktests.py new file mode 100644 index 00000000..c82d3459 --- /dev/null +++ b/tools/checktests.py @@ -0,0 +1,195 @@ +# Copyright 2018, OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import ast +import argparse +import importlib +import json +import os +import re +import sys + + +def get_v1_version(guideline): + schema = guideline.get('schema') + if schema: + v1regex = re.compile("^1.*") + if v1regex.match(schema): + return "1" + return None + + +def get_v2_version(guideline): + metadata = guideline.get('metadata') + if metadata and isinstance(metadata, dict): + schema = metadata.get('schema') + if schema: + v2regex = re.compile("^2.*") + if v2regex.match(schema): + return "2" + return None + + +def get_guideline_version(guideline): + return get_v1_version(guideline) or get_v2_version(guideline) + + +def get_required_tests_v1(guideline): + capabilities = guideline.get('capabilities') + tests = [] + for capability_name in capabilities: + capability = capabilities.get(capability_name) + for test_name in capability.get('tests'): + test = capability.get('tests').get(test_name) + if not test.get('flagged'): + idempotent_id = test.get('idempotent_id') + test_set = {test_name} + if test.get('aliases'): + test_set = test_set.union(set(test.get('aliases'))) + tests.append((idempotent_id, test_set)) + return tests + + +def get_required_tests_v2(guideline): + return get_required_tests_v1(guideline) + + +def get_required_tests(guideline): + if get_v1_version(guideline): + return get_required_tests_v1(guideline) + elif get_v2_version(guideline): + return get_required_tests_v2(guideline) + return None + + +def load_guideline(guideline_file): + with open(guideline_file) as f: + guideline = json.load(f) + return guideline + + +def get_submodules(parent_module_name): + module = importlib.import_module(parent_module_name) + base_path = os.path.abspath(os.path.dirname(module.__file__)) + base_index = len(base_path.split('/')) - 1 + + submodules = {} + for root, dirnames, files in os.walk(base_path): + root_name = '.'.join(root.split('/')[base_index:]) + if not os.path.exists(os.path.join(root, '__init__.py')): + continue + for f in files: + if f.endswith('.py'): + module_path = root + '/' + f + module_name = root_name + '.' + os.path.splitext(f)[0] + if module_name not in submodules: + submodules[module_name] = module_path + return submodules + + +def get_tests(module_name): + submodules = get_submodules(module_name) + tests = {} + for module_name in submodules: + filename = submodules[module_name] + with open(filename, 'r') as f: + source = f.read() + parsed = ast.parse(source) + for node in parsed.body: + if node.__class__ is ast.ClassDef: + for classnode in node.body: + if (classnode.__class__ is ast.FunctionDef and + classnode.name.startswith('test_')): + for decorator in classnode.decorator_list: + if decorator.func.attr == 'idempotent_id': + tests['id-' + decorator.args[0].s] = \ + module_name + "." + node.name + "." + \ + classnode.name + return tests + + +def run(): + + parser = argparse.ArgumentParser() + parser.add_argument('--guideline', action='store', dest='guideline_file', + default='next.json', type=str, + help='The name of the guideline file to to check') + parser.add_argument('--testlib', action='store', dest='testlib', + default='tempest', type=str, + help='The test library, in the PYTHONPATH, to ' + 'check the consistency of the guideline ' + 'against') + + args = parser.parse_args() + + guideline = load_guideline(args.guideline_file) + required = get_required_tests(guideline) + tests = get_tests(args.testlib) + + missing_uuids = [] + missing_tests = {} + + for test in required: + uuid = test[0] + testnames = test[1] + if uuid not in tests: + missing_uuids.append(test) + else: + if tests[uuid] not in testnames: + missing_tests[uuid] = test + + exit_code = 0 + if len(missing_uuids) > 0: + exit_code = 1 + print "### Idempotent ID Errors Detected. To resolve these errors, " \ + "fix the uuid name (format id-) in the guideline to " \ + "match the id in the test suite:" + for test in missing_uuids: + print "Idempotent ID in guideline '%s' does not appear in test " \ + "library '%s'\n" \ + " idempotent_id:\n" \ + " %s\n" \ + " names: " % (args.guideline_file, args.testlib, test[0]) + for testname in test[1]: + print " %s" % (testname) + print "" + + if len(missing_tests) > 0: + exit_code = 1 + print "### Test Name Errors Detected. " \ + "To resolve these errors, update " \ + "the Interop guideline with the missing " \ + "test names:" + for uuid in missing_tests: + print "Test found in test library '%s'\n" \ + " idempotent_id:\n" \ + " %s\n" \ + " name:\n" \ + " %s\n" \ + "Entry in guideline '%s'\n" \ + " idempotent_id:\n" \ + " %s\n" \ + " names: " % (args.testlib, + uuid, tests[uuid], + args.guideline_file, + missing_tests[uuid][0]) + for testname in missing_tests[uuid][1]: + print " %s" % (testname) + print "" + + sys.exit(exit_code) + + +if __name__ == '__main__': + run() diff --git a/tools/consistency.sh b/tools/consistency.sh new file mode 100755 index 00000000..c623e4fa --- /dev/null +++ b/tools/consistency.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +tempestdir=$(mktemp -d) +echo $tempestdir +git clone git://git.openstack.org/openstack/tempest $tempestdir + + +PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline next.json +exit_1=$? + +PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline 2018.02.json +exit_2=$? + +PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline 2017.09.json +exit_3=$? + +rm -rf $tempestdir + +! (( $exit_1 || $exit_2 || $exit_3 )) diff --git a/tox.ini b/tox.ini index 58e715a9..fdbd9d6d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = docs,doc8,pep8,jsonlint,jsonschema +envlist = docs,doc8,pep8,jsonlint,jsonschema,consistency minversion = 1.6 skipsdist = True @@ -73,3 +73,11 @@ commands= jsonschema doc/source/schema/2.0.json -i add-ons/orchestration.next.json jsonschema doc/source/schema/2.0.json -i add-ons/dns.2018.02.json jsonschema doc/source/schema/2.0.json -i add-ons/orchestration.2018.02.json + +[testenv:consistency] +commands= + bash tools/consistency.sh +whitelist_externals = + sh + bash +