Merge "Add tool and tests to check guideline against test suite"

This commit is contained in:
Zuul 2018-08-27 17:36:42 +00:00 committed by Gerrit Code Review
commit 6ad5a13efb
3 changed files with 223 additions and 1 deletions

195
tools/checktests.py Normal file
View File

@ -0,0 +1,195 @@
# Copyright 2018, OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import ast
import argparse
import importlib
import json
import os
import re
import sys
def get_v1_version(guideline):
schema = guideline.get('schema')
if schema:
v1regex = re.compile("^1.*")
if v1regex.match(schema):
return "1"
return None
def get_v2_version(guideline):
metadata = guideline.get('metadata')
if metadata and isinstance(metadata, dict):
schema = metadata.get('schema')
if schema:
v2regex = re.compile("^2.*")
if v2regex.match(schema):
return "2"
return None
def get_guideline_version(guideline):
return get_v1_version(guideline) or get_v2_version(guideline)
def get_required_tests_v1(guideline):
capabilities = guideline.get('capabilities')
tests = []
for capability_name in capabilities:
capability = capabilities.get(capability_name)
for test_name in capability.get('tests'):
test = capability.get('tests').get(test_name)
if not test.get('flagged'):
idempotent_id = test.get('idempotent_id')
test_set = {test_name}
if test.get('aliases'):
test_set = test_set.union(set(test.get('aliases')))
tests.append((idempotent_id, test_set))
return tests
def get_required_tests_v2(guideline):
return get_required_tests_v1(guideline)
def get_required_tests(guideline):
if get_v1_version(guideline):
return get_required_tests_v1(guideline)
elif get_v2_version(guideline):
return get_required_tests_v2(guideline)
return None
def load_guideline(guideline_file):
with open(guideline_file) as f:
guideline = json.load(f)
return guideline
def get_submodules(parent_module_name):
module = importlib.import_module(parent_module_name)
base_path = os.path.abspath(os.path.dirname(module.__file__))
base_index = len(base_path.split('/')) - 1
submodules = {}
for root, dirnames, files in os.walk(base_path):
root_name = '.'.join(root.split('/')[base_index:])
if not os.path.exists(os.path.join(root, '__init__.py')):
continue
for f in files:
if f.endswith('.py'):
module_path = root + '/' + f
module_name = root_name + '.' + os.path.splitext(f)[0]
if module_name not in submodules:
submodules[module_name] = module_path
return submodules
def get_tests(module_name):
submodules = get_submodules(module_name)
tests = {}
for module_name in submodules:
filename = submodules[module_name]
with open(filename, 'r') as f:
source = f.read()
parsed = ast.parse(source)
for node in parsed.body:
if node.__class__ is ast.ClassDef:
for classnode in node.body:
if (classnode.__class__ is ast.FunctionDef and
classnode.name.startswith('test_')):
for decorator in classnode.decorator_list:
if decorator.func.attr == 'idempotent_id':
tests['id-' + decorator.args[0].s] = \
module_name + "." + node.name + "." + \
classnode.name
return tests
def run():
parser = argparse.ArgumentParser()
parser.add_argument('--guideline', action='store', dest='guideline_file',
default='next.json', type=str,
help='The name of the guideline file to to check')
parser.add_argument('--testlib', action='store', dest='testlib',
default='tempest', type=str,
help='The test library, in the PYTHONPATH, to '
'check the consistency of the guideline '
'against')
args = parser.parse_args()
guideline = load_guideline(args.guideline_file)
required = get_required_tests(guideline)
tests = get_tests(args.testlib)
missing_uuids = []
missing_tests = {}
for test in required:
uuid = test[0]
testnames = test[1]
if uuid not in tests:
missing_uuids.append(test)
else:
if tests[uuid] not in testnames:
missing_tests[uuid] = test
exit_code = 0
if len(missing_uuids) > 0:
exit_code = 1
print "### Idempotent ID Errors Detected. To resolve these errors, " \
"fix the uuid name (format id-<uuid>) in the guideline to " \
"match the id in the test suite:"
for test in missing_uuids:
print "Idempotent ID in guideline '%s' does not appear in test " \
"library '%s'\n" \
" idempotent_id:\n" \
" %s\n" \
" names: " % (args.guideline_file, args.testlib, test[0])
for testname in test[1]:
print " %s" % (testname)
print ""
if len(missing_tests) > 0:
exit_code = 1
print "### Test Name Errors Detected. " \
"To resolve these errors, update " \
"the Interop guideline with the missing " \
"test names:"
for uuid in missing_tests:
print "Test found in test library '%s'\n" \
" idempotent_id:\n" \
" %s\n" \
" name:\n" \
" %s\n" \
"Entry in guideline '%s'\n" \
" idempotent_id:\n" \
" %s\n" \
" names: " % (args.testlib,
uuid, tests[uuid],
args.guideline_file,
missing_tests[uuid][0])
for testname in missing_tests[uuid][1]:
print " %s" % (testname)
print ""
sys.exit(exit_code)
if __name__ == '__main__':
run()

19
tools/consistency.sh Executable file
View File

@ -0,0 +1,19 @@
#!/bin/bash
tempestdir=$(mktemp -d)
echo $tempestdir
git clone git://git.openstack.org/openstack/tempest $tempestdir
PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline next.json
exit_1=$?
PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline 2018.02.json
exit_2=$?
PYTHONPATH=$tempestdir python ./tools/checktests.py --guideline 2017.09.json
exit_3=$?
rm -rf $tempestdir
! (( $exit_1 || $exit_2 || $exit_3 ))

10
tox.ini
View File

@ -1,5 +1,5 @@
[tox]
envlist = docs,doc8,pep8,jsonlint,jsonschema
envlist = docs,doc8,pep8,jsonlint,jsonschema,consistency
minversion = 1.6
skipsdist = True
@ -73,3 +73,11 @@ commands=
jsonschema doc/source/schema/2.0.json -i add-ons/orchestration.next.json
jsonschema doc/source/schema/2.0.json -i add-ons/dns.2018.02.json
jsonschema doc/source/schema/2.0.json -i add-ons/orchestration.2018.02.json
[testenv:consistency]
commands=
bash tools/consistency.sh
whitelist_externals =
sh
bash