From a3b403bd4e0ae54c63c85dec9379fa92471cd73c Mon Sep 17 00:00:00 2001 From: Attila Fazekas Date: Fri, 29 Jul 2016 14:56:30 +0200 Subject: [PATCH] Construct a list of test cases instead of passing a regexp The way how we handled the regular expressions had a lot of limitation. - We are not able to pass huge arguments to testr, it makes difficult to have long list if accepted and/or rejected test cases, but we can pass a path to a file of test cases, which can be arbitrary big. - How we wanted to handle the backlists before was not worked together with the regular selecting regex because it consumed the pattern. Now the blacklisting happens in a separated phase after the selecting regex search. This change just allows the new code path to run when both a blacklist_file and a selecting regexp specified. The new way depends on the usual test discovery and just filters the output of the discovery command, this strategy can be the default in the future, now I just wanted to preserve the old behavior as much as possible in all the other cases. Change-Id: Ie8e5928e286d0c9076c4eee23319149c9869a6fa Closes-Bug: #1506215 --- os_testr/ostestr.py | 60 +++++++++++++++---- os_testr/testlist_builder.py | 95 ++++++++++++++++++++++++++++++ os_testr/tests/testlist_builder.py | 79 +++++++++++++++++++++++++ 3 files changed, 224 insertions(+), 10 deletions(-) create mode 100644 os_testr/testlist_builder.py create mode 100644 os_testr/tests/testlist_builder.py diff --git a/os_testr/ostestr.py b/os_testr/ostestr.py index 17ed1bd..9db9d3b 100755 --- a/os_testr/ostestr.py +++ b/os_testr/ostestr.py @@ -14,16 +14,19 @@ # under the License. import argparse +import atexit import copy import os import subprocess import sys +import tempfile import pbr.version from subunit import run as subunit_run from testtools import run as testtools_run from os_testr import regex_builder as rb +from os_testr import testlist_builder as tlb __version__ = pbr.version.VersionInfo('os_testr').version_string() @@ -98,7 +101,7 @@ def get_parser(args): def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur, - until_failure, color, others=None): + until_failure, color, list_of_tests=None, others=None): others = others or [] if parallel: cmd = ['testr', 'run', '--parallel'] @@ -112,7 +115,16 @@ def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur, cmd.append('--subunit') elif not (subunit or pretty) and until_failure: cmd.append('--until-failure') - cmd.append(regex) + if list_of_tests: + test_fd, test_file_name = tempfile.mkstemp() + atexit.register(os.remove, test_file_name) + test_file = os.fdopen(test_fd, 'w') + test_file.write('\n'.join(list_of_tests) + '\n') + test_file.close() + cmd.extend(('--load-list', test_file_name)) + else: + cmd.append(regex) + env = copy.deepcopy(os.environ) if pretty: @@ -194,15 +206,19 @@ def call_subunit_run(test_id, pretty, subunit): testtools_run.main([sys.argv[0], test_id], sys.stdout) -def _select_and_call_runner(opts, exclude_regex, others): - ec = 1 +def _ensure_testr(): if not os.path.isdir('.testrepository'): subprocess.call(['testr', 'init']) + +def _select_and_call_runner(opts, exclude_regex, others): + ec = 1 + _ensure_testr() + if not opts.no_discover and not opts.pdb: ec = call_testr(exclude_regex, opts.subunit, opts.pretty, opts.list, opts.slowest, opts.parallel, opts.concurrency, - opts.until_failure, opts.color, others) + opts.until_failure, opts.color, None, others) else: if others: print('Unexpected arguments: ' + ' '.join(others)) @@ -214,6 +230,20 @@ def _select_and_call_runner(opts, exclude_regex, others): return ec +def _call_testr_with_list(opts, test_list, others): + ec = 1 + _ensure_testr() + + if opts.list: + print("\n".join(test_list)) + return 0 + + ec = call_testr(None, opts.subunit, opts.pretty, opts.list, + opts.slowest, opts.parallel, opts.concurrency, + opts.until_failure, opts.color, test_list, others) + return ec + + def main(): opts, others = get_parser(sys.argv[1:]) if opts.pretty and opts.subunit: @@ -238,11 +268,21 @@ def main(): regex = rb.path_to_regex(opts.path) else: regex = opts.regex - exclude_regex = rb.construct_regex(opts.blacklist_file, - opts.whitelist_file, - regex, - opts.print_exclude) - exit(_select_and_call_runner(opts, exclude_regex, others)) + + if opts.regex and opts.blacklist_file: + # NOTE(afazekas): Now just the minority of the cases is handled + # by the testlist_builder, it can be changed in the future. + list_of_tests = tlb.construct_list(opts.blacklist_file, + opts.whitelist_file, + regex, + opts.print_exclude) + exit(_call_testr_with_list(opts, list_of_tests, others)) + else: + exclude_regex = rb.construct_regex(opts.blacklist_file, + opts.whitelist_file, + regex, + opts.print_exclude) + exit(_select_and_call_runner(opts, exclude_regex, others)) if __name__ == '__main__': main() diff --git a/os_testr/testlist_builder.py b/os_testr/testlist_builder.py new file mode 100644 index 0000000..d49fbfd --- /dev/null +++ b/os_testr/testlist_builder.py @@ -0,0 +1,95 @@ +# Copyright 2016 RedHat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from os_testr import regex_builder +import re + + +def black_reader(blacklist_file): + black_file = open(blacklist_file, 'r') + regex_comment_lst = [] # tupple of (regex_compild, msg, skipped_lst) + for line in black_file: + raw_line = line.strip() + split_line = raw_line.split('#') + # Before the # is the regex + line_regex = split_line[0].strip() + if len(split_line) > 1: + # After the # is a comment + comment = ''.join(split_line[1:]).strip() + else: + comment = 'Skipped because of regex %s:' % line_regex + if not line_regex: + continue + regex_comment_lst.append((re.compile(line_regex), comment, [])) + return regex_comment_lst + + +def print_skips(regex, message, test_list): + for test in test_list: + print(test) + # Extra whitespace to separate + print('\n') + + +def construct_list(blacklist_file, whitelist_file, regex, print_exclude): + """Filters the discovered test cases + + :retrun: iterable of strings. The strings are full + test cases names, including tags like.: + "project.api.TestClass.test_case[positive]" + """ + + if not regex: + regex = '' # handle the other false things + + if whitelist_file: + white_re = regex_builder.get_regex_from_whitelist_file(whitelist_file) + else: + white_re = '' + + if not regex and white_re: + regex = white_re + elif regex and white_re: + regex = '|'.join((regex, white_re)) + + if blacklist_file: + black_data = black_reader(blacklist_file) + else: + black_data = None + + search_filter = re.compile(regex) + + # NOTE(afazekas): we do not want to pass a giant re + # to an external application due to the arg length limitatios + list_of_test_cases = [test_case for test_case in + regex_builder._get_test_list('') + if search_filter.search(test_case)] + set_of_test_cases = set(list_of_test_cases) + + if not black_data: + return set_of_test_cases + + # NOTE(afazekas): We might use a faster logic when the + # print option is not requested + for (rex, msg, s_list) in black_data: + for test_case in list_of_test_cases: + if rex.search(test_case): + set_of_test_cases.remove(test_case) + s_list.append(test_case) + + if print_exclude: + for (rex, msg, s_list) in black_data: + if s_list: + print_skips(rex, msg, s_list) + return set_of_test_cases diff --git a/os_testr/tests/testlist_builder.py b/os_testr/tests/testlist_builder.py new file mode 100644 index 0000000..3f0b6c0 --- /dev/null +++ b/os_testr/tests/testlist_builder.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import re + +import six + +from os_testr import testlist_builder as list_builder +from os_testr.tests import base + + +class TestBlackReader(base.TestCase): + def test_black_reader(self): + blacklist_file = six.StringIO() + for i in range(4): + blacklist_file.write('fake_regex_%s\n' % i) + blacklist_file.write('fake_regex_with_note_%s # note\n' % i) + blacklist_file.seek(0) + with mock.patch('six.moves.builtins.open', + return_value=blacklist_file): + result = list_builder.black_reader('fake_path') + self.assertEqual(2 * 4, len(result)) + note_cnt = 0 + # not assuming ordering, mainly just testing the type + for r in result: + self.assertEqual(r[2], []) + if r[1] == 'note': + note_cnt += 1 + self.assertIn('search', dir(r[0])) # like a compiled regexp + self.assertEqual(note_cnt, 4) + + +class TestConstructList(base.TestCase): + def test_simple_re(self): + test_lists = ['fake_test(scen)[tag,bar])', 'fake_test(scen)[egg,foo])'] + with mock.patch('os_testr.regex_builder._get_test_list', + return_value=test_lists): + result = list_builder.construct_list(None, None, 'foo', False) + self.assertEqual(list(result), ['fake_test(scen)[egg,foo])']) + + def test_blacklist(self): + black_list = [(re.compile('foo'), 'foo not liked', [])] + test_lists = ['fake_test(scen)[tag,bar])', 'fake_test(scen)[egg,foo])'] + with mock.patch('os_testr.regex_builder._get_test_list', + return_value=test_lists): + with mock.patch('os_testr.testlist_builder.black_reader', + return_value=black_list): + result = list_builder.construct_list('file', + None, + 'fake_test', + False) + self.assertEqual(list(result), ['fake_test(scen)[tag,bar])']) + + def test_whitelist(self): + white_list = 'fake_test1|fake_test2' + test_lists = ['fake_test1[tg]', 'fake_test2[tg]', 'fake_test3[tg]'] + white_getter = 'os_testr.regex_builder.get_regex_from_whitelist_file' + with mock.patch('os_testr.regex_builder._get_test_list', + return_value=test_lists): + with mock.patch(white_getter, + return_value=white_list): + result = list_builder.construct_list(None, + 'file', + None, + False) + self.assertEqual(set(result), + set(('fake_test1[tg]', 'fake_test2[tg]')))