diff --git a/os_testr/ostestr.py b/os_testr/ostestr.py index 17ed1bd..9db9d3b 100755 --- a/os_testr/ostestr.py +++ b/os_testr/ostestr.py @@ -14,16 +14,19 @@ # under the License. import argparse +import atexit import copy import os import subprocess import sys +import tempfile import pbr.version from subunit import run as subunit_run from testtools import run as testtools_run from os_testr import regex_builder as rb +from os_testr import testlist_builder as tlb __version__ = pbr.version.VersionInfo('os_testr').version_string() @@ -98,7 +101,7 @@ def get_parser(args): def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur, - until_failure, color, others=None): + until_failure, color, list_of_tests=None, others=None): others = others or [] if parallel: cmd = ['testr', 'run', '--parallel'] @@ -112,7 +115,16 @@ def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur, cmd.append('--subunit') elif not (subunit or pretty) and until_failure: cmd.append('--until-failure') - cmd.append(regex) + if list_of_tests: + test_fd, test_file_name = tempfile.mkstemp() + atexit.register(os.remove, test_file_name) + test_file = os.fdopen(test_fd, 'w') + test_file.write('\n'.join(list_of_tests) + '\n') + test_file.close() + cmd.extend(('--load-list', test_file_name)) + else: + cmd.append(regex) + env = copy.deepcopy(os.environ) if pretty: @@ -194,15 +206,19 @@ def call_subunit_run(test_id, pretty, subunit): testtools_run.main([sys.argv[0], test_id], sys.stdout) -def _select_and_call_runner(opts, exclude_regex, others): - ec = 1 +def _ensure_testr(): if not os.path.isdir('.testrepository'): subprocess.call(['testr', 'init']) + +def _select_and_call_runner(opts, exclude_regex, others): + ec = 1 + _ensure_testr() + if not opts.no_discover and not opts.pdb: ec = call_testr(exclude_regex, opts.subunit, opts.pretty, opts.list, opts.slowest, opts.parallel, opts.concurrency, - opts.until_failure, opts.color, others) + opts.until_failure, opts.color, None, others) else: if others: print('Unexpected arguments: ' + ' '.join(others)) @@ -214,6 +230,20 @@ def _select_and_call_runner(opts, exclude_regex, others): return ec +def _call_testr_with_list(opts, test_list, others): + ec = 1 + _ensure_testr() + + if opts.list: + print("\n".join(test_list)) + return 0 + + ec = call_testr(None, opts.subunit, opts.pretty, opts.list, + opts.slowest, opts.parallel, opts.concurrency, + opts.until_failure, opts.color, test_list, others) + return ec + + def main(): opts, others = get_parser(sys.argv[1:]) if opts.pretty and opts.subunit: @@ -238,11 +268,21 @@ def main(): regex = rb.path_to_regex(opts.path) else: regex = opts.regex - exclude_regex = rb.construct_regex(opts.blacklist_file, - opts.whitelist_file, - regex, - opts.print_exclude) - exit(_select_and_call_runner(opts, exclude_regex, others)) + + if opts.regex and opts.blacklist_file: + # NOTE(afazekas): Now just the minority of the cases is handled + # by the testlist_builder, it can be changed in the future. + list_of_tests = tlb.construct_list(opts.blacklist_file, + opts.whitelist_file, + regex, + opts.print_exclude) + exit(_call_testr_with_list(opts, list_of_tests, others)) + else: + exclude_regex = rb.construct_regex(opts.blacklist_file, + opts.whitelist_file, + regex, + opts.print_exclude) + exit(_select_and_call_runner(opts, exclude_regex, others)) if __name__ == '__main__': main() diff --git a/os_testr/testlist_builder.py b/os_testr/testlist_builder.py new file mode 100644 index 0000000..d49fbfd --- /dev/null +++ b/os_testr/testlist_builder.py @@ -0,0 +1,95 @@ +# Copyright 2016 RedHat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from os_testr import regex_builder +import re + + +def black_reader(blacklist_file): + black_file = open(blacklist_file, 'r') + regex_comment_lst = [] # tupple of (regex_compild, msg, skipped_lst) + for line in black_file: + raw_line = line.strip() + split_line = raw_line.split('#') + # Before the # is the regex + line_regex = split_line[0].strip() + if len(split_line) > 1: + # After the # is a comment + comment = ''.join(split_line[1:]).strip() + else: + comment = 'Skipped because of regex %s:' % line_regex + if not line_regex: + continue + regex_comment_lst.append((re.compile(line_regex), comment, [])) + return regex_comment_lst + + +def print_skips(regex, message, test_list): + for test in test_list: + print(test) + # Extra whitespace to separate + print('\n') + + +def construct_list(blacklist_file, whitelist_file, regex, print_exclude): + """Filters the discovered test cases + + :retrun: iterable of strings. The strings are full + test cases names, including tags like.: + "project.api.TestClass.test_case[positive]" + """ + + if not regex: + regex = '' # handle the other false things + + if whitelist_file: + white_re = regex_builder.get_regex_from_whitelist_file(whitelist_file) + else: + white_re = '' + + if not regex and white_re: + regex = white_re + elif regex and white_re: + regex = '|'.join((regex, white_re)) + + if blacklist_file: + black_data = black_reader(blacklist_file) + else: + black_data = None + + search_filter = re.compile(regex) + + # NOTE(afazekas): we do not want to pass a giant re + # to an external application due to the arg length limitatios + list_of_test_cases = [test_case for test_case in + regex_builder._get_test_list('') + if search_filter.search(test_case)] + set_of_test_cases = set(list_of_test_cases) + + if not black_data: + return set_of_test_cases + + # NOTE(afazekas): We might use a faster logic when the + # print option is not requested + for (rex, msg, s_list) in black_data: + for test_case in list_of_test_cases: + if rex.search(test_case): + set_of_test_cases.remove(test_case) + s_list.append(test_case) + + if print_exclude: + for (rex, msg, s_list) in black_data: + if s_list: + print_skips(rex, msg, s_list) + return set_of_test_cases diff --git a/os_testr/tests/testlist_builder.py b/os_testr/tests/testlist_builder.py new file mode 100644 index 0000000..3f0b6c0 --- /dev/null +++ b/os_testr/tests/testlist_builder.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import re + +import six + +from os_testr import testlist_builder as list_builder +from os_testr.tests import base + + +class TestBlackReader(base.TestCase): + def test_black_reader(self): + blacklist_file = six.StringIO() + for i in range(4): + blacklist_file.write('fake_regex_%s\n' % i) + blacklist_file.write('fake_regex_with_note_%s # note\n' % i) + blacklist_file.seek(0) + with mock.patch('six.moves.builtins.open', + return_value=blacklist_file): + result = list_builder.black_reader('fake_path') + self.assertEqual(2 * 4, len(result)) + note_cnt = 0 + # not assuming ordering, mainly just testing the type + for r in result: + self.assertEqual(r[2], []) + if r[1] == 'note': + note_cnt += 1 + self.assertIn('search', dir(r[0])) # like a compiled regexp + self.assertEqual(note_cnt, 4) + + +class TestConstructList(base.TestCase): + def test_simple_re(self): + test_lists = ['fake_test(scen)[tag,bar])', 'fake_test(scen)[egg,foo])'] + with mock.patch('os_testr.regex_builder._get_test_list', + return_value=test_lists): + result = list_builder.construct_list(None, None, 'foo', False) + self.assertEqual(list(result), ['fake_test(scen)[egg,foo])']) + + def test_blacklist(self): + black_list = [(re.compile('foo'), 'foo not liked', [])] + test_lists = ['fake_test(scen)[tag,bar])', 'fake_test(scen)[egg,foo])'] + with mock.patch('os_testr.regex_builder._get_test_list', + return_value=test_lists): + with mock.patch('os_testr.testlist_builder.black_reader', + return_value=black_list): + result = list_builder.construct_list('file', + None, + 'fake_test', + False) + self.assertEqual(list(result), ['fake_test(scen)[tag,bar])']) + + def test_whitelist(self): + white_list = 'fake_test1|fake_test2' + test_lists = ['fake_test1[tg]', 'fake_test2[tg]', 'fake_test3[tg]'] + white_getter = 'os_testr.regex_builder.get_regex_from_whitelist_file' + with mock.patch('os_testr.regex_builder._get_test_list', + return_value=test_lists): + with mock.patch(white_getter, + return_value=white_list): + result = list_builder.construct_list(None, + 'file', + None, + False) + self.assertEqual(set(result), + set(('fake_test1[tg]', 'fake_test2[tg]')))