Construct a list of test cases instead of passing a regexp

The way how we handled the regular expressions had a lot of
limitation.

 - We are not able to pass huge arguments to testr, it makes
   difficult to have long list if accepted and/or rejected test cases,
   but we can pass a path to a file of test cases,
   which can be arbitrary big.
 - How we wanted to handle the backlists before was not worked together
   with the regular selecting regex because it consumed the pattern.
   Now the blacklisting happens in a separated phase after the selecting
   regex search.

This change just allows the new code path to run when both
 a blacklist_file and a selecting regexp specified.

The new way depends on the usual test discovery and just
filters the output of the discovery command,
this strategy can be the default in the future, now I just
wanted to preserve the old behavior as much as possible in
all the other cases.

Change-Id: Ie8e5928e286d0c9076c4eee23319149c9869a6fa
Closes-Bug: #1506215
This commit is contained in:
Attila Fazekas 2016-07-29 14:56:30 +02:00
parent 6afbb43fce
commit a3b403bd4e
3 changed files with 224 additions and 10 deletions

View File

@ -14,16 +14,19 @@
# under the License.
import argparse
import atexit
import copy
import os
import subprocess
import sys
import tempfile
import pbr.version
from subunit import run as subunit_run
from testtools import run as testtools_run
from os_testr import regex_builder as rb
from os_testr import testlist_builder as tlb
__version__ = pbr.version.VersionInfo('os_testr').version_string()
@ -98,7 +101,7 @@ def get_parser(args):
def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur,
until_failure, color, others=None):
until_failure, color, list_of_tests=None, others=None):
others = others or []
if parallel:
cmd = ['testr', 'run', '--parallel']
@ -112,7 +115,16 @@ def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur,
cmd.append('--subunit')
elif not (subunit or pretty) and until_failure:
cmd.append('--until-failure')
cmd.append(regex)
if list_of_tests:
test_fd, test_file_name = tempfile.mkstemp()
atexit.register(os.remove, test_file_name)
test_file = os.fdopen(test_fd, 'w')
test_file.write('\n'.join(list_of_tests) + '\n')
test_file.close()
cmd.extend(('--load-list', test_file_name))
else:
cmd.append(regex)
env = copy.deepcopy(os.environ)
if pretty:
@ -194,15 +206,19 @@ def call_subunit_run(test_id, pretty, subunit):
testtools_run.main([sys.argv[0], test_id], sys.stdout)
def _select_and_call_runner(opts, exclude_regex, others):
ec = 1
def _ensure_testr():
if not os.path.isdir('.testrepository'):
subprocess.call(['testr', 'init'])
def _select_and_call_runner(opts, exclude_regex, others):
ec = 1
_ensure_testr()
if not opts.no_discover and not opts.pdb:
ec = call_testr(exclude_regex, opts.subunit, opts.pretty, opts.list,
opts.slowest, opts.parallel, opts.concurrency,
opts.until_failure, opts.color, others)
opts.until_failure, opts.color, None, others)
else:
if others:
print('Unexpected arguments: ' + ' '.join(others))
@ -214,6 +230,20 @@ def _select_and_call_runner(opts, exclude_regex, others):
return ec
def _call_testr_with_list(opts, test_list, others):
ec = 1
_ensure_testr()
if opts.list:
print("\n".join(test_list))
return 0
ec = call_testr(None, opts.subunit, opts.pretty, opts.list,
opts.slowest, opts.parallel, opts.concurrency,
opts.until_failure, opts.color, test_list, others)
return ec
def main():
opts, others = get_parser(sys.argv[1:])
if opts.pretty and opts.subunit:
@ -238,11 +268,21 @@ def main():
regex = rb.path_to_regex(opts.path)
else:
regex = opts.regex
exclude_regex = rb.construct_regex(opts.blacklist_file,
opts.whitelist_file,
regex,
opts.print_exclude)
exit(_select_and_call_runner(opts, exclude_regex, others))
if opts.regex and opts.blacklist_file:
# NOTE(afazekas): Now just the minority of the cases is handled
# by the testlist_builder, it can be changed in the future.
list_of_tests = tlb.construct_list(opts.blacklist_file,
opts.whitelist_file,
regex,
opts.print_exclude)
exit(_call_testr_with_list(opts, list_of_tests, others))
else:
exclude_regex = rb.construct_regex(opts.blacklist_file,
opts.whitelist_file,
regex,
opts.print_exclude)
exit(_select_and_call_runner(opts, exclude_regex, others))
if __name__ == '__main__':
main()

View File

@ -0,0 +1,95 @@
# Copyright 2016 RedHat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from os_testr import regex_builder
import re
def black_reader(blacklist_file):
black_file = open(blacklist_file, 'r')
regex_comment_lst = [] # tupple of (regex_compild, msg, skipped_lst)
for line in black_file:
raw_line = line.strip()
split_line = raw_line.split('#')
# Before the # is the regex
line_regex = split_line[0].strip()
if len(split_line) > 1:
# After the # is a comment
comment = ''.join(split_line[1:]).strip()
else:
comment = 'Skipped because of regex %s:' % line_regex
if not line_regex:
continue
regex_comment_lst.append((re.compile(line_regex), comment, []))
return regex_comment_lst
def print_skips(regex, message, test_list):
for test in test_list:
print(test)
# Extra whitespace to separate
print('\n')
def construct_list(blacklist_file, whitelist_file, regex, print_exclude):
"""Filters the discovered test cases
:retrun: iterable of strings. The strings are full
test cases names, including tags like.:
"project.api.TestClass.test_case[positive]"
"""
if not regex:
regex = '' # handle the other false things
if whitelist_file:
white_re = regex_builder.get_regex_from_whitelist_file(whitelist_file)
else:
white_re = ''
if not regex and white_re:
regex = white_re
elif regex and white_re:
regex = '|'.join((regex, white_re))
if blacklist_file:
black_data = black_reader(blacklist_file)
else:
black_data = None
search_filter = re.compile(regex)
# NOTE(afazekas): we do not want to pass a giant re
# to an external application due to the arg length limitatios
list_of_test_cases = [test_case for test_case in
regex_builder._get_test_list('')
if search_filter.search(test_case)]
set_of_test_cases = set(list_of_test_cases)
if not black_data:
return set_of_test_cases
# NOTE(afazekas): We might use a faster logic when the
# print option is not requested
for (rex, msg, s_list) in black_data:
for test_case in list_of_test_cases:
if rex.search(test_case):
set_of_test_cases.remove(test_case)
s_list.append(test_case)
if print_exclude:
for (rex, msg, s_list) in black_data:
if s_list:
print_skips(rex, msg, s_list)
return set_of_test_cases

View File

@ -0,0 +1,79 @@
# -*- coding: utf-8 -*-
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
import re
import six
from os_testr import testlist_builder as list_builder
from os_testr.tests import base
class TestBlackReader(base.TestCase):
def test_black_reader(self):
blacklist_file = six.StringIO()
for i in range(4):
blacklist_file.write('fake_regex_%s\n' % i)
blacklist_file.write('fake_regex_with_note_%s # note\n' % i)
blacklist_file.seek(0)
with mock.patch('six.moves.builtins.open',
return_value=blacklist_file):
result = list_builder.black_reader('fake_path')
self.assertEqual(2 * 4, len(result))
note_cnt = 0
# not assuming ordering, mainly just testing the type
for r in result:
self.assertEqual(r[2], [])
if r[1] == 'note':
note_cnt += 1
self.assertIn('search', dir(r[0])) # like a compiled regexp
self.assertEqual(note_cnt, 4)
class TestConstructList(base.TestCase):
def test_simple_re(self):
test_lists = ['fake_test(scen)[tag,bar])', 'fake_test(scen)[egg,foo])']
with mock.patch('os_testr.regex_builder._get_test_list',
return_value=test_lists):
result = list_builder.construct_list(None, None, 'foo', False)
self.assertEqual(list(result), ['fake_test(scen)[egg,foo])'])
def test_blacklist(self):
black_list = [(re.compile('foo'), 'foo not liked', [])]
test_lists = ['fake_test(scen)[tag,bar])', 'fake_test(scen)[egg,foo])']
with mock.patch('os_testr.regex_builder._get_test_list',
return_value=test_lists):
with mock.patch('os_testr.testlist_builder.black_reader',
return_value=black_list):
result = list_builder.construct_list('file',
None,
'fake_test',
False)
self.assertEqual(list(result), ['fake_test(scen)[tag,bar])'])
def test_whitelist(self):
white_list = 'fake_test1|fake_test2'
test_lists = ['fake_test1[tg]', 'fake_test2[tg]', 'fake_test3[tg]']
white_getter = 'os_testr.regex_builder.get_regex_from_whitelist_file'
with mock.patch('os_testr.regex_builder._get_test_list',
return_value=test_lists):
with mock.patch(white_getter,
return_value=white_list):
result = list_builder.construct_list(None,
'file',
None,
False)
self.assertEqual(set(result),
set(('fake_test1[tg]', 'fake_test2[tg]')))