From 88269b5d5654f4af9026ebba73d1f6434d68fb68 Mon Sep 17 00:00:00 2001 From: Luigi Toscano Date: Tue, 19 Apr 2016 12:34:57 +0200 Subject: [PATCH] Run the tests inside the test directory This change allows the test runner to be executed everywhere without having a .testr.conf file. A copy of .testr.conf file (which is now properly released and installed) is copied to the test directory, which is now used as working directory for the ostestr call. The path to the directory where the execution has been started is now passed to the test template, so that the code can write there data like the test reports, the key (if the resources are kept), etc. Change-Id: I16ddba2ad6e26381ffc4b447666f38f2a7740ecb --- ...fy-testrunner-config-d3a19014db107ff1.yaml | 7 ++++ sahara_tests/scenario/base.py | 8 +++- sahara_tests/scenario/runner.py | 15 +++++-- sahara_tests/scenario/testcase.py.mako | 1 + sahara_tests/scenario/testr.conf | 8 ++++ sahara_tests/unit/scenario/test_runner.py | 39 +++++++++++++------ 6 files changed, 62 insertions(+), 16 deletions(-) create mode 100644 releasenotes/notes/simplify-testrunner-config-d3a19014db107ff1.yaml create mode 100644 sahara_tests/scenario/testr.conf diff --git a/releasenotes/notes/simplify-testrunner-config-d3a19014db107ff1.yaml b/releasenotes/notes/simplify-testrunner-config-d3a19014db107ff1.yaml new file mode 100644 index 00000000..9a9981b3 --- /dev/null +++ b/releasenotes/notes/simplify-testrunner-config-d3a19014db107ff1.yaml @@ -0,0 +1,7 @@ +--- +prelude: > + Removed the need of a .testr.conf file when calling the + test runner. +fixes: + - A .testr.conf file was previously required in the runner + execution directory, now this is handled internally. diff --git a/sahara_tests/scenario/base.py b/sahara_tests/scenario/base.py index cbad9128..6b13c435 100644 --- a/sahara_tests/scenario/base.py +++ b/sahara_tests/scenario/base.py @@ -80,6 +80,7 @@ class BaseTestCase(base.BaseTestCase): cls.testcase = None cls._results = [] cls.report = False + cls.results_dir = '.' cls.default_templ_dir = '.' def setUp(self): @@ -96,7 +97,9 @@ class BaseTestCase(base.BaseTestCase): # save the private key if retain_resources is specified # (useful for debugging purposes) if self.testcase['retain_resources'] or self.key is None: - with open(self.key_name + '.key', 'a') as private_key_file: + private_key_file = os.path.join(self.results_dir, self.key_name + + '.key') + with open(private_key_file, 'a') as private_key_file: private_key_file.write(self.private_key) self.plugin_opts = { 'plugin_name': self.testcase['plugin_name'], @@ -738,7 +741,8 @@ class BaseTestCase(base.BaseTestCase): filename = {"time": time.strftime('%Y%m%d%H%M%S', time.localtime())} filename.update(self._results) - report_file_name = ( + report_file_name = os.path.join( + self.results_dir, '{plugin_name}_{hadoop_version}-{time}'.format(**filename)) time.strftime('%Y%m%d%H%M%S', time.localtime()) with open(report_file_name, 'w') as report_file: diff --git a/sahara_tests/scenario/runner.py b/sahara_tests/scenario/runner.py index cec99494..ebbeec2e 100755 --- a/sahara_tests/scenario/runner.py +++ b/sahara_tests/scenario/runner.py @@ -19,6 +19,7 @@ from __future__ import print_function import argparse import collections import os +import shutil import subprocess import sys import tempfile @@ -43,6 +44,7 @@ DEFAULT_TEMPLATE_VARS = [os.path.join(TEST_TEMPLATE_DIR, 'edp.yaml.mako')] TEST_TEMPLATE_PATH = os.path.join(SCENARIO_RESOURCES_DIR, 'testcase.py.mako') +DEFAULT_TESTR_CONF = os.path.join(SCENARIO_RESOURCES_DIR, 'testr.conf') def set_defaults(config): @@ -315,6 +317,8 @@ def main(): testcases = config['clusters'] for case in range(count - 1): testcases.extend(config['clusters']) + # current directory, where to write reports, key files, etc, if required + results_dir = os.getcwd() default_templ_dir = os.path.abspath(TEST_TEMPLATE_DIR) # create testcase file @@ -322,21 +326,26 @@ def main(): testcase_data = test_template.render(testcases=testcases, credentials=credentials, network=network, report=report, + results_dir=results_dir, default_templ_dir=default_templ_dir) test_dir_path = tempfile.mkdtemp() print("The generated test file located at: %s" % test_dir_path) fileutils.write_to_tempfile(testcase_data.encode("ASCII"), prefix='test_', suffix='.py', path=test_dir_path) + shutil.copyfile(DEFAULT_TESTR_CONF, os.path.join(test_dir_path, + '.testr.conf')) # run tests concurrency = config.get('concurrency') - os.environ['DISCOVER_DIRECTORY'] = test_dir_path command = ['ostestr'] if concurrency: command.extend(['--concurrency', '%d' % concurrency]) - return_code = subprocess.call(command) - sys.exit(return_code) + new_env = os.environ.copy() + new_env['DISCOVER_DIRECTORY'] = '.' + tester_runner = subprocess.Popen(command, env=new_env, cwd=test_dir_path) + tester_runner.communicate() + sys.exit(tester_runner.returncode) if __name__ == '__main__': diff --git a/sahara_tests/scenario/testcase.py.mako b/sahara_tests/scenario/testcase.py.mako index 34f357d2..d953b07b 100644 --- a/sahara_tests/scenario/testcase.py.mako +++ b/sahara_tests/scenario/testcase.py.mako @@ -13,6 +13,7 @@ class ${testcase['class_name']}TestCase(base.BaseTestCase): cls.network = ${network} cls.testcase = ${testcase} cls.report = ${report} + cls.results_dir = '${results_dir}' cls.default_templ_dir = '${default_templ_dir}' def test_plugin(self): diff --git a/sahara_tests/scenario/testr.conf b/sahara_tests/scenario/testr.conf new file mode 100644 index 00000000..077512e8 --- /dev/null +++ b/sahara_tests/scenario/testr.conf @@ -0,0 +1,8 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ + OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ + OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ + ${PYTHON:-python} -m subunit.run discover $DISCOVER_DIRECTORY $LISTOPT $IDOPTION +test_id_option=--load-list $IDFILE +test_list_option=--list +group_regex=([^\.]+\.)+ diff --git a/sahara_tests/unit/scenario/test_runner.py b/sahara_tests/unit/scenario/test_runner.py index 1b303a8d..b04b0050 100644 --- a/sahara_tests/unit/scenario/test_runner.py +++ b/sahara_tests/unit/scenario/test_runner.py @@ -22,6 +22,12 @@ import testtools from sahara_tests.scenario import runner +def _create_subprocess_communicate_mock(): + communicate_mock = mock.Mock() + communicate_mock.communicate.return_value = ["", ""] + return communicate_mock + + class RunnerUnitTest(testtools.TestCase): def _isDictContainSubset(self, sub_dictionary, dictionary): @@ -244,21 +250,24 @@ class RunnerUnitTest(testtools.TestCase): expected_cluster, config)) @mock.patch('sys.exit', return_value=None) - @mock.patch('subprocess.call', return_value=None) + @mock.patch('subprocess.Popen', + return_value=_create_subprocess_communicate_mock()) def test_runner_main(self, mock_sub, mock_sys): sys.argv = ['sahara_tests/scenario/runner.py', 'sahara_tests/unit/scenario/vanilla2_7_1.yaml'] runner.main() @mock.patch('sys.exit', return_value=None) - @mock.patch('subprocess.call', return_value=None) + @mock.patch('subprocess.Popen', + return_value=_create_subprocess_communicate_mock()) def test_runner_template_missing_varfile(self, mock_sub, mock_sys): sys.argv = ['sahara_tests/scenario/runner.py', 'sahara_tests/unit/scenario/vanilla2_7_1.yaml.mako'] self.assertRaises(NameError, runner.main) @mock.patch('sys.exit', return_value=None) - @mock.patch('subprocess.call', return_value=None) + @mock.patch('subprocess.Popen', + return_value=_create_subprocess_communicate_mock()) def test_runner_template_wrong_varfile(self, mock_sub, mock_sys): sys.argv = ['sahara_tests/scenario/runner.py', '-V', @@ -267,7 +276,8 @@ class RunnerUnitTest(testtools.TestCase): self.assertRaises(NameError, runner.main) @mock.patch('sys.exit', return_value=None) - @mock.patch('subprocess.call', return_value=None) + @mock.patch('subprocess.Popen', + return_value=_create_subprocess_communicate_mock()) def test_runner_template_incomplete_varfile(self, mock_sub, mock_sys): sys.argv = ['sahara_tests/scenario/runner.py', '-V', @@ -276,7 +286,8 @@ class RunnerUnitTest(testtools.TestCase): self.assertRaises(NameError, runner.main) @mock.patch('sys.exit', return_value=None) - @mock.patch('subprocess.call', return_value=None) + @mock.patch('subprocess.Popen', + return_value=_create_subprocess_communicate_mock()) def test_runner_template_working(self, mock_sub, mock_sys): sys.argv = ['sahara_tests/scenario/runner.py', '-V', @@ -294,7 +305,8 @@ class RunnerUnitTest(testtools.TestCase): runner.main() @mock.patch('sahara_tests.scenario.validation.validate') - @mock.patch('subprocess.call', return_value=None) + @mock.patch('subprocess.Popen', + return_value=_create_subprocess_communicate_mock()) @mock.patch('sys.exit', return_value=None) def test_runner_from_args(self, mock_sys, mock_sub, mock_validate): sys.argv = ['sahara_tests/scenario/runner.py', @@ -315,7 +327,8 @@ class RunnerUnitTest(testtools.TestCase): self.assertTrue(self._isDictContainSubset( expected, mock_validate.call_args_list[0][0][0]['credentials'])) - @mock.patch('subprocess.call', return_value=None) + @mock.patch('subprocess.Popen', + return_value=_create_subprocess_communicate_mock()) @mock.patch('sys.exit', return_value=None) def test_replace_value_args(self, mock_sys, mock_sub): sys.argv = ['sahara_tests/scenario/runner.py', @@ -328,7 +341,8 @@ class RunnerUnitTest(testtools.TestCase): runner.main() @mock.patch('sahara_tests.scenario.validation.validate') - @mock.patch('subprocess.call', return_value=None) + @mock.patch('subprocess.Popen', + return_value=_create_subprocess_communicate_mock()) @mock.patch('sys.exit', return_value=None) def test_default_templates_non_plugin(self, mock_sys, mock_sub, mock_validate): @@ -339,7 +353,8 @@ class RunnerUnitTest(testtools.TestCase): runner.main() @mock.patch('sahara_tests.scenario.validation.validate') - @mock.patch('subprocess.call', return_value=None) + @mock.patch('subprocess.Popen', + return_value=_create_subprocess_communicate_mock()) @mock.patch('sys.exit', return_value=None) def test_default_templates_negative(self, mock_sys, mock_sub, mock_validate): @@ -351,7 +366,8 @@ class RunnerUnitTest(testtools.TestCase): runner.main() @mock.patch('sahara_tests.scenario.validation.validate') - @mock.patch('subprocess.call', return_value=None) + @mock.patch('subprocess.Popen', + return_value=_create_subprocess_communicate_mock()) @mock.patch('sys.exit', return_value=None) def test_default_templates_kilo(self, mock_sys, mock_sub, mock_validate): sys.argv = ['sahara_tests/scenario/runner.py', @@ -366,7 +382,8 @@ class RunnerUnitTest(testtools.TestCase): mock_validate.call_args[0][0]['clusters'][0][ 'plugin_version']) - @mock.patch('subprocess.call', return_value=None) + @mock.patch('subprocess.Popen', + return_value=_create_subprocess_communicate_mock()) @mock.patch('sys.exit', return_value=None) def test_count(self, mock_sys, mock_sub): sys.argv = ['sahara_tests/scenario/runner.py',