Add tempest run command
This commit adds a new run command to the unified cli endpoint. The intent here is for tempest to control it's own run story. This implements the basic runner and selection functionality to use the command, however it's not necessarily the end state of the command. The functionality in this patch is just a starting point to add the command and the basic functionality needed. It is starting with a limited feature set with the intent to add additional, more complex functionality in self contained patches after the command exists. Co-Authored-by: David Paterson <davpat2112@yahoo.com> Co-Authored-by: Stephen Lowrie <stephen.lowrie@rackspace.com> Partially-Implements bp tempest-run-cmd Depends-On: I09299043e536521d48dbe10632621138e3a366e0 Change-Id: I24588b5c00d005320e8719cf82b5dd95662572cf
This commit is contained in:
Matthew Treinish
parent
cfdea698ff
commit
a051c22ad0
|
|
@ -52,6 +52,7 @@ Command Documentation
|
|||
cleanup
|
||||
javelin
|
||||
workspace
|
||||
run
|
||||
|
||||
==================
|
||||
Indices and tables
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
-----------
|
||||
Tempest Run
|
||||
-----------
|
||||
|
||||
.. automodule:: tempest.cmd.run
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
---
|
||||
features:
|
||||
- Adds the tempest run command to the unified tempest CLI. This new command
|
||||
is used for running tempest tests.
|
||||
|
|
@ -42,6 +42,7 @@ tempest.cm =
|
|||
list-plugins = tempest.cmd.list_plugins:TempestListPlugins
|
||||
verify-config = tempest.cmd.verify_tempest_config:TempestVerifyConfig
|
||||
workspace = tempest.cmd.workspace:TempestWorkspace
|
||||
run = tempest.cmd.run:TempestRun
|
||||
oslo.config.opts =
|
||||
tempest.config = tempest.config:list_opts
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,181 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Runs tempest tests
|
||||
|
||||
This command is used for running the tempest tests
|
||||
|
||||
Test Selection
|
||||
==============
|
||||
Tempest run has several options:
|
||||
|
||||
* **--regex/-r**: This is a selection regex like what testr uses. It will run
|
||||
any tests that match on re.match() with the regex
|
||||
* **--smoke**: Run all the tests tagged as smoke
|
||||
|
||||
You can also use the **--list-tests** option in conjunction with selection
|
||||
arguments to list which tests will be run.
|
||||
|
||||
Test Execution
|
||||
==============
|
||||
There are several options to control how the tests are executed. By default
|
||||
tempest will run in parallel with a worker for each CPU present on the machine.
|
||||
If you want to adjust the number of workers use the **--concurrency** option
|
||||
and if you want to run tests serially use **--serial**
|
||||
|
||||
Test Output
|
||||
===========
|
||||
By default tempest run's output to STDOUT will be generated using the
|
||||
subunit-trace output filter. But, if you would prefer a subunit v2 stream be
|
||||
output to STDOUT use the **--subunit** flag
|
||||
|
||||
"""
|
||||
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
|
||||
from cliff import command
|
||||
from os_testr import subunit_trace
|
||||
from oslo_log import log as logging
|
||||
from testrepository.commands import run_argv
|
||||
|
||||
from tempest import config
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
CONF = config.CONF
|
||||
|
||||
|
||||
class TempestRun(command.Command):
|
||||
|
||||
def _set_env(self):
|
||||
# NOTE(mtreinish): This is needed so that testr doesn't gobble up any
|
||||
# stacktraces on failure.
|
||||
if 'TESTR_PDB' in os.environ:
|
||||
return
|
||||
else:
|
||||
os.environ["TESTR_PDB"] = ""
|
||||
|
||||
def take_action(self, parsed_args):
|
||||
self._set_env()
|
||||
# Local exceution mode
|
||||
if os.path.isfile('.testr.conf'):
|
||||
# If you're running in local execution mode and there is not a
|
||||
# testrepository dir create one
|
||||
if not os.path.isdir('.testrepository'):
|
||||
returncode = run_argv(['testr', 'init'], sys.stdin, sys.stdout,
|
||||
sys.stderr)
|
||||
if returncode:
|
||||
sys.exit(returncode)
|
||||
else:
|
||||
print("No .testr.conf file was found for local exceution")
|
||||
sys.exit(2)
|
||||
|
||||
regex = self._build_regex(parsed_args)
|
||||
if parsed_args.list_tests:
|
||||
argv = ['tempest', 'list-tests', regex]
|
||||
returncode = run_argv(argv, sys.stdin, sys.stdout, sys.stderr)
|
||||
else:
|
||||
options = self._build_options(parsed_args)
|
||||
returncode = self._run(regex, options)
|
||||
sys.exit(returncode)
|
||||
|
||||
def get_description(self):
|
||||
return 'Run tempest'
|
||||
|
||||
def get_parser(self, prog_name):
|
||||
parser = super(TempestRun, self).get_parser(prog_name)
|
||||
parser = self._add_args(parser)
|
||||
return parser
|
||||
|
||||
def _add_args(self, parser):
|
||||
# test selection args
|
||||
regex = parser.add_mutually_exclusive_group()
|
||||
regex.add_argument('--smoke', action='store_true',
|
||||
help="Run the smoke tests only")
|
||||
regex.add_argument('--regex', '-r', default='',
|
||||
help='A normal testr selection regex used to '
|
||||
'specify a subset of tests to run')
|
||||
# list only args
|
||||
parser.add_argument('--list-tests', '-l', action='store_true',
|
||||
help='List tests',
|
||||
default=False)
|
||||
# exectution args
|
||||
parser.add_argument('--concurrency', '-w',
|
||||
help="The number of workers to use, defaults to "
|
||||
"the number of cpus")
|
||||
parallel = parser.add_mutually_exclusive_group()
|
||||
parallel.add_argument('--parallel', dest='parallel',
|
||||
action='store_true',
|
||||
help='Run tests in parallel (this is the'
|
||||
' default)')
|
||||
parallel.add_argument('--serial', dest='parallel',
|
||||
action='store_false',
|
||||
help='Run tests serially')
|
||||
# output args
|
||||
parser.add_argument("--subunit", action='store_true',
|
||||
help='Enable subunit v2 output')
|
||||
|
||||
parser.set_defaults(parallel=True)
|
||||
return parser
|
||||
|
||||
def _build_regex(self, parsed_args):
|
||||
regex = ''
|
||||
if parsed_args.smoke:
|
||||
regex = 'smoke'
|
||||
elif parsed_args.regex:
|
||||
regex = parsed_args.regex
|
||||
return regex
|
||||
|
||||
def _build_options(self, parsed_args):
|
||||
options = []
|
||||
if parsed_args.subunit:
|
||||
options.append("--subunit")
|
||||
if parsed_args.parallel:
|
||||
options.append("--parallel")
|
||||
if parsed_args.concurrency:
|
||||
options.append("--concurrency=%s" % parsed_args.concurrency)
|
||||
return options
|
||||
|
||||
def _run(self, regex, options):
|
||||
returncode = 0
|
||||
argv = ['tempest', 'run', regex] + options
|
||||
if '--subunit' in options:
|
||||
returncode = run_argv(argv, sys.stdin, sys.stdout, sys.stderr)
|
||||
else:
|
||||
argv.append('--subunit')
|
||||
stdin = io.StringIO()
|
||||
stdout_r, stdout_w = os.pipe()
|
||||
subunit_w = os.fdopen(stdout_w, 'wt')
|
||||
subunit_r = os.fdopen(stdout_r)
|
||||
returncodes = {}
|
||||
|
||||
def run_argv_thread():
|
||||
returncodes['testr'] = run_argv(argv, stdin, subunit_w,
|
||||
sys.stderr)
|
||||
subunit_w.close()
|
||||
|
||||
run_thread = threading.Thread(target=run_argv_thread)
|
||||
run_thread.start()
|
||||
returncodes['subunit-trace'] = subunit_trace.trace(subunit_r,
|
||||
sys.stdout)
|
||||
run_thread.join()
|
||||
subunit_r.close()
|
||||
# python version of pipefail
|
||||
if returncodes['testr']:
|
||||
returncode = returncodes['testr']
|
||||
elif returncodes['subunit-trace']:
|
||||
returncode = returncodes['subunit-trace']
|
||||
return returncode
|
||||
|
|
@ -0,0 +1,110 @@
|
|||
# Copyright 2015 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
import mock
|
||||
|
||||
from tempest.cmd import run
|
||||
from tempest.tests import base
|
||||
|
||||
DEVNULL = open(os.devnull, 'wb')
|
||||
|
||||
|
||||
class TestTempestRun(base.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestTempestRun, self).setUp()
|
||||
self.run_cmd = run.TempestRun(None, None)
|
||||
|
||||
def test_build_options(self):
|
||||
args = mock.Mock(spec=argparse.Namespace)
|
||||
setattr(args, "subunit", True)
|
||||
setattr(args, "parallel", False)
|
||||
setattr(args, "concurrency", 10)
|
||||
options = self.run_cmd._build_options(args)
|
||||
self.assertEqual(['--subunit',
|
||||
'--concurrency=10'],
|
||||
options)
|
||||
|
||||
def test__build_regex_default(self):
|
||||
args = mock.Mock(spec=argparse.Namespace)
|
||||
setattr(args, 'smoke', False)
|
||||
setattr(args, 'regex', '')
|
||||
self.assertEqual('', self.run_cmd._build_regex(args))
|
||||
|
||||
def test__build_regex_smoke(self):
|
||||
args = mock.Mock(spec=argparse.Namespace)
|
||||
setattr(args, "smoke", True)
|
||||
setattr(args, 'regex', '')
|
||||
self.assertEqual('smoke', self.run_cmd._build_regex(args))
|
||||
|
||||
def test__build_regex_regex(self):
|
||||
args = mock.Mock(spec=argparse.Namespace)
|
||||
setattr(args, 'smoke', False)
|
||||
setattr(args, "regex", 'i_am_a_fun_little_regex')
|
||||
self.assertEqual('i_am_a_fun_little_regex',
|
||||
self.run_cmd._build_regex(args))
|
||||
|
||||
|
||||
class TestRunReturnCode(base.TestCase):
|
||||
def setUp(self):
|
||||
super(TestRunReturnCode, self).setUp()
|
||||
# Setup test dirs
|
||||
self.directory = tempfile.mkdtemp(prefix='tempest-unit')
|
||||
self.addCleanup(shutil.rmtree, self.directory)
|
||||
self.test_dir = os.path.join(self.directory, 'tests')
|
||||
os.mkdir(self.test_dir)
|
||||
# Setup Test files
|
||||
self.testr_conf_file = os.path.join(self.directory, '.testr.conf')
|
||||
self.setup_cfg_file = os.path.join(self.directory, 'setup.cfg')
|
||||
self.passing_file = os.path.join(self.test_dir, 'test_passing.py')
|
||||
self.failing_file = os.path.join(self.test_dir, 'test_failing.py')
|
||||
self.init_file = os.path.join(self.test_dir, '__init__.py')
|
||||
self.setup_py = os.path.join(self.directory, 'setup.py')
|
||||
shutil.copy('tempest/tests/files/testr-conf', self.testr_conf_file)
|
||||
shutil.copy('tempest/tests/files/passing-tests', self.passing_file)
|
||||
shutil.copy('tempest/tests/files/failing-tests', self.failing_file)
|
||||
shutil.copy('setup.py', self.setup_py)
|
||||
shutil.copy('tempest/tests/files/setup.cfg', self.setup_cfg_file)
|
||||
shutil.copy('tempest/tests/files/__init__.py', self.init_file)
|
||||
# Change directory, run wrapper and check result
|
||||
self.addCleanup(os.chdir, os.path.abspath(os.curdir))
|
||||
os.chdir(self.directory)
|
||||
|
||||
def assertRunExit(self, cmd, expected):
|
||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
out, err = p.communicate()
|
||||
msg = ("Running %s got an unexpected returncode\n"
|
||||
"Stdout: %s\nStderr: %s" % (' '.join(cmd), out, err))
|
||||
self.assertEqual(p.returncode, expected, msg)
|
||||
|
||||
def test_tempest_run_passes(self):
|
||||
# Git init is required for the pbr testr command. pbr requires a git
|
||||
# version or an sdist to work. so make the test directory a git repo
|
||||
# too.
|
||||
subprocess.call(['git', 'init'], stderr=DEVNULL)
|
||||
self.assertRunExit(['tempest', 'run', '--regex', 'passing'], 0)
|
||||
|
||||
def test_tempest_run_fails(self):
|
||||
# Git init is required for the pbr testr command. pbr requires a git
|
||||
# version or an sdist to work. so make the test directory a git repo
|
||||
# too.
|
||||
subprocess.call(['git', 'init'], stderr=DEVNULL)
|
||||
self.assertRunExit(['tempest', 'run'], 1)
|
||||
Loading…
Reference in New Issue