Add new runner for system test

For run the tests from both test suites (fuelweb_test, system_test) we
    may use new runner - run_system_test.py.

Changes in framework:
    - add @testcase decorator use instead of @factory

Features of new runner:
    - auto discovering all test in both test suites
    - show the groups from the test suites
    - explain content of groups
    - run the several groups at the same time
    - combine configuration with the test groups from new suite
    - run old groups
    - use runner in utils/jenkins/system_tests.sh

Changes in tests:
    - remove @factory function
    - add @testcase to each test class

Change-Id: Ic4086dde60ca8a94dcd2ee079376c97ce719ff03
Implemets blueprint template-based-testcases
This commit is contained in:
Dmitry Tyzhnenko 2016-01-29 17:41:31 +02:00 committed by Dmitry Tyzhnenko
parent 5905a0c220
commit c106f6be54
23 changed files with 585 additions and 353 deletions

View File

@ -1,184 +0,0 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import re
import sys
from nose.plugins import Plugin
from paramiko.transport import _join_lingering_threads
class CloseSSHConnectionsPlugin(Plugin):
"""Closes all paramiko's ssh connections after each test case
Plugin fixes proboscis disability to run cleanup of any kind.
'afterTest' calls _join_lingering_threads function from paramiko,
which stops all threads (set the state to inactive and joins for 10s)
"""
name = 'closesshconnections'
def options(self, parser, env=os.environ):
super(CloseSSHConnectionsPlugin, self).options(parser, env=env)
def configure(self, options, conf):
super(CloseSSHConnectionsPlugin, self).configure(options, conf)
self.enabled = True
def afterTest(self, *args, **kwargs):
_join_lingering_threads()
def import_tests():
from tests import test_admin_node # noqa
from tests import test_backup_restore # noqa
from tests import test_ceph # noqa
from tests import test_environment_action # noqa
from tests import test_ironic_base # noqa
from tests import test_neutron # noqa
from tests import test_neutron_public # noqa
from tests import test_neutron_tun # noqa
from tests import test_neutron_ipv6 # noqa
from tests import test_pullrequest # noqa
from tests import test_services # noqa
from tests import test_ha_one_controller # noqa
from tests import test_vcenter # noqa
from tests import test_reduced_footprint # noqa
from tests.tests_cli import test_cli_role # noqa
from tests.tests_cli import test_cli_deploy # noqa
from tests.tests_cli import test_cli_deploy_ceph # noqa
from tests.tests_deployments.tests_neutron_vlan import test_ha_vlan_group_1 # noqa
from tests.tests_deployments.tests_neutron_vlan import test_ha_vlan_group_2 # noqa
from tests.tests_deployments.tests_neutron_vlan import test_ha_vlan_group_3 # noqa
from tests.tests_deployments.tests_neutron_vlan import test_ha_vlan_group_4 # noqa
from tests.tests_deployments.tests_neutron_vlan import test_ha_vlan_group_5 # noqa
from tests.tests_deployments.tests_neutron_vlan import test_ha_vlan_group_6 # noqa
from tests.tests_deployments.tests_neutron_vlan import test_ha_vlan_group_7 # noqa
from tests.tests_deployments.tests_neutron_tun import test_ha_tun_group_1 # noqa
from tests.tests_deployments.tests_neutron_tun import test_ha_tun_group_2 # noqa
from tests.tests_deployments.tests_neutron_tun import test_ha_tun_group_3 # noqa
from tests.tests_multirole import test_multirole_group_1 # noqa
from tests.tests_scale import test_scale_group_1 # noqa
from tests.tests_scale import test_scale_group_2 # noqa
from tests.tests_scale import test_scale_group_3 # noqa
from tests.tests_scale import test_scale_group_4 # noqa
from tests.tests_scale import test_scale_group_5 # noqa
from tests.tests_scale import test_scale_group_6 # noqa
from tests.tests_multirole import test_mongo_multirole # noqa
from tests import test_rh_compute # noqa
from tests.tests_security import test_run_nessus # noqa
from tests.tests_separate_services import test_separate_db # noqa
from tests.tests_separate_services import test_separate_horizon # noqa
from tests.tests_separate_services import test_separate_keystone # noqa
from tests.tests_separate_services import test_separate_multiroles # noqa
from tests.tests_separate_services import test_separate_rabbitmq # noqa
from tests.tests_separate_services import test_separate_db_ceph # noqa
from tests.tests_separate_services import test_separate_keystone_ceph # noqa
from tests.tests_separate_services import test_separate_rabbitmq_ceph # noqa
from tests import test_clone_env # noqa
from tests import test_node_reassignment # noqa
from tests import test_os_upgrade # noqa
from tests.tests_strength import test_failover # noqa
from tests.tests_strength import test_failover_with_ceph # noqa
from tests.tests_strength import test_master_node_failover # noqa
from tests.tests_strength import test_ostf_repeatable_tests # noqa
from tests.tests_strength import test_restart # noqa
from tests.tests_strength import test_huge_environments # noqa
from tests.tests_strength import test_image_based # noqa
from tests.tests_strength import test_cic_maintenance_mode # noqa
from tests.tests_upgrade import test_upgrade # noqa
from tests.tests_upgrade import test_upgrade_chains # noqa
from tests import test_bonding # noqa
from tests import test_offloading_types # noqa
from tests import test_bond_offloading # noqa
from tests.tests_strength import test_neutron # noqa
from tests.plugins.plugin_emc import test_plugin_emc # noqa
from tests.plugins.plugin_elasticsearch import test_plugin_elasticsearch # noqa
from tests.plugins.plugin_example import test_fuel_plugin_example # noqa
from tests.plugins.plugin_example import test_fuel_plugin_example_postdeploy # noqa
from tests.plugins.plugin_contrail import test_fuel_plugin_contrail # noqa
from tests.plugins.plugin_glusterfs import test_plugin_glusterfs # noqa
from tests.plugins.plugin_influxdb import test_plugin_influxdb # noqa
from tests.plugins.plugin_lbaas import test_plugin_lbaas # noqa
from tests.plugins.plugin_lma_collector import test_plugin_lma_collector # noqa
from tests.plugins.plugin_lma_infra_alerting import test_plugin_lma_infra_alerting # noqa
from tests.plugins.plugin_reboot import test_plugin_reboot_task # noqa
from tests.plugins.plugin_vip_reservation import test_plugin_vip_reservation # noqa
from tests.plugins.plugin_zabbix import test_plugin_zabbix # noqa
from tests import test_multiple_networks # noqa
from tests.gd_based_tests import test_neutron # noqa
from tests.gd_based_tests import test_neutron_vlan_ceph_mongo # noqa
from tests.tests_patching import test_patching # noqa
from tests import test_cli # noqa
from tests import test_custom_hostname # noqa
from tests import test_jumbo_frames # noqa
from tests import test_node_reinstallation # noqa
from tests import test_ubuntu_bootstrap # noqa
from tests import test_centos_bootstrap # noqa
from tests import test_net_templates # noqa
from tests.tests_mirrors import test_create_mirror # noqa
from tests.tests_mirrors import test_use_mirror # noqa
from system_test.tests import test_create_deploy_ostf # noqa
from system_test.tests import test_deploy_check_rados # noqa
from system_test.tests import test_redeploy_after_stop # noqa
from system_test.tests import test_redeploy_after_reset # noqa
from system_test.tests import test_delete_after_deploy # noqa
from system_test.tests.strength import destroy_controllers # noqa
from system_test.tests.strength import filling_root # noqa
from system_test.tests import test_fuel_migration # noqa
from system_test.tests.plugins.plugin_example import test_plugin_example # noqa
from system_test.tests.plugins.plugin_example import test_plugin_example_v3 # noqa
from system_test.tests.vcenter import test_vcenter_dvs # noqa
from gates_tests.tests import test_review_in_fuel_agent # noqa
from gates_tests.tests import test_review_fuel_web # noqa
from tests.tests_strength import test_load # noqa
from tests import test_services_reconfiguration # noqa
from gates_tests.tests import test_review_in_ostf # noqa
from gates_tests.tests import test_review_in_fuel_client # noqa
from tests.tests_os_components import test_murano_os_component # noqa
from tests.tests_os_components import test_sahara_os_component # noqa
from tests.tests_os_components import test_mixed_os_components # noqa
from tests.tests_strength import test_failover_group_1 # noqa
from tests.tests_strength import test_failover_mongo # noqa
from tests.tests_strength import test_failover_group_2 # noqa
from gates_tests.tests import test_review_in_astute # noqa
def run_tests():
from proboscis import TestProgram # noqa
# Check if the specified test group starts any test case
if not TestProgram().cases:
from fuelweb_test import logger
logger.fatal('No test cases matched provided groups')
sys.exit(1)
# Run Proboscis and exit.
TestProgram(
addplugins=[CloseSSHConnectionsPlugin()]
).run_and_exit()
if __name__ == '__main__':
from system_test import define_custom_groups
import_tests()
define_custom_groups()
from fuelweb_test.helpers.patching import map_test
if any(re.search(r'--group=patching_master_tests', arg)
for arg in sys.argv):
map_test('master')
elif any(re.search(r'--group=patching.*', arg) for arg in sys.argv):
map_test('environment')
run_tests()

View File

@ -20,8 +20,11 @@ from proboscis import TestPlan
from proboscis.decorators import DEFAULT_REGISTRY
from builds import Build
from fuelweb_test.run_tests import import_tests
from system_test import define_custom_groups
from system_test import discover_import_tests
from system_test import register_system_test_cases
from system_test import tests_directory
from system_test.helpers.utils import get_basepath
from settings import GROUPS_TO_EXPAND
from settings import logger
from settings import TestRailSettings
@ -31,8 +34,10 @@ from testrail_client import TestRailProject
def get_tests_descriptions(milestone_id, tests_include, tests_exclude, groups,
default_test_priority):
from system_test.tests.actions_base import ActionsBase
import_tests()
discover_import_tests(get_basepath(), tests_directory)
define_custom_groups()
for one in groups:
register_system_test_cases(one)
plan = TestPlan.create_from_registry(DEFAULT_REGISTRY)
all_plan_tests = plan.tests[:]

View File

@ -17,7 +17,7 @@ from devops.helpers.helpers import tcp_ping
from devops.helpers.helpers import wait
import netaddr
from proboscis import test
from settings import LOGS_DIR
from fuelweb_test.settings import LOGS_DIR
from fuelweb_test.helpers import decorators
from fuelweb_test.helpers import nessus

195
run_system_test.py Executable file
View File

@ -0,0 +1,195 @@
#!/usr/bin/env python
import sys
import argparse
from proboscis import TestProgram
from proboscis import register
from fuelweb_test.helpers.utils import pretty_log
from system_test import register_system_test_cases
from system_test import get_groups
from system_test import define_custom_groups
from system_test import discover_import_tests
from system_test import tests_directory
from system_test.helpers.utils import collect_yamls
from system_test.helpers.utils import get_path_to_config
from system_test.helpers.utils import get_list_confignames
from system_test.helpers.utils import get_basepath
basedir = get_basepath()
def print_explain(names):
groups_nums = get_groups()
if not isinstance(names, list):
names = [names]
out = []
for name in names:
for i in groups_nums[name]:
if hasattr(i, 'home'):
out.append((i.home._proboscis_entry_.parent.home, i.home))
else:
out.append(i)
print(pretty_log(out))
def clean_argv():
"""Removing argv params unused by Proboscis"""
argv = sys.argv
if '--with-config' in argv:
idx = argv.index('--with-config')
argv.pop(idx)
argv.pop(idx)
if '--explain' in argv:
idx = argv.index('--explain')
argv.pop(idx)
return argv
def cli():
cli = argparse.ArgumentParser(prog="System test runner",
description="Command line tool for run Fuel "
"System Test")
commands = cli.add_subparsers(title="Operation commands",
dest="command")
cli_run = commands.add_parser('run',
help="Run test",
description="Run some test group")
cli_run.add_argument("run_groups", nargs='*', default=None, )
cli_run.add_argument("--with-config", default=False, type=str,
action="store", dest="config_name",
help="Select name of yaml config.")
cli_run.add_argument("--explain", default=False, action="store_true",
help="Show explain for running groups. "
"Will not start Proboscis.")
cli_run.add_argument("--show-plan", default=False, action="store_true",
help="Show Proboscis test plan.")
cli_run.add_argument("--with-xunit", default=False, action="store_true",
help="Use xuint report.")
cli_run.add_argument("--nologcapture", default=False, action="store_true",
help="Disable log capture for Proboscis.")
cli_run.add_argument("-q", default=False, action="store_true",
dest="quite",
help="Run Proboscis in quite mode.")
cli_run.add_argument("-a", default=False, action="store_true",
dest="nose_attr",
help="Provide Nose attr to Proboscis.")
cli_run.add_argument("-A", default=False, action="store_true",
dest="eval_nose",
help="Eval Nose attr to Proboscis.")
cli_run.add_argument("--groups", default=None, action="append", type=str,
help="Test group for testing. "
"(backward compatibility)")
cli_explain_group = commands.add_parser("explain-group",
help="Explain selected group.")
cli_explain_group.add_argument("name",
help="Group name.")
commands.add_parser("show-all-groups",
help="Show all Proboscis groups")
commands.add_parser("show-fuelweb-groups",
help="Show Proboscis groups defined in fuelweb suite")
commands.add_parser("show-systest-groups",
help="Show Proboscis groups defined in Systest suite")
commands.add_parser("show-systest-configs",
help="Show configurations for Systest suite")
if len(sys.argv) == 1:
cli.print_help()
sys.exit(1)
return cli.parse_args()
def run(**kwargs):
config_name = kwargs.get('config_name', None)
groups = kwargs.get('run_groups', None)
old_groups = kwargs.get('groups', None)
explain = kwargs.get('explain', None)
groups_to_run = []
groups.extend(old_groups or [])
for g in groups:
if config_name:
register_system_test_cases(
groups=[g],
configs=[config_name])
groups_to_run.append("{0}({1})".format(g, config_name))
else:
register_system_test_cases(groups=[g])
groups_to_run.append(g)
if explain:
print_explain(groups)
else:
register(groups=["run_system_test"], depends_on_groups=groups_to_run)
TestProgram(groups=['run_system_test'],
argv=clean_argv()).run_and_exit()
def explain_group(**kwargs):
"""Explain selected group."""
name = kwargs.get('name', None)
print_explain(name)
def show_all_groups(**kwargs):
"""Show all Proboscis groups"""
groups_nums = get_groups()
out = {k: len(v) for k, v in groups_nums.iteritems()}
print(pretty_log(out))
def show_fuelweb_groups(**kwargs):
"""Show Proboscis groups defined in fuelweb suite"""
groups_nums = get_groups()
out = {k: len(v) for k, v in groups_nums.iteritems()
if not k.startswith('system_test')}
print(pretty_log(out))
def show_systest_groups(**kwargs):
"""Show Proboscis groups defined in Systest suite"""
groups_nums = get_groups()
out = {k: len(v) for k, v in groups_nums.iteritems()
if k.startswith('system_test')}
print(pretty_log(out))
def show_systest_configs(**kwargs):
"""Show configurations for Systest suite"""
tests_configs = collect_yamls(get_path_to_config())
for c in get_list_confignames(tests_configs):
print(c)
COMMAND_MAP = {
"run": run,
"explain-group": explain_group,
"show-all-groups": show_all_groups,
"show-fuelweb-groups": show_fuelweb_groups,
"show-systest-groups": show_systest_groups,
"show-systest-configs": show_systest_configs
}
def shell():
args = cli()
discover_import_tests(basedir, tests_directory)
define_custom_groups()
COMMAND_MAP[args.command](**vars(args))
if __name__ == '__main__':
shell()

View File

@ -11,14 +11,35 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import re
import fuelweb_test
from system_test.helpers.utils import get_configs
from fuelweb_test import logger
import proboscis.core
from proboscis import register
from proboscis import factory
logger = fuelweb_test.logger
from system_test.core.repository import Repository
from system_test.core.discover import discover_import_tests
from system_test.helpers.utils import get_configs
from system_test.helpers.decorators import testcase
from proboscis.decorators import DEFAULT_REGISTRY
from system_test.tests import base_actions_factory
from system_test.helpers.utils import config_filter
tests_directory = [
'fuelweb_test/tests',
'system_test/tests'
]
__all__ = [
Repository,
discover_import_tests,
testcase,
get_configs,
logger]
def cached_add_group(yamls):
@ -35,6 +56,8 @@ def cached_add_group(yamls):
if validate_config and config_name not in yamls:
raise NameError("Config {} not found".format(config_name))
register_system_test_cases(groups=[systest_group],
configs=[config_name])
register(groups=[group],
depends_on_groups=[
"{systest_group}({config_name})".format(
@ -71,3 +94,108 @@ def define_custom_groups():
add_group(group="fuel_master_migrate",
systest_group="system_test.fuel_migration",
config_name="1ctrl_1comp_neutronTUN")
def get_groups(only_groups=None, exclude=None):
"""Get groups from Proboscis register and count them children"""
groups_childs = {}
groups = {}
if only_groups and isinstance(only_groups, list):
groups = {g: DEFAULT_REGISTRY.groups[g]
for g in DEFAULT_REGISTRY.groups if g in only_groups}
groups.update({g: Repository.index[g]
for g in Repository.index if g in only_groups})
else:
groups = DEFAULT_REGISTRY.groups.copy()
groups.update({g: Repository.index[g] for g in Repository.index})
for group_name, group in groups.items():
klass_entries = set()
entries_in_class = set()
if (exclude and
isinstance(exclude, list) and
any([e in group_name for e in exclude])):
continue
if hasattr(group, 'entries'):
for entry in group.entries:
if isinstance(entry, proboscis.core.TestMethodClassEntry):
klass_entries.add(entry)
for klass in klass_entries:
entries_in_class.update(set(klass.children))
child = set(group.entries) - entries_in_class - klass_entries
for klass in klass_entries:
if (klass.used_by_factory and
base_actions_factory.BaseActionsFactory in
klass.home.__mro__):
child.add(klass)
else:
child.update(set(klass.children))
else:
child = [g for g in group
if base_actions_factory.BaseActionsFactory in g.__mro__]
groups_childs[group_name] = child
return groups_childs
def case_factory(baseclass, configs):
"""Return list of instance """
# configs = get_configs()
return [baseclass.caseclass_factory(g)(c)
for g, c in config_filter(configs).items()]
def case_filter(groups=None):
"""Create Proboscis factories for selected groups. For all by default"""
if groups is None:
return set(Repository)
cases = set()
for g in groups:
if g in Repository.index:
cases.update(Repository.index[g])
return cases
def reg_factory(cases, configs):
def ret():
out = []
for c in cases:
out.extend(case_factory(c, configs))
return out
globals()['system_test_factory'] = factory(ret)
def split_group_config(group):
m = re.search('([\w\.]*)\((\w*)\)', group)
if m:
return m.groups()
def register_system_test_cases(groups=None, configs=None):
to_remove = []
to_add = []
for group in groups:
g_c = split_group_config(group)
if g_c:
g, c = g_c
to_add.append(g)
if configs is None:
configs = []
configs.append(c)
to_remove.append(group)
for one in to_remove:
groups.remove(one)
for one in to_add:
groups.append(one)
cases = case_filter(groups)
configs = config_filter(configs)
if cases:
reg_factory(cases, configs)

View File

View File

@ -0,0 +1,45 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os.path
def discover_test_files(basedir, dirs):
"""Find all files in path"""
ret = []
for path in dirs:
path = os.path.join(basedir, path)
for r, d, f in os.walk(path):
for one in f:
if one.startswith('test_') and one.endswith('.py'):
ret.append(os.path.join(r, one))
return ret
def convert_files_to_modules(basedir, files):
"""Convert files name to modules name"""
ret = []
for one in files:
module = os.path.splitext(
os.path.relpath(one, basedir))[0].replace('/', '.')
ret.append(module)
return ret
def discover_import_tests(basedir, dirs):
"""Walk through directories and import all modules with tests"""
imported_list = []
for module in convert_files_to_modules(basedir,
discover_test_files(basedir, dirs)):
imported_list.append(__import__(module))

View File

@ -0,0 +1,76 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from fuelweb_test.helpers import metaclasses
class TestCaseRepository(set):
__metaclass__ = metaclasses.SingletonMeta
def __init__(self):
super(TestCaseRepository, self).__init__()
self.__index = {}
@property
def index(self):
return self.__index
def __index_add(self, v):
groups = getattr(v, '_base_groups', None)
for g in groups:
if g not in self.__index:
self.__index[g] = set()
self.__index[g].add(v)
def __index_remove(self, v):
groups = getattr(v, '_base_groups', None)
for g in groups:
self.__index[g].remove(v)
if not len(self.__index[g]):
del self.__index[g]
def add(self, value):
super(TestCaseRepository, self).add(value)
self.__index_add(value)
def remove(self, value):
super(TestCaseRepository, self).remove(value)
self.__index_remove(value)
def pop(self, value):
super(TestCaseRepository, self).pop(value)
self.__index_remove(value)
def filter(self, groups=None):
"""Return list of cases related to groups. All by default"""
if groups is None:
return set(self)
cases = set()
for g in groups:
if g in self.index:
cases.update(self.index[g])
return cases
def union(self, *args, **kwargs):
raise AttributeError("'TestCaseRepository' object has no attribute "
" 'union'")
def update(self, *args, **kwargs):
raise AttributeError("'TestCaseRepository' object has no attribute "
" 'update'")
Repository = TestCaseRepository()

View File

@ -16,13 +16,15 @@ import functools
import traceback
import sys
import hashlib
import inspect
import collections
from proboscis import SkipTest
from fuelweb_test.helpers.utils import TimeStat
from fuelweb_test.helpers.utils import pull_out_logs_via_ssh
from fuelweb_test.helpers.decorators import create_diagnostic_snapshot
from system_test import Repository
from system_test import logger
@ -43,26 +45,6 @@ def nested_action(method):
return staticmethod(method)
def step_start_stop(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
with TimeStat(func) as timer:
step_name = getattr(func, '_step_name')
start_step = '[ START {} ]'.format(step_name)
header = "<<< {:-^142} >>>".format(start_step)
logger.info("\n{header}\n".format(header=header))
result = func(*args, **kwargs)
spent_time = timer.spent_time
minutes = int(round(spent_time)) / 60
seconds = int(round(spent_time)) % 60
finish_step = "[ FINISH {} STEP TOOK {} min {} sec ]".format(
step_name, minutes, seconds)
footer = "<<< {:-^142} >>>".format(finish_step)
logger.info("\n{footer}\n".format(footer=footer))
return result
return wrapper
def make_snapshot_if_step_fail(func):
"""Generate diagnostic snapshot if step fail.
@ -122,3 +104,18 @@ def make_snapshot_if_step_fail(func):
raise test_exception, None, exc_trace
return result
return wrapper
def testcase(groups):
"""Use this decorator for mark a test case class"""
def testcase_decorator(cls):
if not inspect.isclass(cls):
raise TypeError("Decorator @testcase should used only "
"with classes")
if not isinstance(groups, collections.Sequence):
raise TypeError("Use list for groups")
cls.get_actions_order()
setattr(cls, '_base_groups', groups)
Repository.add(cls)
return cls
return testcase_decorator

View File

@ -33,6 +33,12 @@ def copy_func(f, name=None):
return fn
def get_basepath():
import system_test
return os.path.join(
os.path.dirname(os.path.dirname(system_test.__file__)))
def get_list_confignames(filelist):
"""Get list of config name from file list"""
return map(get_configname, filelist)
@ -115,7 +121,7 @@ def find_duplicates(yamls):
dup[name].append(one)
else:
dup[name] = [one]
return {k: v for k, v in dup.iteritems() if len(v) > 1}
return {k: v for k, v in dup.items() if len(v) > 1}
def get_configs():
@ -129,7 +135,7 @@ def get_configs():
return {get_configname(y): y for y in yamls}
def case_factory(baseclass):
"""Return list of instance """
configs = get_configs()
return [baseclass.caseclass_factory(g)(c) for g, c in configs.iteritems()]
def config_filter(configs=None):
if configs is None:
return get_configs()
return {k: v for k, v in get_configs().items() if k in configs}

View File

@ -12,14 +12,37 @@
# License for the specific language governing permissions and limitations
# under the License.
from proboscis import test
import functools
from proboscis import after_class
from proboscis import before_class
from proboscis import test
from fuelweb_test.helpers.utils import TimeStat
from fuelweb_test.tests import base_test_case
from system_test.helpers import utils
from system_test.helpers.decorators import step_start_stop
from system_test import logger
def step_start_stop(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
with TimeStat(func) as timer:
step_name = getattr(func, '_step_name')
start_step = '[ START {} ]'.format(step_name)
header = "<<< {:-^142} >>>".format(start_step)
logger.info("\n{header}\n".format(header=header))
result = func(*args, **kwargs)
spent_time = timer.spent_time
minutes = int(round(spent_time)) / 60
seconds = int(round(spent_time)) % 60
finish_step = "[ FINISH {} STEP TOOK {} min {} sec ]".format(
step_name, minutes, seconds)
footer = "<<< {:-^142} >>>".format(finish_step)
logger.info("\n{footer}\n".format(footer=footer))
return result
return wrapper
class BaseActionsFactory(base_test_case.TestBasic):
@ -28,7 +51,7 @@ class BaseActionsFactory(base_test_case.TestBasic):
def get_actions(cls):
"""Return all action methods"""
return {m: getattr(cls, m) for m in
dir(cls) if m.startswith('_action_') or
dir(cls) if
getattr(getattr(cls, m), '_action_method_', False) or
getattr(getattr(cls, m), '_nested_action_method_', False)}
@ -41,9 +64,29 @@ class BaseActionsFactory(base_test_case.TestBasic):
actions_method = cls.get_actions()
linear_order = []
for action in cls.actions_order:
if getattr(actions_method[action],
try:
action_method = actions_method[action]
except KeyError:
import inspect
source = inspect.getsourcelines(inspect.getmodule(cls))[0]
counted_data = [n for n in enumerate(source)]
line_num = [n for (n, l) in counted_data if 'ddd' in l][0]
cutted = counted_data[line_num - 4:line_num + 4]
cutted = [(n, l[:-1] + " " * 20 + "<====\n"
if n == line_num else l)
for (n, l) in cutted]
cutted = ["Line {line_num:04d}: {line}".format(
line_num=n, line=l) for (n, l) in cutted]
raise LookupError("Class {} orders to run '{}' action as {} "
"step,\n\tbut action method doesn't exist "
"in class.\nLook at '{}':\n\n{}".format(
cls, action,
cls.actions_order.index(action),
inspect.getsourcefile(cls),
''.join(cutted)))
if getattr(action_method,
'_nested_action_method_', None):
linear_order.extend(actions_method[action]())
linear_order.extend(action_method())
else:
linear_order.append(action)
@ -140,8 +183,8 @@ class BaseActionsFactory(base_test_case.TestBasic):
runs_after=[teardown_method] if teardown_method else [])
# Generate test case groups
groups = ['{}({})'.format(g, case_group) for g in cls.base_group]
groups = cls.base_group + groups
groups = ['{}({})'.format(g, case_group) for g in cls._base_groups]
groups = cls._base_groups + groups
# Generate test case docstring
test_steps["__doc__"] = "{}\n\n{}\n\nDuration {}".format(

View File

@ -12,13 +12,15 @@
# License for the specific language governing permissions and limitations
# under the License.
from proboscis import factory
from system_test import testcase
from fuelweb_test.settings import EXAMPLE_PLUGIN_PATH
from system_test.helpers.utils import case_factory
from system_test.tests.actions_base import ActionsBase
@testcase(groups=['system_test',
'system_test.plugins',
'system_test.plugins.example_plugin',
'system_test.plugins.example_plugin.simple'])
class DeployWithPluginExample(ActionsBase):
"""Deploy cluster with one controller and example plugin
@ -37,11 +39,6 @@ class DeployWithPluginExample(ActionsBase):
Snapshot deploy_ha_one_controller_neutron_example
"""
base_group = ['system_test',
'system_test.plugins',
'system_test.plugins.example_plugin',
'system_test.plugins.example_plugin.simple']
plugin_name = "fuel_plugin_example"
plugin_path = EXAMPLE_PLUGIN_PATH
@ -58,6 +55,10 @@ class DeployWithPluginExample(ActionsBase):
]
@testcase(groups=['system_test',
'system_test.plugins',
'system_test.plugins.example_plugin',
'system_test.plugins.example_plugin.simple_scale'])
class DeployScaleWithPluginExample(ActionsBase):
"""Deploy and scale cluster in ha mode with example plugin
@ -81,11 +82,6 @@ class DeployScaleWithPluginExample(ActionsBase):
Snapshot deploy_neutron_example_ha_add_node
"""
base_group = ['system_test',
'system_test.plugins',
'system_test.plugins.example_plugin',
'system_test.plugins.example_plugin.simple_scale']
plugin_name = "fuel_plugin_example"
plugin_path = EXAMPLE_PLUGIN_PATH
@ -106,9 +102,3 @@ class DeployScaleWithPluginExample(ActionsBase):
'check_example_plugin',
'health_check',
]
@factory
def cases():
return (case_factory(DeployWithPluginExample) +
case_factory(DeployScaleWithPluginExample))

View File

@ -12,16 +12,19 @@
# License for the specific language governing permissions and limitations
# under the License.
from proboscis import factory
from fuelweb_test.settings import EXAMPLE_PLUGIN_V3_PATH
from system_test.helpers.utils import case_factory
from system_test import testcase
from system_test.helpers.decorators import make_snapshot_if_step_fail
from system_test.helpers.decorators import deferred_decorator
from system_test.helpers.decorators import action
from system_test.tests.actions_base import ActionsBase
@testcase(groups=['system_test',
'system_test.plugins',
'system_test.plugins.example_plugin_v3',
'system_test.plugins.example_plugin_v3.simple'])
class DeployWithPluginExampleV3(ActionsBase):
"""Deploy cluster with one controller and example plugin v3
@ -40,10 +43,6 @@ class DeployWithPluginExampleV3(ActionsBase):
Duration 35m
Snapshot deploy_ha_one_controller_neutron_example_v3
"""
base_group = ['system_test',
'system_test.plugins',
'system_test.plugins.example_plugin_v3',
'system_test.plugins.example_plugin_v3.simple']
plugin_name = 'fuel_plugin_example_v3'
plugin_path = EXAMPLE_PLUGIN_V3_PATH
@ -69,8 +68,3 @@ class DeployWithPluginExampleV3(ActionsBase):
'roles': ['fuel_plugin_example_v3'],
'count': 1
}])
@factory
def cases():
return (case_factory(DeployWithPluginExampleV3))

View File

@ -12,15 +12,17 @@
# License for the specific language governing permissions and limitations
# under the License.
from system_test.helpers.utils import case_factory
from proboscis import factory
from system_test import testcase
from system_test.tests.strength import strength_base
from system_test.helpers.decorators import make_snapshot_if_step_fail
from system_test.helpers.decorators import deferred_decorator
from system_test.helpers.decorators import action
@testcase(groups=['system_test',
'system_test.failover',
'system_test.failover.destroy_controllers',
'system_test.failover.destroy_controllers.first'])
class StrengthDestroyFirstController(strength_base.StrengthBaseActions):
"""Destroy two controllers and check pacemaker status is correct
@ -39,11 +41,6 @@ class StrengthDestroyFirstController(strength_base.StrengthBaseActions):
"""
base_group = ['system_test',
'system_test.failover',
'system_test.failover.destroy_controllers',
'system_test.failover.destroy_controllers.first']
actions_order = [
'setup_master',
'config_release',
@ -72,6 +69,10 @@ class StrengthDestroyFirstController(strength_base.StrengthBaseActions):
self._destroy_controller('slave-01')
@testcase(groups=['system_test',
'system_test.failover',
'system_test.failover.destroy_controllers',
'system_test.failover.destroy_controllers.second'])
class StrengthDestroySecondController(strength_base.StrengthBaseActions):
"""Destroy two controllers and check pacemaker status is correct
@ -90,11 +91,6 @@ class StrengthDestroySecondController(strength_base.StrengthBaseActions):
"""
base_group = ['system_test',
'system_test.failover',
'system_test.failover.destroy_controllers',
'system_test.failover.destroy_controllers.second']
actions_order = [
'setup_master',
'config_release',
@ -121,9 +117,3 @@ class StrengthDestroySecondController(strength_base.StrengthBaseActions):
def destroy_second_controller(self):
"""Destroy second controller"""
self._destroy_controller('slave-02')
@factory
def cases():
return (case_factory(StrengthDestroyFirstController) +
case_factory(StrengthDestroySecondController))

View File

@ -12,13 +12,14 @@
# License for the specific language governing permissions and limitations
# under the License.
from proboscis import factory
from system_test.helpers.utils import case_factory
from system_test import testcase
from system_test.tests.strength import strength_base
@testcase(groups=['system_test',
'system_test.failover',
'system_test.failover.filling_root'])
class FillRootPrimaryController(
strength_base.FillRootBaseActions
):
@ -51,11 +52,6 @@ class FillRootPrimaryController(
21. Run OSTF Sanity, Smoke, HA
"""
base_group = ['system_test',
'system_test.failover',
'system_test.failover.filling_root'
]
actions_order = [
'setup_master',
'config_release',
@ -79,8 +75,3 @@ class FillRootPrimaryController(
'check_starting_resources',
'health_check_sanity_smoke_ha',
]
@factory
def cases():
return case_factory(FillRootPrimaryController)

View File

@ -12,11 +12,11 @@
# License for the specific language governing permissions and limitations
# under the License.
from system_test import testcase
from system_test.tests import actions_base
from system_test.helpers.utils import case_factory
from proboscis import factory
@testcase(groups=['system_test', 'system_test.create_deploy_ostf'])
class CreateDeployOstf(actions_base.ActionsBase):
"""Case deploy Environment
@ -29,7 +29,6 @@ class CreateDeployOstf(actions_base.ActionsBase):
6. Run OSTF
"""
base_group = ['system_test', 'system_test.create_deploy_ostf']
actions_order = [
'prepare_admin_node_with_slaves',
'create_env',
@ -39,8 +38,3 @@ class CreateDeployOstf(actions_base.ActionsBase):
'network_check',
'health_check',
]
@factory
def cases():
return case_factory(CreateDeployOstf)

View File

@ -12,11 +12,11 @@
# License for the specific language governing permissions and limitations
# under the License.
from system_test import testcase
from system_test.tests import actions_base
from system_test.helpers.utils import case_factory
from proboscis import factory
@testcase(groups=['system_test', 'system_test.delete_after_deploy'])
class DeleteAfterDeploy(actions_base.ActionsBase):
"""Case deploy Environment
@ -29,7 +29,6 @@ class DeleteAfterDeploy(actions_base.ActionsBase):
6. Run OSTF
"""
base_group = ['system_test', 'system_test.delete_after_deploy']
actions_order = [
'prepare_admin_node_with_slaves',
'create_env',
@ -40,8 +39,3 @@ class DeleteAfterDeploy(actions_base.ActionsBase):
'health_check',
'delete_cluster',
]
@factory
def cases():
return case_factory(DeleteAfterDeploy)

View File

@ -12,16 +12,17 @@
# License for the specific language governing permissions and limitations
# under the License.
from proboscis import factory
from proboscis.asserts import assert_true
from system_test import testcase
from system_test.tests import actions_base
from system_test.helpers.utils import case_factory
from system_test.helpers.decorators import deferred_decorator
from system_test.helpers.decorators import make_snapshot_if_step_fail
from system_test.helpers.decorators import action
@testcase(groups=['system_test',
'system_test.deploy_and_check_radosgw'])
class DeployCheckRadosGW(actions_base.ActionsBase):
"""Deploy cluster and check RadosGW
@ -38,8 +39,6 @@ class DeployCheckRadosGW(actions_base.ActionsBase):
"""
base_group = ['system_test',
'system_test.deploy_and_check_radosgw']
actions_order = [
'setup_master',
'config_release',
@ -72,8 +71,3 @@ class DeployCheckRadosGW(actions_base.ActionsBase):
'client.radosgw.gateway"')['stdout']) == 3
with self.fuel_web.get_ssh_for_node('slave-01') as remote:
assert_true(radosgw_started(remote), 'radosgw daemon started')
@factory
def cases():
return case_factory(DeployCheckRadosGW)

View File

@ -14,19 +14,20 @@
from devops.helpers.helpers import icmp_ping
from devops.helpers.helpers import wait
from proboscis import factory
from proboscis.asserts import assert_equal
from fuelweb_test import logger
from fuelweb_test.helpers import checkers
from system_test import testcase
from system_test.helpers.decorators import action
from system_test.helpers.decorators import deferred_decorator
from system_test.helpers.decorators import make_snapshot_if_step_fail
from system_test.helpers.utils import case_factory
from system_test.tests.actions_base import ActionsBase
from system_test.tests.actions_base import FuelMasterActions
@testcase(groups=['system_test', 'system_test.fuel_migration'])
class FuelMasterMigrate(ActionsBase, FuelMasterActions):
"""Fuel master migration to VM
@ -41,7 +42,6 @@ class FuelMasterMigrate(ActionsBase, FuelMasterActions):
8. Run OSTF
"""
base_group = ['system_test', 'system_test.fuel_migration']
actions_order = [
'setup_master',
'config_release',
@ -128,8 +128,3 @@ class FuelMasterMigrate(ActionsBase, FuelMasterActions):
wait(lambda: not remote.exists("/notready"),
timeout=900,
timeout_msg=("File wasn't removed in 900 sec"))
@factory
def cases():
return case_factory(FuelMasterMigrate)

View File

@ -12,11 +12,11 @@
# License for the specific language governing permissions and limitations
# under the License.
from system_test import testcase
from system_test.tests import actions_base
from system_test.helpers.utils import case_factory
from proboscis import factory
@testcase(groups=['system_test', 'system_test.redeploy_after_reset'])
class RedeployAfterReset(actions_base.ActionsBase):
"""Case deploy Environment
@ -29,7 +29,6 @@ class RedeployAfterReset(actions_base.ActionsBase):
6. Run OSTF
"""
base_group = ['system_test', 'system_test.redeploy_after_reset']
actions_order = [
'prepare_admin_node_with_slaves',
'create_env',
@ -44,8 +43,3 @@ class RedeployAfterReset(actions_base.ActionsBase):
'network_check',
'health_check',
]
@factory
def cases():
return case_factory(RedeployAfterReset)

View File

@ -12,11 +12,11 @@
# License for the specific language governing permissions and limitations
# under the License.
from system_test import testcase
from system_test.tests import actions_base
from system_test.helpers.utils import case_factory
from proboscis import factory
@testcase(groups=['system_test', 'system_test.redeploy_after_stop'])
class RedeployAfterStop(actions_base.ActionsBase):
"""Case deploy Environment
@ -29,7 +29,6 @@ class RedeployAfterStop(actions_base.ActionsBase):
6. Run OSTF
"""
base_group = ['system_test', 'system_test.redeploy_after_stop']
actions_order = [
'prepare_admin_node_with_slaves',
'create_env',
@ -41,8 +40,3 @@ class RedeployAfterStop(actions_base.ActionsBase):
'network_check',
'health_check',
]
@factory
def cases():
return case_factory(RedeployAfterStop)

View File

@ -13,13 +13,12 @@
# under the License.
from proboscis import factory
from proboscis.asserts import assert_true
from system_test import testcase
from system_test import logger
from system_test.helpers.decorators import make_snapshot_if_step_fail
from system_test.helpers.decorators import deferred_decorator
from system_test.helpers.decorators import action
from system_test.helpers.utils import case_factory
from system_test.tests.actions_base import ActionsBase
from fuelweb_test.settings import DVS_PLUGIN_PATH
from fuelweb_test.settings import DVS_PLUGIN_VERSION
@ -186,6 +185,9 @@ class VMwareActions(ActionsBase):
self.scale_step += 1
@testcase(groups=['system_test',
'system_test.vcenter',
'system_test.vcenter.deploy_vcenter_dvs_run_ostf'])
class DeployWithVMware(VMwareActions):
"""Deploy cluster with vCenter and dvs plugin
@ -203,10 +205,6 @@ class DeployWithVMware(VMwareActions):
Snapshot deploy_vcenter_dvs
"""
base_group = ['system_test',
'system_test.vcenter',
'system_test.vcenter.deploy_vcenter_dvs_run_ostf']
plugin_name = "fuel-plugin-vmware-dvs"
plugin_path = DVS_PLUGIN_PATH
plugin_version = DVS_PLUGIN_VERSION
@ -223,6 +221,9 @@ class DeployWithVMware(VMwareActions):
]
@testcase(groups=['system_test',
'system_test.vcenter',
'system_test.vcenter.scale_vcenter_dvs'])
class ScaleWithVMware(VMwareActions):
"""Deploy and scale cluster with vCenter and dvs plugin
@ -243,10 +244,6 @@ class ScaleWithVMware(VMwareActions):
Snapshot scale_vcenter_dvs
"""
base_group = ['system_test',
'system_test.vcenter',
'system_test.vcenter.scale_vcenter_dvs']
plugin_name = "fuel-plugin-vmware-dvs"
plugin_path = DVS_PLUGIN_PATH
plugin_version = DVS_PLUGIN_VERSION
@ -264,9 +261,3 @@ class ScaleWithVMware(VMwareActions):
'deploy_cluster',
'health_check_sanity_smoke_ha'
]
@factory
def cases():
return (case_factory(DeployWithVMware) +
case_factory(ScaleWithVMware))

View File

@ -425,11 +425,11 @@ RunTest() {
# run python test set to create environments, deploy and test product
if [ "${DRY_RUN}" = "yes" ]; then
echo export PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${WORKSPACE}"
echo python fuelweb_test/run_tests.py -q --nologcapture --with-xunit ${OPTS}
echo python run_system_test.py run -q --nologcapture --with-xunit ${OPTS}
else
export PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${WORKSPACE}"
echo ${PYTHONPATH}
python fuelweb_test/run_tests.py -q --nologcapture --with-xunit ${OPTS}
python run_system_test.py run -q --nologcapture --with-xunit ${OPTS}
fi
ec=$?