diff --git a/fuelweb_test/run_tests.py b/fuelweb_test/run_tests.py deleted file mode 100644 index d2c249b07..000000000 --- a/fuelweb_test/run_tests.py +++ /dev/null @@ -1,184 +0,0 @@ -# Copyright 2015 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -import re -import sys - -from nose.plugins import Plugin -from paramiko.transport import _join_lingering_threads - - -class CloseSSHConnectionsPlugin(Plugin): - """Closes all paramiko's ssh connections after each test case - - Plugin fixes proboscis disability to run cleanup of any kind. - 'afterTest' calls _join_lingering_threads function from paramiko, - which stops all threads (set the state to inactive and joins for 10s) - """ - name = 'closesshconnections' - - def options(self, parser, env=os.environ): - super(CloseSSHConnectionsPlugin, self).options(parser, env=env) - - def configure(self, options, conf): - super(CloseSSHConnectionsPlugin, self).configure(options, conf) - self.enabled = True - - def afterTest(self, *args, **kwargs): - _join_lingering_threads() - - -def import_tests(): - from tests import test_admin_node # noqa - from tests import test_backup_restore # noqa - from tests import test_ceph # noqa - from tests import test_environment_action # noqa - from tests import test_ironic_base # noqa - from tests import test_neutron # noqa - from tests import test_neutron_public # noqa - from tests import test_neutron_tun # noqa - from tests import test_neutron_ipv6 # noqa - from tests import test_pullrequest # noqa - from tests import test_services # noqa - from tests import test_ha_one_controller # noqa - from tests import test_vcenter # noqa - from tests import test_reduced_footprint # noqa - from tests.tests_cli import test_cli_role # noqa - from tests.tests_cli import test_cli_deploy # noqa - from tests.tests_cli import test_cli_deploy_ceph # noqa - from tests.tests_deployments.tests_neutron_vlan import test_ha_vlan_group_1 # noqa - from tests.tests_deployments.tests_neutron_vlan import test_ha_vlan_group_2 # noqa - from tests.tests_deployments.tests_neutron_vlan import test_ha_vlan_group_3 # noqa - from tests.tests_deployments.tests_neutron_vlan import test_ha_vlan_group_4 # noqa - from tests.tests_deployments.tests_neutron_vlan import test_ha_vlan_group_5 # noqa - from tests.tests_deployments.tests_neutron_vlan import test_ha_vlan_group_6 # noqa - from tests.tests_deployments.tests_neutron_vlan import test_ha_vlan_group_7 # noqa - from tests.tests_deployments.tests_neutron_tun import test_ha_tun_group_1 # noqa - from tests.tests_deployments.tests_neutron_tun import test_ha_tun_group_2 # noqa - from tests.tests_deployments.tests_neutron_tun import test_ha_tun_group_3 # noqa - from tests.tests_multirole import test_multirole_group_1 # noqa - from tests.tests_scale import test_scale_group_1 # noqa - from tests.tests_scale import test_scale_group_2 # noqa - from tests.tests_scale import test_scale_group_3 # noqa - from tests.tests_scale import test_scale_group_4 # noqa - from tests.tests_scale import test_scale_group_5 # noqa - from tests.tests_scale import test_scale_group_6 # noqa - from tests.tests_multirole import test_mongo_multirole # noqa - from tests import test_rh_compute # noqa - from tests.tests_security import test_run_nessus # noqa - from tests.tests_separate_services import test_separate_db # noqa - from tests.tests_separate_services import test_separate_horizon # noqa - from tests.tests_separate_services import test_separate_keystone # noqa - from tests.tests_separate_services import test_separate_multiroles # noqa - from tests.tests_separate_services import test_separate_rabbitmq # noqa - from tests.tests_separate_services import test_separate_db_ceph # noqa - from tests.tests_separate_services import test_separate_keystone_ceph # noqa - from tests.tests_separate_services import test_separate_rabbitmq_ceph # noqa - from tests import test_clone_env # noqa - from tests import test_node_reassignment # noqa - from tests import test_os_upgrade # noqa - from tests.tests_strength import test_failover # noqa - from tests.tests_strength import test_failover_with_ceph # noqa - from tests.tests_strength import test_master_node_failover # noqa - from tests.tests_strength import test_ostf_repeatable_tests # noqa - from tests.tests_strength import test_restart # noqa - from tests.tests_strength import test_huge_environments # noqa - from tests.tests_strength import test_image_based # noqa - from tests.tests_strength import test_cic_maintenance_mode # noqa - from tests.tests_upgrade import test_upgrade # noqa - from tests.tests_upgrade import test_upgrade_chains # noqa - from tests import test_bonding # noqa - from tests import test_offloading_types # noqa - from tests import test_bond_offloading # noqa - from tests.tests_strength import test_neutron # noqa - from tests.plugins.plugin_emc import test_plugin_emc # noqa - from tests.plugins.plugin_elasticsearch import test_plugin_elasticsearch # noqa - from tests.plugins.plugin_example import test_fuel_plugin_example # noqa - from tests.plugins.plugin_example import test_fuel_plugin_example_postdeploy # noqa - from tests.plugins.plugin_contrail import test_fuel_plugin_contrail # noqa - from tests.plugins.plugin_glusterfs import test_plugin_glusterfs # noqa - from tests.plugins.plugin_influxdb import test_plugin_influxdb # noqa - from tests.plugins.plugin_lbaas import test_plugin_lbaas # noqa - from tests.plugins.plugin_lma_collector import test_plugin_lma_collector # noqa - from tests.plugins.plugin_lma_infra_alerting import test_plugin_lma_infra_alerting # noqa - from tests.plugins.plugin_reboot import test_plugin_reboot_task # noqa - from tests.plugins.plugin_vip_reservation import test_plugin_vip_reservation # noqa - from tests.plugins.plugin_zabbix import test_plugin_zabbix # noqa - from tests import test_multiple_networks # noqa - from tests.gd_based_tests import test_neutron # noqa - from tests.gd_based_tests import test_neutron_vlan_ceph_mongo # noqa - from tests.tests_patching import test_patching # noqa - from tests import test_cli # noqa - from tests import test_custom_hostname # noqa - from tests import test_jumbo_frames # noqa - from tests import test_node_reinstallation # noqa - from tests import test_ubuntu_bootstrap # noqa - from tests import test_centos_bootstrap # noqa - from tests import test_net_templates # noqa - from tests.tests_mirrors import test_create_mirror # noqa - from tests.tests_mirrors import test_use_mirror # noqa - from system_test.tests import test_create_deploy_ostf # noqa - from system_test.tests import test_deploy_check_rados # noqa - from system_test.tests import test_redeploy_after_stop # noqa - from system_test.tests import test_redeploy_after_reset # noqa - from system_test.tests import test_delete_after_deploy # noqa - from system_test.tests.strength import destroy_controllers # noqa - from system_test.tests.strength import filling_root # noqa - from system_test.tests import test_fuel_migration # noqa - from system_test.tests.plugins.plugin_example import test_plugin_example # noqa - from system_test.tests.plugins.plugin_example import test_plugin_example_v3 # noqa - from system_test.tests.vcenter import test_vcenter_dvs # noqa - from gates_tests.tests import test_review_in_fuel_agent # noqa - from gates_tests.tests import test_review_fuel_web # noqa - from tests.tests_strength import test_load # noqa - from tests import test_services_reconfiguration # noqa - from gates_tests.tests import test_review_in_ostf # noqa - from gates_tests.tests import test_review_in_fuel_client # noqa - from tests.tests_os_components import test_murano_os_component # noqa - from tests.tests_os_components import test_sahara_os_component # noqa - from tests.tests_os_components import test_mixed_os_components # noqa - from tests.tests_strength import test_failover_group_1 # noqa - from tests.tests_strength import test_failover_mongo # noqa - from tests.tests_strength import test_failover_group_2 # noqa - from gates_tests.tests import test_review_in_astute # noqa - - -def run_tests(): - from proboscis import TestProgram # noqa - - # Check if the specified test group starts any test case - if not TestProgram().cases: - from fuelweb_test import logger - logger.fatal('No test cases matched provided groups') - sys.exit(1) - - # Run Proboscis and exit. - TestProgram( - addplugins=[CloseSSHConnectionsPlugin()] - ).run_and_exit() - - -if __name__ == '__main__': - from system_test import define_custom_groups - - import_tests() - define_custom_groups() - from fuelweb_test.helpers.patching import map_test - if any(re.search(r'--group=patching_master_tests', arg) - for arg in sys.argv): - map_test('master') - elif any(re.search(r'--group=patching.*', arg) for arg in sys.argv): - map_test('environment') - run_tests() diff --git a/fuelweb_test/testrail/upload_cases_description.py b/fuelweb_test/testrail/upload_cases_description.py index c2902be86..c1e3863cb 100644 --- a/fuelweb_test/testrail/upload_cases_description.py +++ b/fuelweb_test/testrail/upload_cases_description.py @@ -20,8 +20,11 @@ from proboscis import TestPlan from proboscis.decorators import DEFAULT_REGISTRY from builds import Build -from fuelweb_test.run_tests import import_tests from system_test import define_custom_groups +from system_test import discover_import_tests +from system_test import register_system_test_cases +from system_test import tests_directory +from system_test.helpers.utils import get_basepath from settings import GROUPS_TO_EXPAND from settings import logger from settings import TestRailSettings @@ -31,8 +34,10 @@ from testrail_client import TestRailProject def get_tests_descriptions(milestone_id, tests_include, tests_exclude, groups, default_test_priority): from system_test.tests.actions_base import ActionsBase - import_tests() + discover_import_tests(get_basepath(), tests_directory) define_custom_groups() + for one in groups: + register_system_test_cases(one) plan = TestPlan.create_from_registry(DEFAULT_REGISTRY) all_plan_tests = plan.tests[:] diff --git a/fuelweb_test/tests/tests_security/test_run_nessus.py b/fuelweb_test/tests/tests_security/test_run_nessus.py index ad54734be..d99db9605 100644 --- a/fuelweb_test/tests/tests_security/test_run_nessus.py +++ b/fuelweb_test/tests/tests_security/test_run_nessus.py @@ -17,7 +17,7 @@ from devops.helpers.helpers import tcp_ping from devops.helpers.helpers import wait import netaddr from proboscis import test -from settings import LOGS_DIR +from fuelweb_test.settings import LOGS_DIR from fuelweb_test.helpers import decorators from fuelweb_test.helpers import nessus diff --git a/run_system_test.py b/run_system_test.py new file mode 100755 index 000000000..a7fc8398d --- /dev/null +++ b/run_system_test.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python + +import sys +import argparse + +from proboscis import TestProgram +from proboscis import register + +from fuelweb_test.helpers.utils import pretty_log + +from system_test import register_system_test_cases +from system_test import get_groups +from system_test import define_custom_groups +from system_test import discover_import_tests +from system_test import tests_directory + +from system_test.helpers.utils import collect_yamls +from system_test.helpers.utils import get_path_to_config +from system_test.helpers.utils import get_list_confignames +from system_test.helpers.utils import get_basepath + +basedir = get_basepath() + + +def print_explain(names): + groups_nums = get_groups() + if not isinstance(names, list): + names = [names] + out = [] + for name in names: + for i in groups_nums[name]: + if hasattr(i, 'home'): + out.append((i.home._proboscis_entry_.parent.home, i.home)) + else: + out.append(i) + print(pretty_log(out)) + + +def clean_argv(): + """Removing argv params unused by Proboscis""" + argv = sys.argv + if '--with-config' in argv: + idx = argv.index('--with-config') + argv.pop(idx) + argv.pop(idx) + if '--explain' in argv: + idx = argv.index('--explain') + argv.pop(idx) + + return argv + + +def cli(): + cli = argparse.ArgumentParser(prog="System test runner", + description="Command line tool for run Fuel " + "System Test") + + commands = cli.add_subparsers(title="Operation commands", + dest="command") + + cli_run = commands.add_parser('run', + help="Run test", + description="Run some test group") + + cli_run.add_argument("run_groups", nargs='*', default=None, ) + cli_run.add_argument("--with-config", default=False, type=str, + action="store", dest="config_name", + help="Select name of yaml config.") + cli_run.add_argument("--explain", default=False, action="store_true", + help="Show explain for running groups. " + "Will not start Proboscis.") + cli_run.add_argument("--show-plan", default=False, action="store_true", + help="Show Proboscis test plan.") + cli_run.add_argument("--with-xunit", default=False, action="store_true", + help="Use xuint report.") + cli_run.add_argument("--nologcapture", default=False, action="store_true", + help="Disable log capture for Proboscis.") + cli_run.add_argument("-q", default=False, action="store_true", + dest="quite", + help="Run Proboscis in quite mode.") + cli_run.add_argument("-a", default=False, action="store_true", + dest="nose_attr", + help="Provide Nose attr to Proboscis.") + cli_run.add_argument("-A", default=False, action="store_true", + dest="eval_nose", + help="Eval Nose attr to Proboscis.") + cli_run.add_argument("--groups", default=None, action="append", type=str, + help="Test group for testing. " + "(backward compatibility)") + + cli_explain_group = commands.add_parser("explain-group", + help="Explain selected group.") + cli_explain_group.add_argument("name", + help="Group name.") + + commands.add_parser("show-all-groups", + help="Show all Proboscis groups") + commands.add_parser("show-fuelweb-groups", + help="Show Proboscis groups defined in fuelweb suite") + commands.add_parser("show-systest-groups", + help="Show Proboscis groups defined in Systest suite") + commands.add_parser("show-systest-configs", + help="Show configurations for Systest suite") + + if len(sys.argv) == 1: + cli.print_help() + sys.exit(1) + + return cli.parse_args() + + +def run(**kwargs): + config_name = kwargs.get('config_name', None) + groups = kwargs.get('run_groups', None) + old_groups = kwargs.get('groups', None) + explain = kwargs.get('explain', None) + + groups_to_run = [] + groups.extend(old_groups or []) + for g in groups: + if config_name: + register_system_test_cases( + groups=[g], + configs=[config_name]) + groups_to_run.append("{0}({1})".format(g, config_name)) + else: + register_system_test_cases(groups=[g]) + groups_to_run.append(g) + if explain: + print_explain(groups) + else: + register(groups=["run_system_test"], depends_on_groups=groups_to_run) + TestProgram(groups=['run_system_test'], + argv=clean_argv()).run_and_exit() + + +def explain_group(**kwargs): + """Explain selected group.""" + name = kwargs.get('name', None) + print_explain(name) + + +def show_all_groups(**kwargs): + """Show all Proboscis groups""" + groups_nums = get_groups() + + out = {k: len(v) for k, v in groups_nums.iteritems()} + print(pretty_log(out)) + + +def show_fuelweb_groups(**kwargs): + """Show Proboscis groups defined in fuelweb suite""" + groups_nums = get_groups() + + out = {k: len(v) for k, v in groups_nums.iteritems() + if not k.startswith('system_test')} + print(pretty_log(out)) + + +def show_systest_groups(**kwargs): + """Show Proboscis groups defined in Systest suite""" + groups_nums = get_groups() + + out = {k: len(v) for k, v in groups_nums.iteritems() + if k.startswith('system_test')} + print(pretty_log(out)) + + +def show_systest_configs(**kwargs): + """Show configurations for Systest suite""" + tests_configs = collect_yamls(get_path_to_config()) + + for c in get_list_confignames(tests_configs): + print(c) + + +COMMAND_MAP = { + "run": run, + "explain-group": explain_group, + "show-all-groups": show_all_groups, + "show-fuelweb-groups": show_fuelweb_groups, + "show-systest-groups": show_systest_groups, + "show-systest-configs": show_systest_configs +} + + +def shell(): + args = cli() + discover_import_tests(basedir, tests_directory) + define_custom_groups() + COMMAND_MAP[args.command](**vars(args)) + + +if __name__ == '__main__': + shell() diff --git a/system_test/__init__.py b/system_test/__init__.py index 9c13b247c..3b25d8429 100644 --- a/system_test/__init__.py +++ b/system_test/__init__.py @@ -11,14 +11,35 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import re -import fuelweb_test - -from system_test.helpers.utils import get_configs +from fuelweb_test import logger +import proboscis.core from proboscis import register +from proboscis import factory -logger = fuelweb_test.logger +from system_test.core.repository import Repository +from system_test.core.discover import discover_import_tests +from system_test.helpers.utils import get_configs +from system_test.helpers.decorators import testcase + +from proboscis.decorators import DEFAULT_REGISTRY + +from system_test.tests import base_actions_factory +from system_test.helpers.utils import config_filter + +tests_directory = [ + 'fuelweb_test/tests', + 'system_test/tests' +] + +__all__ = [ + Repository, + discover_import_tests, + testcase, + get_configs, + logger] def cached_add_group(yamls): @@ -35,6 +56,8 @@ def cached_add_group(yamls): if validate_config and config_name not in yamls: raise NameError("Config {} not found".format(config_name)) + register_system_test_cases(groups=[systest_group], + configs=[config_name]) register(groups=[group], depends_on_groups=[ "{systest_group}({config_name})".format( @@ -71,3 +94,108 @@ def define_custom_groups(): add_group(group="fuel_master_migrate", systest_group="system_test.fuel_migration", config_name="1ctrl_1comp_neutronTUN") + + +def get_groups(only_groups=None, exclude=None): + """Get groups from Proboscis register and count them children""" + groups_childs = {} + groups = {} + + if only_groups and isinstance(only_groups, list): + groups = {g: DEFAULT_REGISTRY.groups[g] + for g in DEFAULT_REGISTRY.groups if g in only_groups} + groups.update({g: Repository.index[g] + for g in Repository.index if g in only_groups}) + else: + groups = DEFAULT_REGISTRY.groups.copy() + groups.update({g: Repository.index[g] for g in Repository.index}) + + for group_name, group in groups.items(): + klass_entries = set() + entries_in_class = set() + + if (exclude and + isinstance(exclude, list) and + any([e in group_name for e in exclude])): + continue + + if hasattr(group, 'entries'): + for entry in group.entries: + if isinstance(entry, proboscis.core.TestMethodClassEntry): + klass_entries.add(entry) + + for klass in klass_entries: + entries_in_class.update(set(klass.children)) + + child = set(group.entries) - entries_in_class - klass_entries + + for klass in klass_entries: + if (klass.used_by_factory and + base_actions_factory.BaseActionsFactory in + klass.home.__mro__): + child.add(klass) + else: + child.update(set(klass.children)) + else: + child = [g for g in group + if base_actions_factory.BaseActionsFactory in g.__mro__] + + groups_childs[group_name] = child + + return groups_childs + + +def case_factory(baseclass, configs): + """Return list of instance """ + # configs = get_configs() + return [baseclass.caseclass_factory(g)(c) + for g, c in config_filter(configs).items()] + + +def case_filter(groups=None): + """Create Proboscis factories for selected groups. For all by default""" + if groups is None: + return set(Repository) + + cases = set() + for g in groups: + if g in Repository.index: + cases.update(Repository.index[g]) + return cases + + +def reg_factory(cases, configs): + def ret(): + out = [] + for c in cases: + out.extend(case_factory(c, configs)) + return out + globals()['system_test_factory'] = factory(ret) + + +def split_group_config(group): + m = re.search('([\w\.]*)\((\w*)\)', group) + if m: + return m.groups() + + +def register_system_test_cases(groups=None, configs=None): + to_remove = [] + to_add = [] + for group in groups: + g_c = split_group_config(group) + if g_c: + g, c = g_c + to_add.append(g) + if configs is None: + configs = [] + configs.append(c) + to_remove.append(group) + for one in to_remove: + groups.remove(one) + for one in to_add: + groups.append(one) + cases = case_filter(groups) + configs = config_filter(configs) + if cases: + reg_factory(cases, configs) diff --git a/system_test/core/__init__.py b/system_test/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/system_test/core/discover.py b/system_test/core/discover.py new file mode 100644 index 000000000..843ae1af3 --- /dev/null +++ b/system_test/core/discover.py @@ -0,0 +1,45 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os.path + + +def discover_test_files(basedir, dirs): + """Find all files in path""" + ret = [] + for path in dirs: + path = os.path.join(basedir, path) + for r, d, f in os.walk(path): + for one in f: + if one.startswith('test_') and one.endswith('.py'): + ret.append(os.path.join(r, one)) + return ret + + +def convert_files_to_modules(basedir, files): + """Convert files name to modules name""" + ret = [] + for one in files: + module = os.path.splitext( + os.path.relpath(one, basedir))[0].replace('/', '.') + ret.append(module) + return ret + + +def discover_import_tests(basedir, dirs): + """Walk through directories and import all modules with tests""" + imported_list = [] + for module in convert_files_to_modules(basedir, + discover_test_files(basedir, dirs)): + imported_list.append(__import__(module)) diff --git a/system_test/core/repository.py b/system_test/core/repository.py new file mode 100644 index 000000000..a4fd11cc9 --- /dev/null +++ b/system_test/core/repository.py @@ -0,0 +1,76 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from fuelweb_test.helpers import metaclasses + + +class TestCaseRepository(set): + + __metaclass__ = metaclasses.SingletonMeta + + def __init__(self): + super(TestCaseRepository, self).__init__() + self.__index = {} + + @property + def index(self): + return self.__index + + def __index_add(self, v): + groups = getattr(v, '_base_groups', None) + for g in groups: + if g not in self.__index: + self.__index[g] = set() + self.__index[g].add(v) + + def __index_remove(self, v): + groups = getattr(v, '_base_groups', None) + for g in groups: + self.__index[g].remove(v) + if not len(self.__index[g]): + del self.__index[g] + + def add(self, value): + super(TestCaseRepository, self).add(value) + self.__index_add(value) + + def remove(self, value): + super(TestCaseRepository, self).remove(value) + self.__index_remove(value) + + def pop(self, value): + super(TestCaseRepository, self).pop(value) + self.__index_remove(value) + + def filter(self, groups=None): + """Return list of cases related to groups. All by default""" + if groups is None: + return set(self) + + cases = set() + for g in groups: + if g in self.index: + cases.update(self.index[g]) + return cases + + def union(self, *args, **kwargs): + raise AttributeError("'TestCaseRepository' object has no attribute " + " 'union'") + + def update(self, *args, **kwargs): + raise AttributeError("'TestCaseRepository' object has no attribute " + " 'update'") + + +Repository = TestCaseRepository() diff --git a/system_test/helpers/decorators.py b/system_test/helpers/decorators.py index f3d840fae..b1357a1c3 100644 --- a/system_test/helpers/decorators.py +++ b/system_test/helpers/decorators.py @@ -16,13 +16,15 @@ import functools import traceback import sys import hashlib +import inspect +import collections from proboscis import SkipTest -from fuelweb_test.helpers.utils import TimeStat from fuelweb_test.helpers.utils import pull_out_logs_via_ssh from fuelweb_test.helpers.decorators import create_diagnostic_snapshot +from system_test import Repository from system_test import logger @@ -43,26 +45,6 @@ def nested_action(method): return staticmethod(method) -def step_start_stop(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - with TimeStat(func) as timer: - step_name = getattr(func, '_step_name') - start_step = '[ START {} ]'.format(step_name) - header = "<<< {:-^142} >>>".format(start_step) - logger.info("\n{header}\n".format(header=header)) - result = func(*args, **kwargs) - spent_time = timer.spent_time - minutes = int(round(spent_time)) / 60 - seconds = int(round(spent_time)) % 60 - finish_step = "[ FINISH {} STEP TOOK {} min {} sec ]".format( - step_name, minutes, seconds) - footer = "<<< {:-^142} >>>".format(finish_step) - logger.info("\n{footer}\n".format(footer=footer)) - return result - return wrapper - - def make_snapshot_if_step_fail(func): """Generate diagnostic snapshot if step fail. @@ -122,3 +104,18 @@ def make_snapshot_if_step_fail(func): raise test_exception, None, exc_trace return result return wrapper + + +def testcase(groups): + """Use this decorator for mark a test case class""" + def testcase_decorator(cls): + if not inspect.isclass(cls): + raise TypeError("Decorator @testcase should used only " + "with classes") + if not isinstance(groups, collections.Sequence): + raise TypeError("Use list for groups") + cls.get_actions_order() + setattr(cls, '_base_groups', groups) + Repository.add(cls) + return cls + return testcase_decorator diff --git a/system_test/helpers/utils.py b/system_test/helpers/utils.py index ab6a73268..fcf6cc9f5 100644 --- a/system_test/helpers/utils.py +++ b/system_test/helpers/utils.py @@ -33,6 +33,12 @@ def copy_func(f, name=None): return fn +def get_basepath(): + import system_test + return os.path.join( + os.path.dirname(os.path.dirname(system_test.__file__))) + + def get_list_confignames(filelist): """Get list of config name from file list""" return map(get_configname, filelist) @@ -115,7 +121,7 @@ def find_duplicates(yamls): dup[name].append(one) else: dup[name] = [one] - return {k: v for k, v in dup.iteritems() if len(v) > 1} + return {k: v for k, v in dup.items() if len(v) > 1} def get_configs(): @@ -129,7 +135,7 @@ def get_configs(): return {get_configname(y): y for y in yamls} -def case_factory(baseclass): - """Return list of instance """ - configs = get_configs() - return [baseclass.caseclass_factory(g)(c) for g, c in configs.iteritems()] +def config_filter(configs=None): + if configs is None: + return get_configs() + return {k: v for k, v in get_configs().items() if k in configs} diff --git a/system_test/tests/base_actions_factory.py b/system_test/tests/base_actions_factory.py index 986aea91c..9d8a49615 100644 --- a/system_test/tests/base_actions_factory.py +++ b/system_test/tests/base_actions_factory.py @@ -12,14 +12,37 @@ # License for the specific language governing permissions and limitations # under the License. -from proboscis import test +import functools + from proboscis import after_class from proboscis import before_class +from proboscis import test +from fuelweb_test.helpers.utils import TimeStat from fuelweb_test.tests import base_test_case from system_test.helpers import utils -from system_test.helpers.decorators import step_start_stop +from system_test import logger + + +def step_start_stop(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + with TimeStat(func) as timer: + step_name = getattr(func, '_step_name') + start_step = '[ START {} ]'.format(step_name) + header = "<<< {:-^142} >>>".format(start_step) + logger.info("\n{header}\n".format(header=header)) + result = func(*args, **kwargs) + spent_time = timer.spent_time + minutes = int(round(spent_time)) / 60 + seconds = int(round(spent_time)) % 60 + finish_step = "[ FINISH {} STEP TOOK {} min {} sec ]".format( + step_name, minutes, seconds) + footer = "<<< {:-^142} >>>".format(finish_step) + logger.info("\n{footer}\n".format(footer=footer)) + return result + return wrapper class BaseActionsFactory(base_test_case.TestBasic): @@ -28,7 +51,7 @@ class BaseActionsFactory(base_test_case.TestBasic): def get_actions(cls): """Return all action methods""" return {m: getattr(cls, m) for m in - dir(cls) if m.startswith('_action_') or + dir(cls) if getattr(getattr(cls, m), '_action_method_', False) or getattr(getattr(cls, m), '_nested_action_method_', False)} @@ -41,9 +64,29 @@ class BaseActionsFactory(base_test_case.TestBasic): actions_method = cls.get_actions() linear_order = [] for action in cls.actions_order: - if getattr(actions_method[action], + try: + action_method = actions_method[action] + except KeyError: + import inspect + source = inspect.getsourcelines(inspect.getmodule(cls))[0] + counted_data = [n for n in enumerate(source)] + line_num = [n for (n, l) in counted_data if 'ddd' in l][0] + cutted = counted_data[line_num - 4:line_num + 4] + cutted = [(n, l[:-1] + " " * 20 + "<====\n" + if n == line_num else l) + for (n, l) in cutted] + cutted = ["Line {line_num:04d}: {line}".format( + line_num=n, line=l) for (n, l) in cutted] + raise LookupError("Class {} orders to run '{}' action as {} " + "step,\n\tbut action method doesn't exist " + "in class.\nLook at '{}':\n\n{}".format( + cls, action, + cls.actions_order.index(action), + inspect.getsourcefile(cls), + ''.join(cutted))) + if getattr(action_method, '_nested_action_method_', None): - linear_order.extend(actions_method[action]()) + linear_order.extend(action_method()) else: linear_order.append(action) @@ -140,8 +183,8 @@ class BaseActionsFactory(base_test_case.TestBasic): runs_after=[teardown_method] if teardown_method else []) # Generate test case groups - groups = ['{}({})'.format(g, case_group) for g in cls.base_group] - groups = cls.base_group + groups + groups = ['{}({})'.format(g, case_group) for g in cls._base_groups] + groups = cls._base_groups + groups # Generate test case docstring test_steps["__doc__"] = "{}\n\n{}\n\nDuration {}".format( diff --git a/system_test/tests/plugins/plugin_example/test_plugin_example.py b/system_test/tests/plugins/plugin_example/test_plugin_example.py index 2f3c7bca4..6ec1c30ab 100644 --- a/system_test/tests/plugins/plugin_example/test_plugin_example.py +++ b/system_test/tests/plugins/plugin_example/test_plugin_example.py @@ -12,13 +12,15 @@ # License for the specific language governing permissions and limitations # under the License. -from proboscis import factory - +from system_test import testcase from fuelweb_test.settings import EXAMPLE_PLUGIN_PATH -from system_test.helpers.utils import case_factory from system_test.tests.actions_base import ActionsBase +@testcase(groups=['system_test', + 'system_test.plugins', + 'system_test.plugins.example_plugin', + 'system_test.plugins.example_plugin.simple']) class DeployWithPluginExample(ActionsBase): """Deploy cluster with one controller and example plugin @@ -37,11 +39,6 @@ class DeployWithPluginExample(ActionsBase): Snapshot deploy_ha_one_controller_neutron_example """ - base_group = ['system_test', - 'system_test.plugins', - 'system_test.plugins.example_plugin', - 'system_test.plugins.example_plugin.simple'] - plugin_name = "fuel_plugin_example" plugin_path = EXAMPLE_PLUGIN_PATH @@ -58,6 +55,10 @@ class DeployWithPluginExample(ActionsBase): ] +@testcase(groups=['system_test', + 'system_test.plugins', + 'system_test.plugins.example_plugin', + 'system_test.plugins.example_plugin.simple_scale']) class DeployScaleWithPluginExample(ActionsBase): """Deploy and scale cluster in ha mode with example plugin @@ -81,11 +82,6 @@ class DeployScaleWithPluginExample(ActionsBase): Snapshot deploy_neutron_example_ha_add_node """ - base_group = ['system_test', - 'system_test.plugins', - 'system_test.plugins.example_plugin', - 'system_test.plugins.example_plugin.simple_scale'] - plugin_name = "fuel_plugin_example" plugin_path = EXAMPLE_PLUGIN_PATH @@ -106,9 +102,3 @@ class DeployScaleWithPluginExample(ActionsBase): 'check_example_plugin', 'health_check', ] - - -@factory -def cases(): - return (case_factory(DeployWithPluginExample) + - case_factory(DeployScaleWithPluginExample)) diff --git a/system_test/tests/plugins/plugin_example/test_plugin_example_v3.py b/system_test/tests/plugins/plugin_example/test_plugin_example_v3.py index c028d8692..af11d7dc4 100644 --- a/system_test/tests/plugins/plugin_example/test_plugin_example_v3.py +++ b/system_test/tests/plugins/plugin_example/test_plugin_example_v3.py @@ -12,16 +12,19 @@ # License for the specific language governing permissions and limitations # under the License. -from proboscis import factory from fuelweb_test.settings import EXAMPLE_PLUGIN_V3_PATH -from system_test.helpers.utils import case_factory +from system_test import testcase from system_test.helpers.decorators import make_snapshot_if_step_fail from system_test.helpers.decorators import deferred_decorator from system_test.helpers.decorators import action from system_test.tests.actions_base import ActionsBase +@testcase(groups=['system_test', + 'system_test.plugins', + 'system_test.plugins.example_plugin_v3', + 'system_test.plugins.example_plugin_v3.simple']) class DeployWithPluginExampleV3(ActionsBase): """Deploy cluster with one controller and example plugin v3 @@ -40,10 +43,6 @@ class DeployWithPluginExampleV3(ActionsBase): Duration 35m Snapshot deploy_ha_one_controller_neutron_example_v3 """ - base_group = ['system_test', - 'system_test.plugins', - 'system_test.plugins.example_plugin_v3', - 'system_test.plugins.example_plugin_v3.simple'] plugin_name = 'fuel_plugin_example_v3' plugin_path = EXAMPLE_PLUGIN_V3_PATH @@ -69,8 +68,3 @@ class DeployWithPluginExampleV3(ActionsBase): 'roles': ['fuel_plugin_example_v3'], 'count': 1 }]) - - -@factory -def cases(): - return (case_factory(DeployWithPluginExampleV3)) diff --git a/system_test/tests/strength/destroy_controllers.py b/system_test/tests/strength/test_destroy_controllers.py similarity index 92% rename from system_test/tests/strength/destroy_controllers.py rename to system_test/tests/strength/test_destroy_controllers.py index c0d7f8459..a42ba5141 100644 --- a/system_test/tests/strength/destroy_controllers.py +++ b/system_test/tests/strength/test_destroy_controllers.py @@ -12,15 +12,17 @@ # License for the specific language governing permissions and limitations # under the License. -from system_test.helpers.utils import case_factory -from proboscis import factory - +from system_test import testcase from system_test.tests.strength import strength_base from system_test.helpers.decorators import make_snapshot_if_step_fail from system_test.helpers.decorators import deferred_decorator from system_test.helpers.decorators import action +@testcase(groups=['system_test', + 'system_test.failover', + 'system_test.failover.destroy_controllers', + 'system_test.failover.destroy_controllers.first']) class StrengthDestroyFirstController(strength_base.StrengthBaseActions): """Destroy two controllers and check pacemaker status is correct @@ -39,11 +41,6 @@ class StrengthDestroyFirstController(strength_base.StrengthBaseActions): """ - base_group = ['system_test', - 'system_test.failover', - 'system_test.failover.destroy_controllers', - 'system_test.failover.destroy_controllers.first'] - actions_order = [ 'setup_master', 'config_release', @@ -72,6 +69,10 @@ class StrengthDestroyFirstController(strength_base.StrengthBaseActions): self._destroy_controller('slave-01') +@testcase(groups=['system_test', + 'system_test.failover', + 'system_test.failover.destroy_controllers', + 'system_test.failover.destroy_controllers.second']) class StrengthDestroySecondController(strength_base.StrengthBaseActions): """Destroy two controllers and check pacemaker status is correct @@ -90,11 +91,6 @@ class StrengthDestroySecondController(strength_base.StrengthBaseActions): """ - base_group = ['system_test', - 'system_test.failover', - 'system_test.failover.destroy_controllers', - 'system_test.failover.destroy_controllers.second'] - actions_order = [ 'setup_master', 'config_release', @@ -121,9 +117,3 @@ class StrengthDestroySecondController(strength_base.StrengthBaseActions): def destroy_second_controller(self): """Destroy second controller""" self._destroy_controller('slave-02') - - -@factory -def cases(): - return (case_factory(StrengthDestroyFirstController) + - case_factory(StrengthDestroySecondController)) diff --git a/system_test/tests/strength/filling_root.py b/system_test/tests/strength/test_filling_root.py similarity index 90% rename from system_test/tests/strength/filling_root.py rename to system_test/tests/strength/test_filling_root.py index f782e5b06..c667ec145 100644 --- a/system_test/tests/strength/filling_root.py +++ b/system_test/tests/strength/test_filling_root.py @@ -12,13 +12,14 @@ # License for the specific language governing permissions and limitations # under the License. -from proboscis import factory - -from system_test.helpers.utils import case_factory +from system_test import testcase from system_test.tests.strength import strength_base +@testcase(groups=['system_test', + 'system_test.failover', + 'system_test.failover.filling_root']) class FillRootPrimaryController( strength_base.FillRootBaseActions ): @@ -51,11 +52,6 @@ class FillRootPrimaryController( 21. Run OSTF Sanity, Smoke, HA """ - base_group = ['system_test', - 'system_test.failover', - 'system_test.failover.filling_root' - ] - actions_order = [ 'setup_master', 'config_release', @@ -79,8 +75,3 @@ class FillRootPrimaryController( 'check_starting_resources', 'health_check_sanity_smoke_ha', ] - - -@factory -def cases(): - return case_factory(FillRootPrimaryController) diff --git a/system_test/tests/test_create_deploy_ostf.py b/system_test/tests/test_create_deploy_ostf.py index 1a18f1d15..5c3328e5b 100644 --- a/system_test/tests/test_create_deploy_ostf.py +++ b/system_test/tests/test_create_deploy_ostf.py @@ -12,11 +12,11 @@ # License for the specific language governing permissions and limitations # under the License. +from system_test import testcase from system_test.tests import actions_base -from system_test.helpers.utils import case_factory -from proboscis import factory +@testcase(groups=['system_test', 'system_test.create_deploy_ostf']) class CreateDeployOstf(actions_base.ActionsBase): """Case deploy Environment @@ -29,7 +29,6 @@ class CreateDeployOstf(actions_base.ActionsBase): 6. Run OSTF """ - base_group = ['system_test', 'system_test.create_deploy_ostf'] actions_order = [ 'prepare_admin_node_with_slaves', 'create_env', @@ -39,8 +38,3 @@ class CreateDeployOstf(actions_base.ActionsBase): 'network_check', 'health_check', ] - - -@factory -def cases(): - return case_factory(CreateDeployOstf) diff --git a/system_test/tests/test_delete_after_deploy.py b/system_test/tests/test_delete_after_deploy.py index addd50efa..fd342d047 100644 --- a/system_test/tests/test_delete_after_deploy.py +++ b/system_test/tests/test_delete_after_deploy.py @@ -12,11 +12,11 @@ # License for the specific language governing permissions and limitations # under the License. +from system_test import testcase from system_test.tests import actions_base -from system_test.helpers.utils import case_factory -from proboscis import factory +@testcase(groups=['system_test', 'system_test.delete_after_deploy']) class DeleteAfterDeploy(actions_base.ActionsBase): """Case deploy Environment @@ -29,7 +29,6 @@ class DeleteAfterDeploy(actions_base.ActionsBase): 6. Run OSTF """ - base_group = ['system_test', 'system_test.delete_after_deploy'] actions_order = [ 'prepare_admin_node_with_slaves', 'create_env', @@ -40,8 +39,3 @@ class DeleteAfterDeploy(actions_base.ActionsBase): 'health_check', 'delete_cluster', ] - - -@factory -def cases(): - return case_factory(DeleteAfterDeploy) diff --git a/system_test/tests/test_deploy_check_rados.py b/system_test/tests/test_deploy_check_rados.py index 561aa9867..aeebd8c4d 100644 --- a/system_test/tests/test_deploy_check_rados.py +++ b/system_test/tests/test_deploy_check_rados.py @@ -12,16 +12,17 @@ # License for the specific language governing permissions and limitations # under the License. -from proboscis import factory from proboscis.asserts import assert_true +from system_test import testcase from system_test.tests import actions_base -from system_test.helpers.utils import case_factory from system_test.helpers.decorators import deferred_decorator from system_test.helpers.decorators import make_snapshot_if_step_fail from system_test.helpers.decorators import action +@testcase(groups=['system_test', + 'system_test.deploy_and_check_radosgw']) class DeployCheckRadosGW(actions_base.ActionsBase): """Deploy cluster and check RadosGW @@ -38,8 +39,6 @@ class DeployCheckRadosGW(actions_base.ActionsBase): """ - base_group = ['system_test', - 'system_test.deploy_and_check_radosgw'] actions_order = [ 'setup_master', 'config_release', @@ -72,8 +71,3 @@ class DeployCheckRadosGW(actions_base.ActionsBase): 'client.radosgw.gateway"')['stdout']) == 3 with self.fuel_web.get_ssh_for_node('slave-01') as remote: assert_true(radosgw_started(remote), 'radosgw daemon started') - - -@factory -def cases(): - return case_factory(DeployCheckRadosGW) diff --git a/system_test/tests/test_fuel_migration.py b/system_test/tests/test_fuel_migration.py index ff0a9aaee..28ed744f4 100644 --- a/system_test/tests/test_fuel_migration.py +++ b/system_test/tests/test_fuel_migration.py @@ -14,19 +14,20 @@ from devops.helpers.helpers import icmp_ping from devops.helpers.helpers import wait -from proboscis import factory from proboscis.asserts import assert_equal from fuelweb_test import logger from fuelweb_test.helpers import checkers + +from system_test import testcase from system_test.helpers.decorators import action from system_test.helpers.decorators import deferred_decorator from system_test.helpers.decorators import make_snapshot_if_step_fail -from system_test.helpers.utils import case_factory from system_test.tests.actions_base import ActionsBase from system_test.tests.actions_base import FuelMasterActions +@testcase(groups=['system_test', 'system_test.fuel_migration']) class FuelMasterMigrate(ActionsBase, FuelMasterActions): """Fuel master migration to VM @@ -41,7 +42,6 @@ class FuelMasterMigrate(ActionsBase, FuelMasterActions): 8. Run OSTF """ - base_group = ['system_test', 'system_test.fuel_migration'] actions_order = [ 'setup_master', 'config_release', @@ -128,8 +128,3 @@ class FuelMasterMigrate(ActionsBase, FuelMasterActions): wait(lambda: not remote.exists("/notready"), timeout=900, timeout_msg=("File wasn't removed in 900 sec")) - - -@factory -def cases(): - return case_factory(FuelMasterMigrate) diff --git a/system_test/tests/test_redeploy_after_reset.py b/system_test/tests/test_redeploy_after_reset.py index 35a37f9cf..049f7ff7e 100644 --- a/system_test/tests/test_redeploy_after_reset.py +++ b/system_test/tests/test_redeploy_after_reset.py @@ -12,11 +12,11 @@ # License for the specific language governing permissions and limitations # under the License. +from system_test import testcase from system_test.tests import actions_base -from system_test.helpers.utils import case_factory -from proboscis import factory +@testcase(groups=['system_test', 'system_test.redeploy_after_reset']) class RedeployAfterReset(actions_base.ActionsBase): """Case deploy Environment @@ -29,7 +29,6 @@ class RedeployAfterReset(actions_base.ActionsBase): 6. Run OSTF """ - base_group = ['system_test', 'system_test.redeploy_after_reset'] actions_order = [ 'prepare_admin_node_with_slaves', 'create_env', @@ -44,8 +43,3 @@ class RedeployAfterReset(actions_base.ActionsBase): 'network_check', 'health_check', ] - - -@factory -def cases(): - return case_factory(RedeployAfterReset) diff --git a/system_test/tests/test_redeploy_after_stop.py b/system_test/tests/test_redeploy_after_stop.py index 7de2fd513..a04495978 100644 --- a/system_test/tests/test_redeploy_after_stop.py +++ b/system_test/tests/test_redeploy_after_stop.py @@ -12,11 +12,11 @@ # License for the specific language governing permissions and limitations # under the License. +from system_test import testcase from system_test.tests import actions_base -from system_test.helpers.utils import case_factory -from proboscis import factory +@testcase(groups=['system_test', 'system_test.redeploy_after_stop']) class RedeployAfterStop(actions_base.ActionsBase): """Case deploy Environment @@ -29,7 +29,6 @@ class RedeployAfterStop(actions_base.ActionsBase): 6. Run OSTF """ - base_group = ['system_test', 'system_test.redeploy_after_stop'] actions_order = [ 'prepare_admin_node_with_slaves', 'create_env', @@ -41,8 +40,3 @@ class RedeployAfterStop(actions_base.ActionsBase): 'network_check', 'health_check', ] - - -@factory -def cases(): - return case_factory(RedeployAfterStop) diff --git a/system_test/tests/vcenter/test_vcenter_dvs.py b/system_test/tests/vcenter/test_vcenter_dvs.py index 187fbde83..f997b06c5 100644 --- a/system_test/tests/vcenter/test_vcenter_dvs.py +++ b/system_test/tests/vcenter/test_vcenter_dvs.py @@ -13,13 +13,12 @@ # under the License. -from proboscis import factory from proboscis.asserts import assert_true +from system_test import testcase from system_test import logger from system_test.helpers.decorators import make_snapshot_if_step_fail from system_test.helpers.decorators import deferred_decorator from system_test.helpers.decorators import action -from system_test.helpers.utils import case_factory from system_test.tests.actions_base import ActionsBase from fuelweb_test.settings import DVS_PLUGIN_PATH from fuelweb_test.settings import DVS_PLUGIN_VERSION @@ -186,6 +185,9 @@ class VMwareActions(ActionsBase): self.scale_step += 1 +@testcase(groups=['system_test', + 'system_test.vcenter', + 'system_test.vcenter.deploy_vcenter_dvs_run_ostf']) class DeployWithVMware(VMwareActions): """Deploy cluster with vCenter and dvs plugin @@ -203,10 +205,6 @@ class DeployWithVMware(VMwareActions): Snapshot deploy_vcenter_dvs """ - base_group = ['system_test', - 'system_test.vcenter', - 'system_test.vcenter.deploy_vcenter_dvs_run_ostf'] - plugin_name = "fuel-plugin-vmware-dvs" plugin_path = DVS_PLUGIN_PATH plugin_version = DVS_PLUGIN_VERSION @@ -223,6 +221,9 @@ class DeployWithVMware(VMwareActions): ] +@testcase(groups=['system_test', + 'system_test.vcenter', + 'system_test.vcenter.scale_vcenter_dvs']) class ScaleWithVMware(VMwareActions): """Deploy and scale cluster with vCenter and dvs plugin @@ -243,10 +244,6 @@ class ScaleWithVMware(VMwareActions): Snapshot scale_vcenter_dvs """ - base_group = ['system_test', - 'system_test.vcenter', - 'system_test.vcenter.scale_vcenter_dvs'] - plugin_name = "fuel-plugin-vmware-dvs" plugin_path = DVS_PLUGIN_PATH plugin_version = DVS_PLUGIN_VERSION @@ -264,9 +261,3 @@ class ScaleWithVMware(VMwareActions): 'deploy_cluster', 'health_check_sanity_smoke_ha' ] - - -@factory -def cases(): - return (case_factory(DeployWithVMware) + - case_factory(ScaleWithVMware)) diff --git a/utils/jenkins/system_tests.sh b/utils/jenkins/system_tests.sh index 9825bf2d9..50d196d30 100755 --- a/utils/jenkins/system_tests.sh +++ b/utils/jenkins/system_tests.sh @@ -425,11 +425,11 @@ RunTest() { # run python test set to create environments, deploy and test product if [ "${DRY_RUN}" = "yes" ]; then echo export PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${WORKSPACE}" - echo python fuelweb_test/run_tests.py -q --nologcapture --with-xunit ${OPTS} + echo python run_system_test.py run -q --nologcapture --with-xunit ${OPTS} else export PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${WORKSPACE}" echo ${PYTHONPATH} - python fuelweb_test/run_tests.py -q --nologcapture --with-xunit ${OPTS} + python run_system_test.py run -q --nologcapture --with-xunit ${OPTS} fi ec=$?