Revert only the latest snapshot that matches the test requirements

- add pytest marks 'revert_snapshot' to all required fixtures instead
  of reverting the snapshots inside them
- add get_top_fixtures_marks() that extracts all the marks
  'revert_snapshot' from the test and it's fixtures, order the marks
  in the same way as the fixtures depends on each other,
- in the fixture 'revert_snapshot' try to find the most suitable
  snapshot for reverting, from latest to earliest.
- mark 'revert_snapshot' removed from system tests which use fixtures
  with the same mark (for example, 'k8s_deployed' mark removed from
  system tests where the fixture 'k8scluster' is used as a top fixture)

Change-Id: Iad9e56d96d870aa4204ba23e76a5b4df01d4385b
Co-Authored-By: Dmitry Tyzhnenko <dtyzhnenko@mirantis.com>
This commit is contained in:
Dennis Dmitriev 2016-11-17 13:23:29 +02:00
parent cc292bdd50
commit a736489db9
13 changed files with 90 additions and 96 deletions

View File

@ -33,6 +33,7 @@ def ccp_actions(config, underlay):
return ccpmanager.CCPManager(config, underlay)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.ccp_deployed)
@pytest.fixture(scope='function')
def ccpcluster(revert_snapshot, config, hardware,
underlay, k8scluster, ccp_actions):
@ -61,14 +62,6 @@ def ccpcluster(revert_snapshot, config, hardware,
ccp_actions.default_params = settings.CCP_CLI_PARAMS
# If no snapshot was reverted, then try to revert the snapshot
# that belongs to the fixture.
# Note: keep fixtures in strict dependences from each other!
if not revert_snapshot:
if hardware.has_snapshot(ext.SNAPSHOT.ccp_deployed) and \
hardware.has_snapshot_config(ext.SNAPSHOT.ccp_deployed):
hardware.revert_snapshot(ext.SNAPSHOT.ccp_deployed)
# Install CCP
if config.ccp.os_host == '0.0.0.0':
ccp_actions.install_ccp()

View File

@ -36,6 +36,7 @@ def k8s_actions(config, underlay):
return k8smanager.K8SManager(config, underlay)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
@pytest.fixture(scope='function')
def k8scluster(revert_snapshot, request, config,
hardware, underlay, k8s_actions):
@ -61,14 +62,6 @@ def k8scluster(revert_snapshot, request, config,
If you want to revert 'k8s_deployed' snapshot, please use mark:
@pytest.mark.revert_snapshot("k8s_deployed")
"""
# If no snapshot was reverted, then try to revert the snapshot
# that belongs to the fixture.
# Note: keep fixtures in strict dependences from each other!
if not revert_snapshot:
if hardware.has_snapshot(ext.SNAPSHOT.k8s_deployed) and \
hardware.has_snapshot_config(ext.SNAPSHOT.k8s_deployed):
hardware.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
# Create k8s cluster
if config.k8s.kube_host == '0.0.0.0':
kube_settings = getattr(request.instance, 'kube_settings',

View File

@ -21,6 +21,7 @@ from fuel_ccp_tests.managers.osmanager import OSManager
LOG = logger.logger
@pytest.mark.revert_snapshot(ext.SNAPSHOT.os_deployed)
@pytest.fixture(scope='function')
def os_deployed(ccpcluster,
hardware,
@ -30,14 +31,6 @@ def os_deployed(ccpcluster,
k8s_actions):
"""Deploy openstack
"""
# If no snapshot was reverted, then try to revert the snapshot
# that belongs to the fixture.
# Note: keep fixtures in strict dependences from each other!
if not revert_snapshot:
if hardware.has_snapshot(ext.SNAPSHOT.os_deployed) and \
hardware.has_snapshot_config(ext.SNAPSHOT.os_deployed):
hardware.revert_snapshot(ext.SNAPSHOT.os_deployed)
osmanager = OSManager(config, underlay, k8s_actions, ccpcluster)
if not config.os.running:
LOG.info("Preparing openstack log collector fixture...")

View File

@ -16,6 +16,7 @@ import pytest
from datetime import datetime
from fuel_ccp_tests.helpers import ext
from fuel_ccp_tests.helpers import utils
from fuel_ccp_tests import logger
from fuel_ccp_tests import settings
from fuel_ccp_tests.managers import envmanager_devops
@ -25,20 +26,6 @@ from fuel_ccp_tests.managers import underlay_ssh_manager
LOG = logger.logger
def extract_name_from_mark(mark):
"""Simple function to extract name from mark
:param mark: pytest.mark.MarkInfo
:rtype: string or None
"""
if mark:
if len(mark.args) > 0:
return mark.args[0]
elif 'name' in mark.kwargs:
return mark.kwargs['name']
return None
@pytest.fixture(scope="session")
def hardware(request, config):
"""Fixture for manage the hardware layer.
@ -110,17 +97,19 @@ def revert_snapshot(request, hardware):
:rtype string: name of the reverted snapshot or None
"""
revert_snapshot = request.keywords.get('revert_snapshot', None)
snapshot_name = extract_name_from_mark(revert_snapshot)
top_fixtures_snapshots = utils.get_top_fixtures_marks(
request, 'revert_snapshot')
if snapshot_name and \
hardware.has_snapshot(snapshot_name) and \
hardware.has_snapshot_config(snapshot_name):
hardware.revert_snapshot(snapshot_name)
return snapshot_name
else:
hardware.revert_snapshot(ext.SNAPSHOT.hardware)
return None
# Try to revert the best matches snapshot for the test
for snapshot_name in top_fixtures_snapshots:
if hardware.has_snapshot(snapshot_name) and \
hardware.has_snapshot_config(snapshot_name):
hardware.revert_snapshot(snapshot_name)
return snapshot_name
# Fallback to the basic snapshot
hardware.revert_snapshot(ext.SNAPSHOT.hardware)
return None
@pytest.fixture(scope='function', autouse=True)
@ -146,7 +135,7 @@ def snapshot(request, hardware):
request.node.function.__name__)
if hasattr(request.node, 'rep_call') and request.node.rep_call.passed \
and snapshot_needed:
snapshot_name = extract_name_from_mark(snapshot_needed) or \
snapshot_name = utils.extract_name_from_mark(snapshot_needed) or \
"{}_passed".format(default_snapshot_name)
hardware.create_snapshot(snapshot_name)
@ -163,6 +152,7 @@ def snapshot(request, hardware):
request.addfinalizer(test_fin)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.underlay)
@pytest.fixture(scope="function")
def underlay(revert_snapshot, config, hardware):
"""Fixture that should provide SSH access to underlay objects.
@ -179,14 +169,6 @@ def underlay(revert_snapshot, config, hardware):
- provide SSH access to underlay nodes using
node names or node IPs.
"""
# If no snapshot was reverted, then try to revert the snapshot
# that belongs to the fixture.
# Note: keep fixtures in strict dependences from each other!
if not revert_snapshot:
if hardware.has_snapshot(ext.SNAPSHOT.underlay) and \
hardware.has_snapshot_config(ext.SNAPSHOT.underlay):
hardware.revert_snapshot(ext.SNAPSHOT.underlay)
# Create Underlay
if not config.underlay.ssh:
# If config.underlay.ssh wasn't provided from external config, then

View File

@ -411,3 +411,75 @@ class YamlEditor(object):
if self.content == self.__original_content:
return
self.write_content()
def extract_name_from_mark(mark):
"""Simple function to extract name from pytest mark
:param mark: pytest.mark.MarkInfo
:rtype: string or None
"""
if mark:
if len(mark.args) > 0:
return mark.args[0]
elif 'name' in mark.kwargs:
return mark.kwargs['name']
return None
def get_top_fixtures_marks(request, mark_name):
"""Order marks according to fixtures order
When a test use fixtures that depend on each other in some order,
that fixtures can have the same pytest mark.
This method extracts such marks from fixtures that are used in the
current test and return the content of the marks ordered by the
fixture dependences.
If the test case have the same mark, than the content of this mark
will be the first element in the resulting list.
:param request: pytest 'request' fixture
:param mark_name: name of the mark to search on the fixtures and the test
:rtype list: marks content, from last to first executed.
"""
fixtureinfo = request.session._fixturemanager.getfixtureinfo(
request.node, request.function, request.cls)
top_fixtures_names = []
for _ in enumerate(fixtureinfo.name2fixturedefs):
parent_fixtures = set()
child_fixtures = set()
for name in sorted(fixtureinfo.name2fixturedefs):
if name in top_fixtures_names:
continue
parent_fixtures.add(name)
child_fixtures.update(
fixtureinfo.name2fixturedefs[name][0].argnames)
top_fixtures_names.extend(list(parent_fixtures - child_fixtures))
top_fixtures_marks = []
if mark_name in request.function.func_dict:
# The top priority is the 'revert_snapshot' mark on the test
top_fixtures_marks.append(
extract_name_from_mark(
request.function.func_dict[mark_name]))
for top_fixtures_name in top_fixtures_names:
fd = fixtureinfo.name2fixturedefs[top_fixtures_name][0]
if mark_name in fd.func.func_dict:
fixture_mark = extract_name_from_mark(
fd.func.func_dict[mark_name])
# Append the snapshot names in the order that fixtures are called
# starting from the last called fixture to the first one
top_fixtures_marks.append(fixture_mark)
LOG.debug("Fixtures ordered from last to first called: {0}"
.format(top_fixtures_names))
LOG.debug("Marks ordered from most to least preffered: {0}"
.format(top_fixtures_marks))
return top_fixtures_marks

View File

@ -20,7 +20,6 @@ import yaml
from fuel_ccp_tests import logger
from fuel_ccp_tests import settings
from fuel_ccp_tests.helpers import ext
LOG = logger.logger
@ -141,7 +140,6 @@ class TestAppController(object):
}
@pytest.mark.ac_linear_test
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
@pytest.mark.skipif(settings.AC_PATH is None,
reason="ApplicationController repo path is not set!")
def test_linear(self, underlay, k8scluster, show_step):

View File

@ -13,7 +13,6 @@
# under the License.
import pytest
from fuel_ccp_tests.helpers import ext
from fuel_ccp_tests import logger
from fuel_ccp_tests import settings
@ -30,7 +29,6 @@ class TestK8sDashboard(object):
"searchdomains": settings.SEARCH_DOMAINS,
}
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
@pytest.mark.fail_snapshot
@pytest.mark.dashboard_exists
def test_k8s_dashboard_exists(self, k8scluster):

View File

@ -14,7 +14,6 @@
import pytest
from fuel_ccp_tests import logger
from fuel_ccp_tests.helpers import ext
LOG = logger.logger
@ -38,7 +37,6 @@ class TestLVMPluginUsage(object):
timeout=120)
@pytest.mark.nginx_with_lvm
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
def test_create_nginx_with_lvm(self, underlay, k8scluster):
"""Test creating pod with LVM plugin

View File

@ -17,7 +17,6 @@ import pytest
import base_test
from fuel_ccp_tests import settings
from fuel_ccp_tests.helpers import ext
class FuelCCPInstallerConfigMixin:
@ -77,7 +76,6 @@ class TestFuelCCPInstaller(base_test.SystemBaseTest,
@pytest.mark.k8s_installed_default
@pytest.mark.snapshot_needed
@pytest.mark.revert_snapshot(ext.SNAPSHOT.underlay)
@pytest.mark.fail_snapshot
def test_k8s_installed_default(self, underlay, k8s_actions):
"""Test for deploying an k8s environment and check it
@ -107,7 +105,6 @@ class TestFuelCCPInstaller(base_test.SystemBaseTest,
@pytest.mark.k8s_installed_custom
@pytest.mark.snapshot_needed
@pytest.mark.revert_snapshot(ext.SNAPSHOT.underlay)
@pytest.mark.fail_snapshot
@pytest.mark.bvt
def test_k8s_installed_custom(self, underlay, k8s_actions, show_step):
@ -151,7 +148,6 @@ class TestFuelCCPInstaller(base_test.SystemBaseTest,
@pytest.mark.k8s_installed_with_etcd_on_host
@pytest.mark.snapshot_needed
@pytest.mark.revert_snapshot(ext.SNAPSHOT.underlay)
@pytest.mark.fail_snapshot
def test_k8s_installed_with_etcd_on_host(self, underlay, k8s_actions):
"""Test for deploying an k8s environment and check it
@ -192,7 +188,6 @@ class TestFuelCCPInstaller(base_test.SystemBaseTest,
@pytest.mark.k8s_installed_with_etcd_in_container
@pytest.mark.snapshot_needed
@pytest.mark.revert_snapshot(ext.SNAPSHOT.underlay)
@pytest.mark.fail_snapshot
def test_k8s_installed_with_etcd_in_container(self, underlay, k8s_actions):
"""Test for deploying an k8s environment and check it
@ -233,7 +228,6 @@ class TestFuelCCPInstaller(base_test.SystemBaseTest,
@pytest.mark.k8s_installed_with_ready_ssh_keys
@pytest.mark.snapshot_needed
@pytest.mark.revert_snapshot(ext.SNAPSHOT.underlay)
@pytest.mark.fail_snapshot
def test_k8s_installed_with_ready_ssh_keys(self, ssh_keys_dir,
underlay, k8s_actions):
@ -267,7 +261,6 @@ class TestFuelCCPInstaller(base_test.SystemBaseTest,
k8s_actions.check_pod_delete(pod)
@pytest.mark.test_k8s_installed_with_ipip
@pytest.mark.revert_snapshot(ext.SNAPSHOT.underlay)
@pytest.mark.fail_snapshot
def test_k8s_installed_with_ipip(self, underlay, k8s_actions, show_step):
"""Test for deploying an k8s environment with IPIP tunnels for Calico
@ -326,7 +319,6 @@ class TestFuelCCPInstallerIdempotency(base_test.SystemBaseTest,
@pytest.mark.ccp_idempotency_default
@pytest.mark.snapshot_needed
@pytest.mark.revert_snapshot(ext.SNAPSHOT.underlay)
@pytest.mark.fail_snapshot
def test_k8s_ccp_idempotency_default(self, config, underlay, k8s_actions):
"""Test for deploying an k8s environment and check it
@ -350,7 +342,6 @@ class TestFuelCCPInstallerIdempotency(base_test.SystemBaseTest,
@pytest.mark.ccp_idempotency_with_etcd_on_host
@pytest.mark.snapshot_needed
@pytest.mark.revert_snapshot(ext.SNAPSHOT.underlay)
@pytest.mark.fail_snapshot
def test_ccp_idempotency_with_etcd_on_host(self, config, underlay,
k8s_actions):
@ -383,7 +374,6 @@ class TestFuelCCPInstallerIdempotency(base_test.SystemBaseTest,
@pytest.mark.ccp_idempotency_with_etcd_in_container
@pytest.mark.snapshot_needed
@pytest.mark.revert_snapshot(ext.SNAPSHOT.underlay)
@pytest.mark.fail_snapshot
def test_ccp_idempotency_with_etcd_in_container(self, config, underlay,
k8s_actions):
@ -418,7 +408,6 @@ class TestFuelCCPInstallerIdempotency(base_test.SystemBaseTest,
@pytest.mark.ccp_idempotency_with_ready_ssh_keys
@pytest.mark.snapshot_needed
@pytest.mark.revert_snapshot(ext.SNAPSHOT.underlay)
@pytest.mark.fail_snapshot
def test_ccp_idempotency_with_ready_ssh_keys(self, ssh_keys_dir,
config, underlay,

View File

@ -17,7 +17,6 @@ import time
from devops.helpers import helpers
import pytest
from fuel_ccp_tests.helpers import ext
from fuel_ccp_tests import logger
LOG = logger.logger
@ -173,7 +172,6 @@ class TestDaemonsetsUpdates():
start_time_after_rollout)
)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
@pytest.mark.fail_snapshot
@pytest.mark.snapshot_needed
def test_daemonset_rollingupdate_noop(self, k8scluster, show_step):
@ -254,7 +252,6 @@ class TestDaemonsetsUpdates():
# Pods should have the new image version
self.check_nginx_pods_image(k8sclient, self.to_nginx_image)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
@pytest.mark.fail_snapshot
@pytest.mark.snapshot_needed
def test_daemonset_rollingupdate(self, k8scluster, show_step):
@ -322,7 +319,6 @@ class TestDaemonsetsUpdates():
self.to_nginx_image),
timeout=2 * 60)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
@pytest.mark.fail_snapshot
@pytest.mark.snapshot_needed
def test_daemonset_rollout_rollingupdate(self, underlay,
@ -373,7 +369,6 @@ class TestDaemonsetsUpdates():
timeout=2 * 60
)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
@pytest.mark.fail_snapshot
@pytest.mark.snapshot_needed
def test_daemonset_rollout_noop(self, underlay,
@ -441,7 +436,6 @@ class TestDaemonsetsUpdates():
# Pods should have the old image version
self.check_nginx_pods_image(k8sclient, self.from_nginx_image)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
@pytest.mark.fail_snapshot
@pytest.mark.snapshot_needed
def test_daemonset_multirollout_rollingupdate(self, underlay, k8scluster,
@ -523,7 +517,6 @@ class TestDaemonsetsUpdates():
timeout=2 * 60
)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
@pytest.mark.fail_snapshot
@pytest.mark.snapshot_needed
def test_daemonset_multirollout_rollingupdate_revision(self,
@ -688,7 +681,6 @@ class TestDaemonsetsUpdates():
timeout=2 * 60
)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
@pytest.mark.fail_snapshot
@pytest.mark.snapshot_needed
def test_daemonset_multirollout_rollingupdate_default(self, underlay,
@ -772,7 +764,6 @@ class TestDaemonsetsUpdates():
timeout=2 * 60
)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
@pytest.mark.fail_snapshot
@pytest.mark.snapshot_needed
def test_daemonset_skip_rollout(self, underlay, k8scluster,
@ -821,7 +812,6 @@ class TestDaemonsetsUpdates():
show_step(6)
self.check_rollout_skipping(k8sclient, config.k8s.kube_host, underlay)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
@pytest.mark.fail_snapshot
@pytest.mark.snapshot_needed
def test_daemonset_skip_rollout_revision(self, underlay, k8scluster,
@ -871,7 +861,6 @@ class TestDaemonsetsUpdates():
self.check_rollout_skipping(k8sclient, config.k8s.kube_host,
underlay, revision=True)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
@pytest.mark.fail_snapshot
@pytest.mark.snapshot_needed
def test_daemonset_rollout_revision_negative_1(self, underlay, k8scluster,
@ -930,7 +919,6 @@ class TestDaemonsetsUpdates():
pods_start_time_after_cmd)
)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
@pytest.mark.fail_snapshot
@pytest.mark.snapshot_needed
def test_daemonset_rollout_revision_negative_2(self, underlay, k8scluster,
@ -990,7 +978,6 @@ class TestDaemonsetsUpdates():
pods_start_time_after_cmd)
)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
@pytest.mark.fail_snapshot
@pytest.mark.snapshot_needed
def test_daemonset_rollout_revision_negative_3(self, underlay, k8scluster,
@ -1049,7 +1036,6 @@ class TestDaemonsetsUpdates():
pods_start_time_after_cmd)
)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
@pytest.mark.fail_snapshot
@pytest.mark.snapshot_needed
def test_daemonset_rollout_revision_negative_4(self, underlay, k8scluster,

View File

@ -21,7 +21,6 @@ import base_test
from fuel_ccp_tests import logger
from fuel_ccp_tests import settings
from fuel_ccp_tests.helpers import post_os_deploy_checks
from fuel_ccp_tests.helpers import ext
LOG = logger.logger
@ -32,7 +31,6 @@ class TestDeployOpenstack(base_test.SystemBaseTest):
pytest.mark: deploy_openstack
"""
@pytest.mark.revert_snapshot(ext.SNAPSHOT.ccp_deployed)
@pytest.mark.deploy_openstack
@pytest.mark.fail_snapshot
@pytest.mark.smoke
@ -70,7 +68,6 @@ class TestDeployOpenstack(base_test.SystemBaseTest):
settings.IFACES['public']),
timeout=600)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.ccp_deployed)
@pytest.mark.fail_snapshot
@pytest.mark.openstack_tempest
def test_deploy_openstack_run_tempest(self, underlay, config,

View File

@ -18,7 +18,6 @@ import pytest
import base_test
from fuel_ccp_tests import logger
from fuel_ccp_tests import settings
from fuel_ccp_tests.helpers import ext
from fuel_ccp_tests.helpers import post_os_deploy_checks
LOG = logger.logger
@ -28,7 +27,6 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
"""Deploy Two OpenStack clusters with CCP
"""
@pytest.mark.revert_snapshot(ext.SNAPSHOT.ccp_deployed)
@pytest.mark.deploy_two_os
@pytest.mark.fail_snapshot
@pytest.mark.system_few_os
@ -97,7 +95,6 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
settings.IFACES['public']),
timeout=600)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.ccp_deployed)
@pytest.mark.snapshot_needed(name="two_os")
@pytest.mark.deploy_two_os
@pytest.mark.fail_snapshot
@ -168,7 +165,6 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
settings.IFACES['public']),
timeout=600)
@pytest.mark.revert_snapshot(ext.SNAPSHOT.ccp_deployed)
@pytest.mark.deploy_two_os
@pytest.mark.fail_snapshot
@pytest.mark.system_few_os

View File

@ -26,7 +26,6 @@ class TestLCMScaleK8s(base_test.SystemBaseTest):
"""
@pytest.mark.snapshot_needed
@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed)
@pytest.mark.fail_snapshot
def test_lcm_k8s_scale_up(self, hardware, underlay, k8scluster):
"""Test for scale an k8s environment