From c6516412d5ac38ef64d3774e26fcc24934e1d9ca Mon Sep 17 00:00:00 2001 From: Kirill Zaitsev Date: Wed, 5 Oct 2016 17:06:00 +0300 Subject: [PATCH] Add retry to check_plugin_online Add retry decorator to cope with race-condition and overly optimistic tests timeouts for environment deletion. Also replace tests_platform with check_plugin_online ostf Add lightweight sanity check function 'check_plugin_sanity' to use it with failover tests to determine that nodes with murano come back from reboots. Change-Id: I6dfd40820cea74c45d33c57672ba555d0fa04372 Closes-Bug: #1630594 --- murano_plugin_tests/murano_plugin/api.py | 19 ++++++++++++++++--- .../murano_plugin/test_failover.py | 3 ++- .../murano_plugin/test_murano_plugin_bvt.py | 15 ++++++++++----- .../test_murano_plugin_update.py | 9 ++++++--- .../murano_plugin/test_post_install.py | 3 ++- .../murano_plugin/test_system.py | 12 ++++++------ 6 files changed, 42 insertions(+), 19 deletions(-) diff --git a/murano_plugin_tests/murano_plugin/api.py b/murano_plugin_tests/murano_plugin/api.py index 9791d93..6dbf2c9 100644 --- a/murano_plugin_tests/murano_plugin/api.py +++ b/murano_plugin_tests/murano_plugin/api.py @@ -15,6 +15,7 @@ import functools from devops.helpers import helpers as devops_helpers +from fuelweb_test.helpers.decorators import retry from fuelweb_test import logger from fuelweb_test.tests import base_test_case @@ -105,8 +106,9 @@ class MuranoPluginApi(object): self.helpers.activate_plugin( self.settings.name, self.settings.version, options) + @retry(count=3, delay=120) def check_plugin_online(self): - """Checks that plugin is working.""" + """Checks that plugin is working. Runs platform Murano OSTF.""" test_name = ('fuel_health.tests.tests_platform.test_murano_linux.' 'MuranoDeployLinuxServicesTests.' 'test_deploy_dummy_app_with_glare') @@ -114,6 +116,17 @@ class MuranoPluginApi(object): test_name=test_name, timeout=60 * 20) + @retry(count=3, delay=120) + def check_plugin_sanity(self): + """Checks that plugin is working. Runs sanity Murano OSTF.""" + test_name = ('fuel_health.tests.sanity.test_sanity_murano.' + 'MuranoSanityTests.' + 'test_create_and_delete_service') + + self.helpers.run_single_ostf(test_sets=['tests_platform'], + test_name=test_name, + timeout=5 * 10) + def uninstall_plugin(self): """Uninstall plugin from Fuel.""" return self.helpers.uninstall_plugin(self.settings.name, @@ -123,13 +136,13 @@ class MuranoPluginApi(object): return self.helpers.check_plugin_cannot_be_uninstalled( self.settings.name, self.settings.version) - def wait_plugin_online(self, timeout=5 * 60): + def wait_plugin_online(self, timeout=5 * 120): """Wait until the plugin will start working properly. """ def check_availability(): try: - self.check_plugin_online() + self.check_plugin_sanity() return True except AssertionError: return False diff --git a/murano_plugin_tests/murano_plugin/test_failover.py b/murano_plugin_tests/murano_plugin/test_failover.py index b470bb7..c30c345 100644 --- a/murano_plugin_tests/murano_plugin/test_failover.py +++ b/murano_plugin_tests/murano_plugin/test_failover.py @@ -29,7 +29,8 @@ class TestMuranoFailover(api.MuranoPluginApi): self.check_plugin_failover(operation, role_name) - self.run_ostf(['sanity', 'smoke', 'tests_platform']) + self.run_ostf(['sanity', 'smoke']) + self.check_plugin_online() @test(depends_on_groups=["deploy_murano_plugin"], groups=["failover", "murano", "system", "destructive", diff --git a/murano_plugin_tests/murano_plugin/test_murano_plugin_bvt.py b/murano_plugin_tests/murano_plugin/test_murano_plugin_bvt.py index 5cdf24f..fb07e0d 100644 --- a/murano_plugin_tests/murano_plugin/test_murano_plugin_bvt.py +++ b/murano_plugin_tests/murano_plugin/test_murano_plugin_bvt.py @@ -52,7 +52,8 @@ class TestMuranoPluginBvt(api.MuranoPluginApi): self.helpers.deploy_cluster(self.only_controllers) - self.run_ostf(['sanity', 'smoke', 'tests_platform']) + self.run_ostf(['sanity', 'smoke']) + self.check_plugin_online() self.env.make_snapshot("deploy_murano_plugin_on_controller", is_make=True) @@ -87,7 +88,8 @@ class TestMuranoPluginBvt(api.MuranoPluginApi): self.helpers.deploy_cluster(self.only_controllers_ha) - self.run_ostf(['sanity', 'smoke', 'tests_platform']) + self.run_ostf(['sanity', 'smoke']) + self.check_plugin_online() self.env.make_snapshot("deploy_murano_plugin_on_controller_ha", is_make=True) @@ -123,7 +125,8 @@ class TestMuranoPluginBvt(api.MuranoPluginApi): self.helpers.deploy_cluster(self.base_nodes) - self.run_ostf(['sanity', 'smoke', 'tests_platform']) + self.run_ostf(['sanity', 'smoke']) + self.check_plugin_online() self.env.make_snapshot("deploy_murano_plugin", is_make=True) @@ -158,7 +161,8 @@ class TestMuranoPluginBvt(api.MuranoPluginApi): self.helpers.deploy_cluster(self.ha_nodes) - self.run_ostf(['sanity', 'smoke', 'ha', 'tests_platform']) + self.run_ostf(['sanity', 'smoke', 'ha']) + self.check_plugin_online() self.env.make_snapshot("deploy_murano_plugin_ha", is_make=True) @@ -193,7 +197,8 @@ class TestMuranoPluginBvt(api.MuranoPluginApi): self.helpers.deploy_cluster(self.full_ha_nodes) - self.run_ostf(['sanity', 'smoke', 'ha', 'tests_platform']) + self.run_ostf(['sanity', 'smoke', 'ha']) + self.check_plugin_online() self.env.make_snapshot("deploy_murano_plugin_full_ha", is_make=True) diff --git a/murano_plugin_tests/murano_plugin/test_murano_plugin_update.py b/murano_plugin_tests/murano_plugin/test_murano_plugin_update.py index 2ef26a1..921e541 100644 --- a/murano_plugin_tests/murano_plugin/test_murano_plugin_update.py +++ b/murano_plugin_tests/murano_plugin/test_murano_plugin_update.py @@ -44,7 +44,8 @@ class TestMuranoPluginUpdate(api.MuranoPluginApi): self.helpers.deploy_cluster(self.only_controllers) - self.helpers.run_ostf(['sanity', 'smoke', 'tests_platform']) + self.helpers.run_ostf(['sanity', 'smoke']) + self.check_plugin_online() self.env.make_snapshot("deploy_murano_out_of_the_box", is_make=True) @@ -75,7 +76,8 @@ class TestMuranoPluginUpdate(api.MuranoPluginApi): self.helpers.apply_changes() - self.helpers.run_ostf(['sanity', 'smoke', 'tests_platform']) + self.helpers.run_ostf(['sanity', 'smoke']) + self.check_plugin_online() self.env.make_snapshot( "deploy_murano_plugin_in_environment_with_murano", @@ -112,7 +114,8 @@ class TestMuranoPluginUpdate(api.MuranoPluginApi): 'slave-03': plugin_settings.role_name, }) - self.helpers.run_ostf(['sanity', 'smoke', 'tests_platform']) + self.helpers.run_ostf(['sanity', 'smoke']) + self.check_plugin_online() self.env.make_snapshot("deploy_murano_node_in_environment_with_murano", is_make=False) diff --git a/murano_plugin_tests/murano_plugin/test_post_install.py b/murano_plugin_tests/murano_plugin/test_post_install.py index 897e4cb..a8be345 100644 --- a/murano_plugin_tests/murano_plugin/test_post_install.py +++ b/murano_plugin_tests/murano_plugin/test_post_install.py @@ -48,7 +48,8 @@ class TestMuranoPostInstallation(api.MuranoPluginApi): self.helpers.deploy_cluster(self.only_controllers) - self.run_ostf(['sanity', 'smoke', 'tests_platform']) + self.run_ostf(['sanity', 'smoke']) + self.check_plugin_online() self.env.make_snapshot("deploy_environment_without_murano_plugin", is_make=True) diff --git a/murano_plugin_tests/murano_plugin/test_system.py b/murano_plugin_tests/murano_plugin/test_system.py index 11f6bf7..e21b683 100644 --- a/murano_plugin_tests/murano_plugin/test_system.py +++ b/murano_plugin_tests/murano_plugin/test_system.py @@ -60,7 +60,7 @@ class TestSystemMuranoPlugin(api.MuranoPluginApi): self.check_plugin_online() - self.run_ostf(['sanity', 'smoke', 'ha', 'tests_platform']) + self.run_ostf(['sanity', 'smoke', 'ha']) compute_manipulated_node = {'slave-04': ['compute', 'cinder']} @@ -69,7 +69,7 @@ class TestSystemMuranoPlugin(api.MuranoPluginApi): self.check_plugin_online() - self.run_ostf(['sanity', 'smoke', 'ha', 'tests_platform']) + self.run_ostf(['sanity', 'smoke', 'ha']) # Add controller # NOTE(rpromyshlennikov): test can fail here before @@ -79,14 +79,14 @@ class TestSystemMuranoPlugin(api.MuranoPluginApi): self.check_plugin_online() - self.run_ostf(['sanity', 'smoke', 'ha', 'tests_platform']) + self.run_ostf(['sanity', 'smoke', 'ha']) # Add compute self.helpers.add_nodes_to_cluster(compute_manipulated_node) self.check_plugin_online() - self.run_ostf(['sanity', 'smoke', 'ha', 'tests_platform']) + self.run_ostf(['sanity', 'smoke', 'ha']) self.env.make_snapshot("add_remove_controller_compute_murano", is_make=False) @@ -122,13 +122,13 @@ class TestSystemMuranoPlugin(api.MuranoPluginApi): self.check_plugin_online() - self.run_ostf(['sanity', 'smoke', 'ha', 'tests_platform']) + self.run_ostf(['sanity', 'smoke', 'ha']) # Add Murano Detach node self.helpers.add_nodes_to_cluster(manipulated_node) self.check_plugin_online() - self.run_ostf(['sanity', 'smoke', 'ha', 'tests_platform']) + self.run_ostf(['sanity', 'smoke', 'ha']) self.env.make_snapshot("add_remove_murano_node", is_make=False)