Add failover tests

Add power off/shutdown tests for murano and controller
nodes.

Change-Id: I20bb076bbbe7dac30bce751b576d12b513b99bf0

murano_plugin_tests/murano_plugin/api.py

@@ -12,6 +12,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import functools
+
+from devops.helpers import helpers as devops_helpers
+from fuelweb_test import logger
 from fuelweb_test.tests import base_test_case
 
 from murano_plugin_tests.helpers import checkers
@@ -101,3 +105,33 @@ class MuranoPluginApi(object):
     def check_uninstall_failure(self):
         return self.helpers.check_plugin_cannot_be_uninstalled(
             self.settings.name, self.settings.version)
+
+    def wait_plugin_online(self, timeout=5 * 60):
+        """Wait until the plugin will start working properly.
+        """
+
+        def check_availability():
+            try:
+                self.check_plugin_online()
+                return True
+            except AssertionError:
+                return False
+
+        logger.info('Wait a plugin become online')
+        msg = "Plugin has not become online after a waiting period"
+        devops_helpers.wait(
+            check_availability, interval=30, timeout=timeout, timeout_msg=msg)
+
+    def check_plugin_failover(self, operation, role_name):
+        fuel_web_client = self.helpers.fuel_web
+        operations = {
+            "soft_reboot": fuel_web_client.warm_restart_nodes,
+            "hard_reboot": functools.partial(
+                fuel_web_client.cold_restart_nodes, wait_offline=False)
+        }
+        nailgun_nodes = fuel_web_client.get_nailgun_cluster_nodes_by_roles(
+            self.helpers.cluster_id, role_name)
+        target_node = fuel_web_client.get_devops_nodes_by_nailgun_nodes(
+            nailgun_nodes[:1])
+        operations[operation](target_node)
+        self.wait_plugin_online()

murano_plugin_tests/murano_plugin/test_failover.py

@@ -0,0 +1,100 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from fuelweb_test.helpers.decorators import log_snapshot_after_test
+from proboscis import test
+
+from murano_plugin_tests.murano_plugin import api
+
+
+@test(groups=["plugin"])
+class TestMuranoFailover(api.MuranoPluginApi):
+    """Class for testing that the Murano Detach plugin works properly
+    on controller or detached node fail.
+    """
+
+    def _test_failover(self, operation, role_name):
+        self.env.revert_snapshot("deploy_murano_plugin")
+
+        self.check_plugin_failover(operation, role_name)
+
+        self.run_ostf()
+
+    @test(depends_on_groups=["deploy_murano_plugin"],
+          groups=["failover", "murano", "system", "destructive",
+                  "soft_reboot_murano_node"])
+    @log_snapshot_after_test
+    def soft_reboot_murano_node(self):
+        """Verify that failover for Murano plugin works
+        on Murano detached node soft_reboot.
+
+        Scenario:
+            1. Soft reboot murano node.
+            2. Check that plugin is working.
+            3. Run OSTF.
+
+        Duration 30m
+        """
+        self._test_failover("soft_reboot", self.settings.role_name)
+
+    @test(depends_on_groups=["deploy_murano_plugin"],
+          groups=["failover", "murano", "system", "destructive",
+                  "hard_reboot_murano_node"])
+    @log_snapshot_after_test
+    def hard_reboot_murano_node(self):
+        """Verify that failover for Murano plugin works
+        on Murano detached node power off.
+
+        Scenario:
+            1. Hard reboot murano node.
+            2. Check that plugin is working.
+            3. Run OSTF.
+
+        Duration 30m
+        """
+        self._test_failover("hard_reboot", self.settings.role_name)
+
+    @test(depends_on_groups=["deploy_murano_plugin"],
+          groups=["failover", "murano", "system", "destructive",
+                  "soft_reboot_controller_node_murano_plugin"])
+    @log_snapshot_after_test
+    def soft_reboot_controller_node_murano_plugin(self):
+        """Verify that failover for Murano plugin works
+        on controller node soft_reboot.
+
+        Scenario:
+            1. Soft reboot controller node.
+            2. Check that plugin is working.
+            3. Run OSTF.
+
+        Duration 30m
+        """
+        self._test_failover("soft_reboot", ["controller"])
+
+    @test(depends_on_groups=["deploy_murano_plugin"],
+          groups=["failover", "murano", "system", "destructive",
+                  "hard_reboot_controller_node_murano_plugin"])
+    @log_snapshot_after_test
+    def hard_reboot_controller_node_murano_plugin(self):
+        """Verify that failover for Murano plugin works
+        on controller node power off.
+
+        Scenario:
+            1. Hard reboot controller node.
+            2. Check that plugin is working.
+            3. Run OSTF.
+
+        Duration 30m
+        """
+        self._test_failover("hard_reboot", ["controller"])

murano_plugin_tests/run_tests.py

@@ -39,6 +39,7 @@ class CloseSSHConnectionsPlugin(plugins.Plugin):
 
 
 def import_tests():
+    from murano_plugin import test_failover  # noqa
     from murano_plugin import test_murano_plugin_bvt  # noqa
     from murano_plugin import test_post_install  # noqa
     from murano_plugin import test_system  # noqa