Evacuate instances one by one

Evacuate instances using live-migration cmd for instance.

Change-Id: Id0a4c54c63e81c8cb9b2664daa88bdfba57cca0b

octane/handlers/upgrade/compute.py

@@ -87,9 +87,11 @@ class ComputeUpgrade(upgrade.UpgradeHandler):
             nova.run_nova_cmd(
                 ["nova", "service-disable", node_fqdn, "nova-compute"],
                 controller, False)
-        nova.run_nova_cmd(['nova', 'host-evacuate-live', node_fqdn],
-                          controller, False)
-        nova.waiting_for_status_completed(controller, node_fqdn, "MIGRATING")
+        for instance_id in nova.get_active_instances(controller, node_fqdn):
+            nova.run_nova_cmd(
+                ["nova", "live-migration", instance_id], controller, False)
+            nova.waiting_for_status_completed(
+                controller, node_fqdn, "MIGRATING")
         if nova.do_nova_instances_exist(controller, node_fqdn):
             raise Exception(
                 "There are instances on {hostname} after host-evacuation, "
@@ -112,17 +114,9 @@ class ComputeUpgrade(upgrade.UpgradeHandler):
                 "please fix this problem and start upgrade_node "
                 "command again".format(hostname=node_fqdn))
 
-        instances_stdout = nova.run_nova_cmd([
-            "nova", "list",
-            "--host", node_fqdn,
-            "--limit", "-1",
-            "--status", "ACTIVE",
-            "--minimal"],
-            controller)
-        instances = nova.nova_stdout_parser(instances_stdout)
-        for instance in instances:
+        for instance_id in nova.get_active_instances(controller, node_fqdn):
             nova.run_nova_cmd(
-                ["nova", "stop", instance['ID']], controller, output=False)
+                ["nova", "stop", instance_id], controller, output=False)
         nova.waiting_for_status_completed(controller, node_fqdn, "ACTIVE")
 
     def backup_iscsi_initiator_info(self):

octane/tests/test_compute_handlers.py

@@ -23,8 +23,9 @@ import pytest
 @pytest.mark.parametrize("node_fqdn", ["node-disabled-1", "node-enabled-1"])
 @pytest.mark.parametrize("nodes_in_error_state", [True, False])
 @pytest.mark.parametrize("fuel_version", ["7.0", "8.0"])
+@pytest.mark.parametrize("instances", [["instance_1", "instance_2"]])
 def test_evacuate_host(mocker, enabled, disabled, node_fqdn,
-                       nodes_in_error_state, fuel_version):
+                       nodes_in_error_state, fuel_version, instances):
     env = mock.Mock()
     controller = mock.Mock()
     node = mock.Mock()
@@ -44,6 +45,9 @@ def test_evacuate_host(mocker, enabled, disabled, node_fqdn,
         "octane.util.nova.do_nova_instances_exist",
         return_value=nodes_in_error_state)
 
+    get_instances_mock = mocker.patch(
+        "octane.util.nova.get_active_instances", return_value=instances)
+
     mock_waiting = mocker.patch(
         "octane.util.nova.waiting_for_status_completed")
 
@@ -64,15 +68,19 @@ def test_evacuate_host(mocker, enabled, disabled, node_fqdn,
         nova_calls.append(mock.call(
             ["nova", "service-disable", node_fqdn, "nova-compute"],
             controller, False))
-    nova_calls.append(mock.call(
-        ['nova', 'host-evacuate-live', node_fqdn], controller, False))
+    for instance in instances:
+        nova_calls.append(mock.call(
+            ["nova", "live-migration", instance], controller, False))
     if error:
         assert not run_nova_cmd.called
         assert not mock_waiting.called
+        assert not get_instances_mock.called
     else:
         assert run_nova_cmd.call_args_list == nova_calls
-        mock_waiting.assert_called_once_with(
-            controller, node_fqdn, "MIGRATING")
+        get_instances_mock.assert_called_once_with(controller, node_fqdn)
+        waiting_calls = [mock.call(controller, node_fqdn, "MIGRATING")
+                         for i in instances]
+        assert waiting_calls == mock_waiting.call_args_list
     if [node_fqdn] == enabled:
         assert not mock_is_nova_state.called
     else:

octane/tests/test_util_nova.py

@@ -189,3 +189,32 @@ def test_get_compute_lists(mocker, cmd_output, enabled, disabled):
     assert (enabled, disabled) == nova.get_compute_lists(controller)
     run_nova_cmd.assert_called_once_with(
         ["nova", "service-list", "--binary", "nova-compute"], controller)
+
+
+@pytest.mark.parametrize("cmd_out,result", [(
+    "+--------------------------------------+\n"
+    "| ID                                   |\n"
+    "+--------------------------------------+\n"
+    "| d5c35583-f498-4841-a032-069ec066d2d5 |\n"
+    "| 8d274e6b-91db-4d76-a5e8-13a23c3335c9 |\n"
+    "| 093c55f2-4a30-4a74-95ea-d7c39fcb4e3a |\n"
+    "+--------------------------------------+\n",
+    [
+        "d5c35583-f498-4841-a032-069ec066d2d5",
+        "8d274e6b-91db-4d76-a5e8-13a23c3335c9",
+        "093c55f2-4a30-4a74-95ea-d7c39fcb4e3a",
+    ]),
+])
+@pytest.mark.parametrize("node_fqdn", ["node_fqdn"])
+def test_get_active_instances(mocker, cmd_out, result, node_fqdn):
+    controller = mock.Mock()
+    nova_mock = mocker.patch(
+        "octane.util.nova.run_nova_cmd", return_value=cmd_out)
+    assert result == nova.get_active_instances(controller, node_fqdn)
+    nova_mock.assert_called_once_with([
+        "nova", "list",
+        "--host", node_fqdn,
+        "--limit", "-1",
+        "--status", "ACTIVE",
+        "--minimal"],
+        controller)

octane/util/nova.py

@@ -91,3 +91,15 @@ def get_compute_lists(controller):
         elif service['Status'] == 'disabled':
             disabled_computes.append(service['Host'])
     return (enabled_computes, disabled_computes)
+
+
+def get_active_instances(controller, node_fqdn):
+    instances_stdout = run_nova_cmd([
+        "nova", "list",
+        "--host", node_fqdn,
+        "--limit", "-1",
+        "--status", "ACTIVE",
+        "--minimal"],
+        controller)
+    instances = nova_stdout_parser(instances_stdout)
+    return [i["ID"] for i in instances]