diff --git a/masakari/engine/drivers/taskflow/host_failure.py b/masakari/engine/drivers/taskflow/host_failure.py index 355292b9..c4590a46 100644 --- a/masakari/engine/drivers/taskflow/host_failure.py +++ b/masakari/engine/drivers/taskflow/host_failure.py @@ -177,6 +177,7 @@ class EvacuateInstancesTask(base.MasakariTask): try: vm_state = getattr(instance, "OS-EXT-STS:vm_state") + task_state = getattr(instance, "OS-EXT-STS:task_state") # Nova evacuates an instance only when vm_state is in active, # stopped or error state. If an instance is in other than active, @@ -192,6 +193,15 @@ class EvacuateInstancesTask(base.MasakariTask): vm_state = getattr(instance, "OS-EXT-STS:vm_state") + elif task_state is not None: + # Nova fails evacuation when the instance's task_state is not + # none. In this case, masakari resets the instance's vm_state + # to 'error' and task_state to none. + self.novaclient.reset_instance_state(context, instance.id) + instance = self.novaclient.get_server(context, instance.id) + if vm_state == 'active': + stop_instance = False + # evacuate the instance self.novaclient.evacuate_instance( context, instance.id, @@ -207,7 +217,7 @@ class EvacuateInstancesTask(base.MasakariTask): CONF.wait_period_after_evacuation, periodic_call.wait) - if vm_state not in ['active', 'stopped']: + if vm_state != 'active': if stop_instance: self._stop_after_evacuation(context, instance) # If the instance was in 'error' state before failure diff --git a/masakari/tests/unit/engine/drivers/taskflow/test_host_failure_flow.py b/masakari/tests/unit/engine/drivers/taskflow/test_host_failure_flow.py index 1ee2e56f..5f8ccb04 100644 --- a/masakari/tests/unit/engine/drivers/taskflow/test_host_failure_flow.py +++ b/masakari/tests/unit/engine/drivers/taskflow/test_host_failure_flow.py @@ -314,3 +314,45 @@ class HostFailureTestCase(test.TestCase): task = host_failure.PrepareHAEnabledInstancesTask(self.novaclient) self.assertRaises(exception.SkipHostRecoveryException, task.execute, self.ctxt, self.instance_host) + + @mock.patch.object(nova.API, 'stop_server') + @mock.patch.object(nova.API, 'reset_instance_state') + @mock.patch('masakari.compute.nova.novaclient') + def test_host_failure_flow_for_task_state_not_none( + self, _mock_novaclient, mock_reset, mock_stop, mock_unlock, + mock_lock, mock_enable_disable): + _mock_novaclient.return_value = self.fake_client + + # create ha_enabled test data + self.fake_client.servers.create(id="1", host=self.instance_host, + vm_state='active', + task_state='fake_task_state', + power_state=None, + ha_enabled=True) + self.fake_client.servers.create(id="2", host=self.instance_host, + vm_state='stopped', + task_state='fake_task_state', + power_state=None, + ha_enabled=True) + self.fake_client.servers.create(id="3", host=self.instance_host, + vm_state='error', + task_state='fake_task_state', + power_state=None, + ha_enabled=True) + instance_list = { + "instance_list": self.fake_client.servers.list() + } + + # execute EvacuateInstancesTask + self._evacuate_instances(instance_list, mock_enable_disable) + + reset_calls = [mock.call(self.ctxt, "1"), + mock.call(self.ctxt, "2"), + mock.call(self.ctxt, "3"), + mock.call(self.ctxt, "3")] + mock_reset.assert_has_calls(reset_calls) + self.assertEqual(4, mock_reset.call_count) + stop_calls = [mock.call(self.ctxt, "2"), + mock.call(self.ctxt, "3")] + mock_stop.assert_has_calls(stop_calls) + self.assertEqual(2, mock_stop.call_count) diff --git a/masakari/tests/unit/fakes.py b/masakari/tests/unit/fakes.py index 7fd4907c..c7efd8c3 100644 --- a/masakari/tests/unit/fakes.py +++ b/masakari/tests/unit/fakes.py @@ -24,12 +24,14 @@ NOW = timeutils.utcnow().replace(microsecond=0) class FakeNovaClient(object): class Server(object): def __init__(self, id=None, uuid=None, host=None, vm_state=None, - power_state=1, ha_enabled=None, locked=False): + task_state=None, power_state=1, ha_enabled=None, + locked=False): self.id = id self.uuid = uuid or uuidutils.generate_uuid() self.host = host setattr(self, 'OS-EXT-SRV-ATTR:hypervisor_hostname', host) setattr(self, 'OS-EXT-STS:vm_state', vm_state) + setattr(self, 'OS-EXT-STS:task_state', task_state) setattr(self, 'OS-EXT-STS:power_state', power_state) self.metadata = {"HA_Enabled": ha_enabled} self.locked = locked @@ -39,9 +41,10 @@ class FakeNovaClient(object): self._servers = [] def create(self, id, uuid=None, host=None, vm_state='active', - power_state=1, ha_enabled=False): + task_state=None, power_state=1, ha_enabled=False): server = FakeNovaClient.Server(id=id, uuid=uuid, host=host, vm_state=vm_state, + task_state=task_state, power_state=power_state, ha_enabled=ha_enabled) self._servers.append(server)