summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKengo Takahara <takahara-kn@njk.co.jp>2017-10-25 19:13:51 +0900
committerKengo Takahara <takahara-kn@njk.co.jp>2017-12-11 18:44:21 +0900
commitd621267402cf85eb607e0a0f328958ca9c9c3754 (patch)
treeb725ae7ea8dd3a5f543944c406d6c8c661ef2824
parent42e48076265fc626403c451da2a4d6e0be269acc (diff)
Evacuates instances which task_state is not none
This patch added implementation so that masakari can evacuate instances which task_state is not none. After the instance evacuated, it is recovered with original vm_state. So if the instance's vm_state was 'stopped', it is recovered with 'stopped', and if 'error', it is recovered with 'error'. Change-Id: I7af8552de0ee77b948a071b7f787514a81ccebc3 Closes-Bug: #1721742
Notes
Notes (review): Code-Review+2: Tushar Patil <tushar.vitthal.patil@gmail.com> Code-Review+2: Rikimaru Honjo <honjo.rikimaru@po.ntt-tx.co.jp> Workflow+1: Rikimaru Honjo <honjo.rikimaru@po.ntt-tx.co.jp> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Tue, 12 Dec 2017 01:35:07 +0000 Reviewed-on: https://review.openstack.org/515013 Project: openstack/masakari Branch: refs/heads/master
-rw-r--r--masakari/engine/drivers/taskflow/host_failure.py12
-rw-r--r--masakari/tests/unit/engine/drivers/taskflow/test_host_failure_flow.py42
-rw-r--r--masakari/tests/unit/fakes.py7
3 files changed, 58 insertions, 3 deletions
diff --git a/masakari/engine/drivers/taskflow/host_failure.py b/masakari/engine/drivers/taskflow/host_failure.py
index 355292b..c4590a4 100644
--- a/masakari/engine/drivers/taskflow/host_failure.py
+++ b/masakari/engine/drivers/taskflow/host_failure.py
@@ -177,6 +177,7 @@ class EvacuateInstancesTask(base.MasakariTask):
177 177
178 try: 178 try:
179 vm_state = getattr(instance, "OS-EXT-STS:vm_state") 179 vm_state = getattr(instance, "OS-EXT-STS:vm_state")
180 task_state = getattr(instance, "OS-EXT-STS:task_state")
180 181
181 # Nova evacuates an instance only when vm_state is in active, 182 # Nova evacuates an instance only when vm_state is in active,
182 # stopped or error state. If an instance is in other than active, 183 # stopped or error state. If an instance is in other than active,
@@ -192,6 +193,15 @@ class EvacuateInstancesTask(base.MasakariTask):
192 193
193 vm_state = getattr(instance, "OS-EXT-STS:vm_state") 194 vm_state = getattr(instance, "OS-EXT-STS:vm_state")
194 195
196 elif task_state is not None:
197 # Nova fails evacuation when the instance's task_state is not
198 # none. In this case, masakari resets the instance's vm_state
199 # to 'error' and task_state to none.
200 self.novaclient.reset_instance_state(context, instance.id)
201 instance = self.novaclient.get_server(context, instance.id)
202 if vm_state == 'active':
203 stop_instance = False
204
195 # evacuate the instance 205 # evacuate the instance
196 self.novaclient.evacuate_instance( 206 self.novaclient.evacuate_instance(
197 context, instance.id, 207 context, instance.id,
@@ -207,7 +217,7 @@ class EvacuateInstancesTask(base.MasakariTask):
207 CONF.wait_period_after_evacuation, 217 CONF.wait_period_after_evacuation,
208 periodic_call.wait) 218 periodic_call.wait)
209 219
210 if vm_state not in ['active', 'stopped']: 220 if vm_state != 'active':
211 if stop_instance: 221 if stop_instance:
212 self._stop_after_evacuation(context, instance) 222 self._stop_after_evacuation(context, instance)
213 # If the instance was in 'error' state before failure 223 # If the instance was in 'error' state before failure
diff --git a/masakari/tests/unit/engine/drivers/taskflow/test_host_failure_flow.py b/masakari/tests/unit/engine/drivers/taskflow/test_host_failure_flow.py
index 1ee2e56..5f8ccb0 100644
--- a/masakari/tests/unit/engine/drivers/taskflow/test_host_failure_flow.py
+++ b/masakari/tests/unit/engine/drivers/taskflow/test_host_failure_flow.py
@@ -314,3 +314,45 @@ class HostFailureTestCase(test.TestCase):
314 task = host_failure.PrepareHAEnabledInstancesTask(self.novaclient) 314 task = host_failure.PrepareHAEnabledInstancesTask(self.novaclient)
315 self.assertRaises(exception.SkipHostRecoveryException, task.execute, 315 self.assertRaises(exception.SkipHostRecoveryException, task.execute,
316 self.ctxt, self.instance_host) 316 self.ctxt, self.instance_host)
317
318 @mock.patch.object(nova.API, 'stop_server')
319 @mock.patch.object(nova.API, 'reset_instance_state')
320 @mock.patch('masakari.compute.nova.novaclient')
321 def test_host_failure_flow_for_task_state_not_none(
322 self, _mock_novaclient, mock_reset, mock_stop, mock_unlock,
323 mock_lock, mock_enable_disable):
324 _mock_novaclient.return_value = self.fake_client
325
326 # create ha_enabled test data
327 self.fake_client.servers.create(id="1", host=self.instance_host,
328 vm_state='active',
329 task_state='fake_task_state',
330 power_state=None,
331 ha_enabled=True)
332 self.fake_client.servers.create(id="2", host=self.instance_host,
333 vm_state='stopped',
334 task_state='fake_task_state',
335 power_state=None,
336 ha_enabled=True)
337 self.fake_client.servers.create(id="3", host=self.instance_host,
338 vm_state='error',
339 task_state='fake_task_state',
340 power_state=None,
341 ha_enabled=True)
342 instance_list = {
343 "instance_list": self.fake_client.servers.list()
344 }
345
346 # execute EvacuateInstancesTask
347 self._evacuate_instances(instance_list, mock_enable_disable)
348
349 reset_calls = [mock.call(self.ctxt, "1"),
350 mock.call(self.ctxt, "2"),
351 mock.call(self.ctxt, "3"),
352 mock.call(self.ctxt, "3")]
353 mock_reset.assert_has_calls(reset_calls)
354 self.assertEqual(4, mock_reset.call_count)
355 stop_calls = [mock.call(self.ctxt, "2"),
356 mock.call(self.ctxt, "3")]
357 mock_stop.assert_has_calls(stop_calls)
358 self.assertEqual(2, mock_stop.call_count)
diff --git a/masakari/tests/unit/fakes.py b/masakari/tests/unit/fakes.py
index 7fd4907..c7efd8c 100644
--- a/masakari/tests/unit/fakes.py
+++ b/masakari/tests/unit/fakes.py
@@ -24,12 +24,14 @@ NOW = timeutils.utcnow().replace(microsecond=0)
24class FakeNovaClient(object): 24class FakeNovaClient(object):
25 class Server(object): 25 class Server(object):
26 def __init__(self, id=None, uuid=None, host=None, vm_state=None, 26 def __init__(self, id=None, uuid=None, host=None, vm_state=None,
27 power_state=1, ha_enabled=None, locked=False): 27 task_state=None, power_state=1, ha_enabled=None,
28 locked=False):
28 self.id = id 29 self.id = id
29 self.uuid = uuid or uuidutils.generate_uuid() 30 self.uuid = uuid or uuidutils.generate_uuid()
30 self.host = host 31 self.host = host
31 setattr(self, 'OS-EXT-SRV-ATTR:hypervisor_hostname', host) 32 setattr(self, 'OS-EXT-SRV-ATTR:hypervisor_hostname', host)
32 setattr(self, 'OS-EXT-STS:vm_state', vm_state) 33 setattr(self, 'OS-EXT-STS:vm_state', vm_state)
34 setattr(self, 'OS-EXT-STS:task_state', task_state)
33 setattr(self, 'OS-EXT-STS:power_state', power_state) 35 setattr(self, 'OS-EXT-STS:power_state', power_state)
34 self.metadata = {"HA_Enabled": ha_enabled} 36 self.metadata = {"HA_Enabled": ha_enabled}
35 self.locked = locked 37 self.locked = locked
@@ -39,9 +41,10 @@ class FakeNovaClient(object):
39 self._servers = [] 41 self._servers = []
40 42
41 def create(self, id, uuid=None, host=None, vm_state='active', 43 def create(self, id, uuid=None, host=None, vm_state='active',
42 power_state=1, ha_enabled=False): 44 task_state=None, power_state=1, ha_enabled=False):
43 server = FakeNovaClient.Server(id=id, uuid=uuid, host=host, 45 server = FakeNovaClient.Server(id=id, uuid=uuid, host=host,
44 vm_state=vm_state, 46 vm_state=vm_state,
47 task_state=task_state,
45 power_state=power_state, 48 power_state=power_state,
46 ha_enabled=ha_enabled) 49 ha_enabled=ha_enabled)
47 self._servers.append(server) 50 self._servers.append(server)