# Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import ddt import mock from nova.compute import power_state from nova.compute import task_states from nova.compute import vm_states from nova import exception from nova.network.neutronv2 import api as network_api from nova import objects from nova.virt import event as virtevent from os_win import constants as os_win_const from os_win import exceptions as os_win_exc from compute_hyperv.nova.cluster import clusterops import compute_hyperv.nova.conf from compute_hyperv.tests import fake_instance from compute_hyperv.tests.unit import test_base CONF = compute_hyperv.nova.conf.CONF @ddt.ddt class ClusterOpsTestCase(test_base.HyperVBaseTestCase): """Unit tests for the Hyper-V ClusterOps class.""" _autospec_classes = [ clusterops.hostops.HostOps, network_api.API, clusterops.vmops.VMOps, clusterops.serialconsoleops.SerialConsoleOps, clusterops.placement_utils.PlacementUtils, ] _FAKE_INSTANCE_NAME = 'fake_instance_name' def setUp(self): super(ClusterOpsTestCase, self).setUp() self.context = 'fake_context' self.clusterops = clusterops.ClusterOps() self.clusterops._context = self.context self._clustutils = self.clusterops._clustutils self._network_api = self.clusterops._network_api self._placement = self.clusterops._placement def test_get_instance_host(self): mock_instance = fake_instance.fake_instance_obj(self.context) self.clusterops.get_instance_host(mock_instance) self.clusterops._clustutils.get_vm_host.assert_called_once_with( mock_instance.name) def test_add_to_cluster(self): mock_instance = fake_instance.fake_instance_obj(self.context) self.clusterops.add_to_cluster(mock_instance) mock_add_vm = self.clusterops._clustutils.add_vm_to_cluster mock_add_vm.assert_called_once_with( mock_instance.name, CONF.hyperv.max_failover_count, CONF.hyperv.failover_period, CONF.hyperv.auto_failback) self.assertEqual(mock_instance.uuid, self.clusterops._instance_map[mock_instance.name]) @mock.patch.object(clusterops, 'LOG') def test_add_to_cluster_exception(self, mock_LOG): mock_instance = fake_instance.fake_instance_obj(self.context) mock_add_vm = self.clusterops._clustutils.add_vm_to_cluster mock_add_vm.side_effect = os_win_exc.HyperVClusterException self.clusterops.add_to_cluster(mock_instance) self.assertTrue(mock_LOG.exception.called) def test_remove_from_cluster(self): mock_instance = fake_instance.fake_instance_obj(self.context) self.clusterops.remove_from_cluster(mock_instance) self.clusterops._clustutils.vm_exists.assert_called_once_with( mock_instance.name) self.clusterops._clustutils.delete.assert_called_once_with( mock_instance.name) self.assertIsNone(self.clusterops._instance_map.get( mock_instance.name)) @mock.patch.object(clusterops, 'LOG') def test_remove_from_cluster_exception(self, mock_LOG): mock_instance = fake_instance.fake_instance_obj(self.context) mock_delete = self.clusterops._clustutils.delete mock_delete.side_effect = os_win_exc.HyperVClusterException self.clusterops.remove_from_cluster(mock_instance) self.assertTrue(mock_LOG.exception.called) def test_post_migration(self): mock_instance = fake_instance.fake_instance_obj(self.context) self.clusterops.post_migration(mock_instance) self.assertEqual( self.clusterops._instance_map[mock_instance.name], mock_instance.uuid) @mock.patch('nova.utils.spawn_n') def test_start_failover_listener_daemon(self, mock_spawn): self.clusterops.start_failover_listener_daemon() spawn_args = mock_spawn.call_args_list[0][0] self.assertEqual( self._clustutils.get_vm_owner_change_listener_v2.return_value, spawn_args[0]) cbk = spawn_args[1] cbk() mock_spawn.assert_called_with(self.clusterops._failover_migrate) @mock.patch('nova.utils.spawn_n') @mock.patch.object(clusterops.ClusterOps, '_failover_migrate') @mock.patch.object(clusterops.ClusterOps, '_get_nova_instances') def test_reclaim_failovered_instances(self, mock_get_instances, mock_failover_migrate, mock_spawn): self.clusterops._this_node = 'fake_node' mock_instance1 = mock.MagicMock(host='other_host') mock_instance2 = mock.MagicMock(host=self.clusterops._this_node) mock_get_instances.return_value = [mock_instance1, mock_instance2] self.clusterops.reclaim_failovered_instances() self.clusterops._vmops.list_instance_uuids.assert_called_once_with() mock_get_instances.assert_called_once_with( ['id', 'uuid', 'name', 'host'], self.clusterops._vmops.list_instance_uuids.return_value) mock_spawn.assert_called_once_with( mock_failover_migrate, mock_instance1.name, mock_instance1.host, self.clusterops._this_node) @mock.patch.object(clusterops.ClusterOps, '_wait_for_pending_instance') @mock.patch.object(clusterops, 'LOG') @mock.patch.object(clusterops.ClusterOps, '_get_instance_by_name') def test_failover_migrate_no_instance(self, mock_get_instance_by_name, mock_LOG, mock_wait_pending_instance): mock_get_instance_by_name.return_value = None self.clusterops._failover_migrate(mock.sentinel.instance_name, mock.sentinel.new_host) mock_LOG.debug.assert_called_once_with( 'Instance %s does not exist in nova. Skipping.', mock.sentinel.instance_name) self.assertFalse( self.clusterops._network_api.get_instance_nw_info.called) @mock.patch.object(clusterops.ClusterOps, '_wait_for_pending_instance') @mock.patch.object(clusterops, 'LOG') @mock.patch.object(clusterops.ClusterOps, '_get_instance_by_name') def test_failover_migrate_migrating(self, mock_get_instance_by_name, mock_LOG, mock_wait_pending_instance): instance = mock_get_instance_by_name.return_value instance.task_state = task_states.MIGRATING self.clusterops._failover_migrate(mock.sentinel.instance_name, 'new_host') mock_LOG.debug.assert_called_once_with( 'Instance %s is live migrating.', mock.sentinel.instance_name) @mock.patch.object(clusterops.ClusterOps, '_wait_for_pending_instance') @mock.patch.object(clusterops.ClusterOps, '_get_instance_by_name') def test_failover_migrate_at_source_node(self, mock_get_instance_by_name, mock_wait_pending_instance): instance = mock_get_instance_by_name.return_value instance.host = 'old_host' self.clusterops._this_node = instance.host self.clusterops._failover_migrate(mock.sentinel.instance_name, 'new_host') self.clusterops._vmops.unplug_vifs.assert_called_once_with(instance, self.clusterops._network_api.get_instance_nw_info.return_value) @mock.patch.object(clusterops.ClusterOps, '_wait_for_pending_instance') @mock.patch.object(clusterops, 'LOG') @mock.patch.object(clusterops.ClusterOps, '_get_instance_by_name') def test_failover_migrate_not_this_node(self, mock_get_instance_by_name, mock_LOG, mock_wait_pending_instance): self.clusterops._this_node = 'new_host' self.clusterops._failover_migrate(mock.sentinel.instance_name, 'host') mock_LOG.debug.assert_called_once_with( 'Instance %s did not failover to this node.', mock.sentinel.instance_name) @mock.patch.object(clusterops.ClusterOps, '_wait_for_pending_instance') @mock.patch.object(clusterops.ClusterOps, '_failover_migrate_networks') @mock.patch.object(clusterops.ClusterOps, '_nova_failover_server') @mock.patch.object(clusterops.ClusterOps, '_get_instance_by_name') def test_failover_migrate_changed_host(self, mock_get_instance_by_name, mock_nova_failover_server, mock_failover_migrate_networks, mock_wait_pending_instance): instance = mock_get_instance_by_name.return_value old_host = 'old_host' new_host = 'new_host' instance.host = old_host self.clusterops._this_node = new_host self._clustutils.get_vm_host.return_value = new_host # Placement exceptions shouldn't break the rest of the failover logic. self._placement.move_compute_node_allocations.side_effect = ( exception.NovaException) self.clusterops._failover_migrate(mock.sentinel.instance_name, new_host) mock_wait_pending_instance.assert_called_once_with( mock.sentinel.instance_name) self._clustutils.get_vm_host.assert_called_once_with( mock.sentinel.instance_name) mock_get_instance_by_name.assert_called_once_with( mock.sentinel.instance_name) get_inst_nw_info = self.clusterops._network_api.get_instance_nw_info get_inst_nw_info.assert_called_once_with(self.clusterops._context, instance) mock_nova_failover_server.assert_called_once_with(instance, new_host) mock_failover_migrate_networks.assert_called_once_with( instance, old_host) self._placement.move_compute_node_allocations.assert_called_once_with( self.clusterops._context, instance, old_host, new_host, merge_existing=False) self.clusterops._vmops.plug_vifs.assert_called_once_with( instance, get_inst_nw_info.return_value) c_handler = self.clusterops._serial_console_ops.start_console_handler c_handler.assert_called_once_with(mock.sentinel.instance_name) @mock.patch.object(clusterops.ClusterOps, '_wait_for_pending_instance') @mock.patch.object(clusterops.ClusterOps, '_failover_migrate_networks') @mock.patch.object(clusterops.ClusterOps, '_nova_failover_server') @mock.patch.object(clusterops.ClusterOps, '_get_instance_by_name') def test_failover_same_node(self, mock_get_instance_by_name, mock_nova_failover_server, mock_failover_migrate_networks, mock_wait_pending_instance): # In some cases, the instances may bounce between hosts. We're testing # the case in which the instance is actually returning to the initial # host during the time in which we're processing events. instance = mock_get_instance_by_name.return_value old_host = 'old_host' new_host = 'new_host' instance.host = old_host self.clusterops._this_node = old_host self._clustutils.get_vm_host.return_value = old_host self.clusterops._failover_migrate(mock.sentinel.instance_name, new_host) get_inst_nw_info = self.clusterops._network_api.get_instance_nw_info get_inst_nw_info.assert_called_once_with(self.clusterops._context, instance) mock_nova_failover_server.assert_called_once_with(instance, old_host) self.clusterops._vmops.unplug_vifs.assert_not_called() self.clusterops._vmops.plug_vifs.assert_called_once_with( instance, get_inst_nw_info.return_value) self._placement.move_compute_node_allocations.assert_not_called() mock_failover_migrate_networks.assert_not_called() c_handler = self.clusterops._serial_console_ops.start_console_handler c_handler.assert_called_once_with(mock.sentinel.instance_name) @mock.patch('time.sleep') def test_wait_for_pending_instance(self, mock_sleep): self._clustutils.get_cluster_group_state_info.side_effect = [ dict(state=os_win_const.CLUSTER_GROUP_PENDING), dict(state=os_win_const.CLUSTER_GROUP_ONLINE)] self.clusterops._wait_for_pending_instance(mock.sentinel.instance_name) self._clustutils.get_cluster_group_state_info.assert_has_calls( [mock.call(mock.sentinel.instance_name)] * 2) mock_sleep.assert_called_once_with(2) def test_failover_migrate_networks(self): mock_instance = fake_instance.fake_instance_obj(self.context) fake_source = mock.MagicMock() fake_migration = {'source_compute': fake_source, 'dest_compute': self.clusterops._this_node} self.clusterops._failover_migrate_networks(mock_instance, fake_source) mock_network_api = self.clusterops._network_api calls = [mock.call(self.clusterops._context, mock_instance, self.clusterops._this_node), mock.call(self.clusterops._context, mock_instance, self.clusterops._this_node), mock.call(self.clusterops._context, mock_instance, self.clusterops._this_node), mock.call(self.clusterops._context, mock_instance, fake_source, teardown=True)] mock_network_api.setup_networks_on_host.assert_has_calls(calls) mock_network_api.migrate_instance_start.assert_called_once_with( self.clusterops._context, mock_instance, fake_migration) mock_network_api.migrate_instance_finish.assert_called_once_with( self.clusterops._context, mock_instance, fake_migration) @mock.patch.object(objects.Instance, 'get_by_uuid') def test_get_instance_by_name(self, mock_get_by_uuid): mock_instance = fake_instance.fake_instance_obj(self.context) mock_get_by_uuid.return_value = mock_instance self.clusterops._instance_map[mock_instance.name] = mock_instance.uuid ret = self.clusterops._get_instance_by_name(mock_instance.name) self.assertEqual(ret, mock_instance) @mock.patch.object(objects.Instance, 'get_by_uuid') def test_get_instance_by_name_not_in_cache(self, mock_get_by_uuid): mock_instance = fake_instance.fake_instance_obj(self.context) self.clusterops._vmutils.get_instance_uuid.return_value = ( mock_instance.uuid) mock_get_by_uuid.return_value = mock_instance ret = self.clusterops._get_instance_by_name(mock_instance.name) self.assertEqual(ret, mock_instance) self.assertEqual(mock_instance.uuid, self.clusterops._instance_map[mock_instance.name]) @mock.patch.object(objects.Instance, 'get_by_uuid') def test_get_instance_by_name_not_update_map(self, mock_get_by_uuid): mock_instance = fake_instance.fake_instance_obj(self.context) self.clusterops._vmutils.get_instance_uuid.side_effect = ( os_win_exc.HyperVVMNotFoundException(vm_name=mock_instance.name)) self.clusterops._update_instance_map = mock.MagicMock() self.clusterops._instance_map = mock.MagicMock() self.clusterops._instance_map.get.side_effect = [None, mock_instance.uuid] mock_get_by_uuid.return_value = mock_instance ret = self.clusterops._get_instance_by_name(mock_instance.name) self.assertEqual(ret, mock_instance) self.clusterops._update_instance_map.assert_called_with() @mock.patch.object(clusterops.ClusterOps, '_get_nova_instances') def test_update_instance_map(self, mock_get_instances): mock_instance = mock.MagicMock(uuid=mock.sentinel.uuid) mock_instance.configure_mock(name=mock.sentinel.name) mock_get_instances.return_value = [mock_instance] self.clusterops._update_instance_map() self.assertEqual(mock.sentinel.uuid, self.clusterops._instance_map[mock.sentinel.name]) @ddt.data({'instance_uuids': None}, {'instance_uuids': []}, {'instance_uuids': mock.sentinel.uuid}) @ddt.unpack @mock.patch.object(clusterops.objects.InstanceList, 'get_by_filters') def test_get_nova_instances(self, mock_get_by_filters, instance_uuids): instances = self.clusterops._get_nova_instances( instance_uuids=instance_uuids) self.assertEqual(mock_get_by_filters.return_value, instances) expected_attrs = ['id', 'uuid', 'name'] expected_filters = {'deleted': False} if instance_uuids is not None: expected_filters['uuid'] = instance_uuids mock_get_by_filters.assert_called_once_with( self.clusterops._context, expected_filters, expected_attrs=expected_attrs) @mock.patch.object(clusterops.block_device, 'DriverVolumeBlockDevice') @mock.patch.object(clusterops.objects.BlockDeviceMappingList, 'get_by_instance_uuid') def test_get_instance_block_device_mappings(self, mock_get_by_uuid, mock_DriverVBD): mock_get_by_uuid.return_value = [mock.sentinel.bdm] mock_instance = mock.MagicMock() bdms = self.clusterops._get_instance_block_device_mappings( mock_instance) self.assertEqual([mock_DriverVBD.return_value], bdms) mock_get_by_uuid.assert_called_once_with(self.clusterops._context, mock_instance.uuid) mock_DriverVBD.assert_called_once_with(mock.sentinel.bdm) def test_nova_failover_server(self): mock_instance = mock.MagicMock(vm_state=vm_states.ERROR, power_state=power_state.NOSTATE) self.clusterops._nova_failover_server(mock_instance, mock.sentinel.host) self.assertEqual(vm_states.ACTIVE, mock_instance.vm_state) self.assertEqual(power_state.RUNNING, mock_instance.power_state) self.assertEqual(mock.sentinel.host, mock_instance.host) self.assertEqual(mock.sentinel.host, mock_instance.node) mock_instance.save.assert_called_once_with(expected_task_state=[None]) @mock.patch.object(clusterops.ClusterOps, '_get_instance_by_name') def test_instance_state_change_callback(self, mock_get_instance_by_name): event = mock.Mock(transition=virtevent.EVENT_LIFECYCLE_STARTED) mock_instance = mock_get_instance_by_name.return_value self.clusterops.instance_state_change_callback(event) mock_get_instance_by_name.assert_called_once_with(event.name) self._network_api.get_instance_nw_info.assert_called_once_with( self.context, mock_instance) self.clusterops._vmops.plug_vifs.assert_called_once_with( mock_instance, self._network_api.get_instance_nw_info.return_value)