Merge "VMware: Live migration of instances"

This commit is contained in:
Zuul 2018-10-08 19:12:42 +00:00 committed by Gerrit Code Review
commit f8e46a5cf4
11 changed files with 444 additions and 21 deletions

View File

@ -382,3 +382,16 @@ Block migration
- Block migration works only with EXT local storage storage repositories,
and the server must not have any volumes attached.
VMware
~~~~~~
.. :ref:`_configuring-migrations-vmware`
.. _configuring-migrations-vmware:
vSphere configuration
---------------------
Enable vMotion on all ESX hosts which are managed by Nova by following the
instructions in `this <https://kb.vmware.com/s/article/2054994>`_ KB article.

View File

@ -468,8 +468,7 @@ driver.libvirt-kvm-s390x=complete
driver.libvirt-qemu-x86=complete
driver.libvirt-lxc=missing
driver.libvirt-xen=complete
driver.vmware=missing
driver-notes.vmware=https://bugs.launchpad.net/nova/+bug/1192192
driver.vmware=complete
driver.hyperv=complete
driver.ironic=missing
driver.libvirt-vz-vm=complete

View File

@ -501,3 +501,13 @@ class PowerVMLiveMigrateData(LiveMigrateData):
for field in self.fields:
if field in legacy:
setattr(self, field, legacy[field])
@obj_base.NovaObjectRegistry.register
class VMwareLiveMigrateData(LiveMigrateData):
VERSION = '1.0'
fields = {
'cluster_name': fields.StringField(nullable=False),
'datastore_regex': fields.StringField(nullable=False),
}

View File

@ -1166,6 +1166,7 @@ object_data = {
'VirtCPUTopology': '1.0-fc694de72e20298f7c6bab1083fd4563',
'VirtualInterface': '1.3-efd3ca8ebcc5ce65fff5a25f31754c54',
'VirtualInterfaceList': '1.0-9750e2074437b3077e46359102779fc6',
'VMwareLiveMigrateData': '1.0-a3cc858a2bf1d3806d6f57cfaa1fb98a',
'VolumeUsage': '1.0-6c8190c46ce1469bb3286a1f21c2e475',
'XenDeviceBus': '1.0-272a4f899b24e31e42b2b9a7ed7e9194',
'XenapiLiveMigrateData': '1.4-7dc9417e921b2953faa6751f18785f3f',

View File

@ -2349,23 +2349,20 @@ class VMwareAPIVMTestCase(test.NoDBTestCase,
self.assertEqual(2, len(ds_util._DS_DC_MAPPING))
def test_pre_live_migration(self):
migrate_data = objects.migrate_data.LiveMigrateData()
self.assertRaises(NotImplementedError,
self.conn.pre_live_migration, self.context,
'fake_instance', 'fake_block_device_info',
'fake_network_info', 'fake_disk_info', migrate_data)
def test_live_migration(self):
self.assertRaises(NotImplementedError,
self.conn.live_migration, self.context,
'fake_instance', 'fake_dest', 'fake_post_method',
'fake_recover_method')
migrate_data = objects.VMwareLiveMigrateData()
migrate_data.cluster_name = 'fake-cluster'
migrate_data.datastore_regex = 'datastore1'
ret = self.conn.pre_live_migration(self.context, 'fake-instance',
'fake-block-dev-info', 'fake-net-info',
'fake-disk-info', migrate_data)
self.assertIs(migrate_data, ret)
def test_rollback_live_migration_at_destination(self):
self.assertRaises(NotImplementedError,
self.conn.rollback_live_migration_at_destination,
self.context, 'fake_instance', 'fake_network_info',
'fake_block_device_info')
with mock.patch.object(self.conn, "destroy") as mock_destroy:
self.conn.rollback_live_migration_at_destination(self.context,
"instance", [], None)
mock_destroy.assert_called_once_with(self.context,
"instance", [], None)
def test_post_live_migration(self):
self.assertIsNone(self.conn.post_live_migration(self.context,

View File

@ -181,6 +181,67 @@ class VMwareVMUtilTestCase(test.NoDBTestCase):
self.assertEqual(expected, result)
def test_update_vif_spec_opaque_net(self):
fake_factory = fake.FakeFactory()
vif_info = {'network_name': 'br100',
'mac_address': '00:00:00:ca:fe:01',
'network_ref': {'type': 'OpaqueNetwork',
'network-id': 'fake-network-id',
'network-type': 'fake-net',
'use-external-id': False},
'iface_id': 7,
'vif_model': 'VirtualE1000'}
device = fake_factory.create('ns0:VirtualDevice')
actual = vm_util.update_vif_spec(fake_factory, vif_info, device)
spec = fake_factory.create('ns0:VirtualDeviceConfigSpec')
spec.device = fake_factory.create('ns0:VirtualDevice')
spec.device.backing = fake_factory.create(
'ns0:VirtualEthernetCardOpaqueNetworkBackingInfo')
spec.device.backing.opaqueNetworkType = 'fake-net'
spec.device.backing.opaqueNetworkId = 'fake-network-id'
spec.operation = 'edit'
self.assertEqual(spec, actual)
def test_update_vif_spec_dvpg(self):
fake_factory = fake.FakeFactory()
vif_info = {'network_name': 'br100',
'mac_address': '00:00:00:ca:fe:01',
'network_ref': {'type': 'DistributedVirtualPortgroup',
'dvsw': 'fake-network-id',
'dvpg': 'fake-group'},
'iface_id': 7,
'vif_model': 'VirtualE1000'}
device = fake_factory.create('ns0:VirtualDevice')
actual = vm_util.update_vif_spec(fake_factory, vif_info, device)
spec = fake_factory.create('ns0:VirtualDeviceConfigSpec')
spec.device = fake_factory.create('ns0:VirtualDevice')
spec.device.backing = fake_factory.create(
'ns0:VirtualEthernetCardDistributedVirtualPortBackingInfo')
spec.device.backing.port = fake_factory.create(
'ns0:DistributedVirtualSwitchPortConnection')
spec.device.backing.port.portgroupKey = 'fake-group'
spec.device.backing.port.switchUuid = 'fake-network-id'
spec.operation = 'edit'
self.assertEqual(spec, actual)
def test_update_vif_spec_network(self):
fake_factory = fake.FakeFactory()
vif_info = {'network_name': 'br100',
'mac_address': '00:00:00:ca:fe:01',
'network_ref': {'type': 'Network',
'name': 'net1'},
'iface_id': 7,
'vif_model': 'VirtualE1000'}
device = fake_factory.create('ns0:VirtualDevice')
actual = vm_util.update_vif_spec(fake_factory, vif_info, device)
spec = fake_factory.create('ns0:VirtualDeviceConfigSpec')
spec.device = fake_factory.create('ns0:VirtualDevice')
spec.device.backing = fake_factory.create(
'ns0:VirtualEthernetCardNetworkBackingInfo')
spec.device.backing.deviceName = 'br100'
spec.operation = 'edit'
self.assertEqual(spec, actual)
def test_get_cdrom_attach_config_spec(self):
fake_factory = fake.FakeFactory()
datastore = fake.Datastore()

View File

@ -82,6 +82,14 @@ class VMwareVMOpsTestCase(test.NoDBTestCase):
vmFolder='fake_vm_folder')
cluster = vmwareapi_fake.create_cluster('fake_cluster', fake_ds_ref)
self._uuid = uuidsentinel.foo
fake_info_cache = {
'created_at': None,
'updated_at': None,
'deleted_at': None,
'deleted': False,
'instance_uuid': self._uuid,
'network_info': '[]',
}
self._instance_values = {
'name': 'fake_name',
'display_name': 'fake_display_name',
@ -91,7 +99,8 @@ class VMwareVMOpsTestCase(test.NoDBTestCase):
'image_ref': self._image_id,
'root_gb': 10,
'node': '%s(%s)' % (cluster.mo_id, cluster.name),
'expected_attrs': ['system_metadata'],
'info_cache': fake_info_cache,
'expected_attrs': ['system_metadata', 'info_cache'],
}
self._instance = fake_instance.fake_instance_obj(
self._context, **self._instance_values)
@ -793,6 +802,127 @@ class VMwareVMOpsTestCase(test.NoDBTestCase):
def test_finish_revert_migration_power_off(self):
self._test_finish_revert_migration(power_on=False)
def _test_find_esx_host(self, cluster_hosts, ds_hosts):
def mock_call_method(module, method, *args, **kwargs):
if args[0] == 'fake_cluster':
ret = mock.MagicMock()
ret.ManagedObjectReference = cluster_hosts
return ret
elif args[0] == 'fake_ds':
ret = mock.MagicMock()
ret.DatastoreHostMount = ds_hosts
return ret
with mock.patch.object(self._session, '_call_method',
mock_call_method):
return self._vmops._find_esx_host('fake_cluster', 'fake_ds')
def test_find_esx_host(self):
ch1 = vmwareapi_fake.ManagedObjectReference(value='host-10')
ch2 = vmwareapi_fake.ManagedObjectReference(value='host-12')
ch3 = vmwareapi_fake.ManagedObjectReference(value='host-15')
dh1 = vmwareapi_fake.DatastoreHostMount('host-8')
dh2 = vmwareapi_fake.DatastoreHostMount('host-12')
dh3 = vmwareapi_fake.DatastoreHostMount('host-17')
ret = self._test_find_esx_host([ch1, ch2, ch3], [dh1, dh2, dh3])
self.assertEqual('host-12', ret.value)
def test_find_esx_host_none(self):
ch1 = vmwareapi_fake.ManagedObjectReference(value='host-10')
ch2 = vmwareapi_fake.ManagedObjectReference(value='host-12')
ch3 = vmwareapi_fake.ManagedObjectReference(value='host-15')
dh1 = vmwareapi_fake.DatastoreHostMount('host-8')
dh2 = vmwareapi_fake.DatastoreHostMount('host-13')
dh3 = vmwareapi_fake.DatastoreHostMount('host-17')
ret = self._test_find_esx_host([ch1, ch2, ch3], [dh1, dh2, dh3])
self.assertIsNone(ret)
@mock.patch.object(vm_util, 'get_vmdk_info')
@mock.patch.object(ds_obj, 'get_datastore_by_ref')
def test_find_datastore_for_migration(self, mock_get_ds, mock_get_vmdk):
def mock_call_method(module, method, *args, **kwargs):
ds1 = vmwareapi_fake.ManagedObjectReference(value='datastore-10')
ds2 = vmwareapi_fake.ManagedObjectReference(value='datastore-12')
ds3 = vmwareapi_fake.ManagedObjectReference(value='datastore-15')
ret = mock.MagicMock()
ret.ManagedObjectReference = [ds1, ds2, ds3]
return ret
ds_ref = vmwareapi_fake.ManagedObjectReference(value='datastore-12')
vmdk_dev = mock.MagicMock()
vmdk_dev.device.backing.datastore = ds_ref
mock_get_vmdk.return_value = vmdk_dev
ds = ds_obj.Datastore(ds_ref, 'datastore1')
mock_get_ds.return_value = ds
with mock.patch.object(self._session, '_call_method',
mock_call_method):
ret = self._vmops._find_datastore_for_migration(self._instance,
'fake_vm', 'cluster_ref',
None)
self.assertIs(ds, ret)
mock_get_vmdk.assert_called_once_with(self._session, 'fake_vm',
uuid=self._instance.uuid)
mock_get_ds.assert_called_once_with(self._session, ds_ref)
@mock.patch.object(vm_util, 'get_vmdk_info')
@mock.patch.object(ds_util, 'get_datastore')
def test_find_datastore_for_migration_other(self, mock_get_ds,
mock_get_vmdk):
def mock_call_method(module, method, *args, **kwargs):
ds1 = vmwareapi_fake.ManagedObjectReference(value='datastore-10')
ds2 = vmwareapi_fake.ManagedObjectReference(value='datastore-12')
ds3 = vmwareapi_fake.ManagedObjectReference(value='datastore-15')
ret = mock.MagicMock()
ret.ManagedObjectReference = [ds1, ds2, ds3]
return ret
ds_ref = vmwareapi_fake.ManagedObjectReference(value='datastore-18')
vmdk_dev = mock.MagicMock()
vmdk_dev.device.backing.datastore = ds_ref
mock_get_vmdk.return_value = vmdk_dev
ds = ds_obj.Datastore(ds_ref, 'datastore1')
mock_get_ds.return_value = ds
with mock.patch.object(self._session, '_call_method',
mock_call_method):
ret = self._vmops._find_datastore_for_migration(self._instance,
'fake_vm', 'cluster_ref',
None)
self.assertIs(ds, ret)
mock_get_vmdk.assert_called_once_with(self._session, 'fake_vm',
uuid=self._instance.uuid)
mock_get_ds.assert_called_once_with(self._session, 'cluster_ref',
None)
@mock.patch.object(vm_util, 'relocate_vm')
@mock.patch.object(vm_util, 'get_vm_ref', return_value='fake_vm')
@mock.patch.object(vm_util, 'get_cluster_ref_by_name',
return_value='fake_cluster')
@mock.patch.object(vm_util, 'get_res_pool_ref', return_value='fake_pool')
@mock.patch.object(vmops.VMwareVMOps, '_find_datastore_for_migration')
@mock.patch.object(vmops.VMwareVMOps, '_find_esx_host',
return_value='fake_host')
def test_live_migration(self, mock_find_host, mock_find_datastore,
mock_get_respool, mock_get_cluster, mock_get_vm,
mock_relocate):
post_method = mock.MagicMock()
migrate_data = objects.VMwareLiveMigrateData()
migrate_data.cluster_name = 'fake-cluster'
migrate_data.datastore_regex = 'ds1|ds2'
mock_find_datastore.return_value = ds_obj.Datastore('ds_ref', 'ds')
with mock.patch.object(self._session, '_call_method',
return_value='hardware-devices'):
self._vmops.live_migration(
self._context, self._instance, 'fake-host',
post_method, None, False, migrate_data)
mock_get_vm.assert_called_once_with(self._session, self._instance)
mock_get_cluster.assert_called_once_with(self._session, 'fake-cluster')
mock_find_datastore.assert_called_once_with(self._instance, 'fake_vm',
'fake_cluster', mock.ANY)
mock_find_host.assert_called_once_with('fake_cluster', 'ds_ref')
mock_relocate.assert_called_once_with(self._session, 'fake_vm',
'fake_pool', 'ds_ref', 'fake_host',
devices=[])
post_method.assert_called_once_with(self._context, self._instance,
'fake-host', False, migrate_data)
@mock.patch.object(vmops.VMwareVMOps, '_get_instance_metadata')
@mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs')
@mock.patch.object(vm_util, 'reconfigure_vm')

View File

@ -38,6 +38,7 @@ from nova.compute import utils as compute_utils
import nova.conf
from nova import exception
from nova.i18n import _
from nova import objects
import nova.privsep.path
from nova import rc_fields as fields
from nova.virt import driver
@ -269,6 +270,71 @@ class VMwareVCDriver(driver.ComputeDriver):
network_info, image_meta, resize_instance,
block_device_info, power_on)
def ensure_filtering_rules_for_instance(self, instance, network_info):
pass
def pre_live_migration(self, context, instance, block_device_info,
network_info, disk_info, migrate_data):
return migrate_data
def post_live_migration_at_source(self, context, instance, network_info):
pass
def post_live_migration_at_destination(self, context, instance,
network_info,
block_migration=False,
block_device_info=None):
pass
def cleanup_live_migration_destination_check(self, context,
dest_check_data):
pass
def live_migration(self, context, instance, dest,
post_method, recover_method, block_migration=False,
migrate_data=None):
"""Live migration of an instance to another host."""
self._vmops.live_migration(context, instance, dest, post_method,
recover_method, block_migration,
migrate_data)
def check_can_live_migrate_source(self, context, instance,
dest_check_data, block_device_info=None):
cluster_name = dest_check_data.cluster_name
cluster_ref = vm_util.get_cluster_ref_by_name(self._session,
cluster_name)
if cluster_ref is None:
msg = (_("Cannot find destination cluster %s for live migration") %
cluster_name)
raise exception.MigrationPreCheckError(reason=msg)
res_pool_ref = vm_util.get_res_pool_ref(self._session, cluster_ref)
if res_pool_ref is None:
msg = _("Cannot find destination resource pool for live migration")
raise exception.MigrationPreCheckError(reason=msg)
return dest_check_data
def check_can_live_migrate_destination(self, context, instance,
src_compute_info, dst_compute_info,
block_migration=False,
disk_over_commit=False):
# the information that we need for the destination compute node
# is the name of its cluster and datastore regex
data = objects.VMwareLiveMigrateData()
data.cluster_name = CONF.vmware.cluster_name
data.datastore_regex = CONF.vmware.datastore_regex
return data
def unfilter_instance(self, instance, network_info):
pass
def rollback_live_migration_at_destination(self, context, instance,
network_info,
block_device_info,
destroy_disks=True,
migrate_data=None):
"""Clean up destination node after a failed live migration."""
self.destroy(context, instance, network_info, block_device_info)
def get_instance_disk_info(self, instance, block_device_info=None):
pass

View File

@ -519,6 +519,44 @@ def get_network_detach_config_spec(client_factory, device, port_index):
return config_spec
def update_vif_spec(client_factory, vif_info, device):
"""Updates the backing for the VIF spec."""
network_spec = client_factory.create('ns0:VirtualDeviceConfigSpec')
network_spec.operation = 'edit'
network_ref = vif_info['network_ref']
network_name = vif_info['network_name']
if network_ref and network_ref['type'] == 'OpaqueNetwork':
backing = client_factory.create(
'ns0:VirtualEthernetCardOpaqueNetworkBackingInfo')
backing.opaqueNetworkId = network_ref['network-id']
backing.opaqueNetworkType = network_ref['network-type']
# Configure externalId
if network_ref['use-external-id']:
if hasattr(device, 'externalId'):
device.externalId = vif_info['iface_id']
else:
dp = client_factory.create('ns0:DynamicProperty')
dp.name = "__externalId__"
dp.val = vif_info['iface_id']
device.dynamicProperty = [dp]
elif (network_ref and
network_ref['type'] == "DistributedVirtualPortgroup"):
backing = client_factory.create(
'ns0:VirtualEthernetCardDistributedVirtualPortBackingInfo')
portgroup = client_factory.create(
'ns0:DistributedVirtualSwitchPortConnection')
portgroup.switchUuid = network_ref['dvsw']
portgroup.portgroupKey = network_ref['dvpg']
backing.port = portgroup
else:
backing = client_factory.create(
'ns0:VirtualEthernetCardNetworkBackingInfo')
backing.deviceName = network_name
device.backing = backing
network_spec.device = device
return network_spec
def get_storage_profile_spec(session, storage_policy):
"""Gets the vm profile spec configured for storage policy."""
profile_id = pbm.get_profile_id_by_name(session, storage_policy)
@ -938,20 +976,24 @@ def clone_vm_spec(client_factory, location,
def relocate_vm_spec(client_factory, res_pool=None, datastore=None, host=None,
disk_move_type="moveAllDiskBackingsAndAllowSharing"):
disk_move_type="moveAllDiskBackingsAndAllowSharing",
devices=None):
rel_spec = client_factory.create('ns0:VirtualMachineRelocateSpec')
rel_spec.datastore = datastore
rel_spec.host = host
rel_spec.pool = res_pool
rel_spec.diskMoveType = disk_move_type
if devices is not None:
rel_spec.deviceChange = devices
return rel_spec
def relocate_vm(session, vm_ref, res_pool=None, datastore=None, host=None,
disk_move_type="moveAllDiskBackingsAndAllowSharing"):
disk_move_type="moveAllDiskBackingsAndAllowSharing",
devices=None):
client_factory = session.vim.client.factory
rel_spec = relocate_vm_spec(client_factory, res_pool, datastore, host,
disk_move_type)
disk_move_type, devices)
relocate_task = session._call_method(session.vim, "RelocateVM_Task",
vm_ref, spec=rel_spec)
session._wait_for_task(relocate_task)

View File

@ -21,6 +21,7 @@ Class for VM tasks like spawn, snapshot, suspend, resume etc.
import collections
import os
import re
import time
import decorator
@ -1552,6 +1553,103 @@ class VMwareVMOps(object):
step=6,
total_steps=RESIZE_TOTAL_STEPS)
def _find_esx_host(self, cluster_ref, ds_ref):
"""Find ESX host in the specified cluster which is also connected to
the specified datastore.
"""
cluster_hosts = self._session._call_method(vutil,
'get_object_property',
cluster_ref, 'host')
ds_hosts = self._session._call_method(vutil, 'get_object_property',
ds_ref, 'host')
for ds_host in ds_hosts.DatastoreHostMount:
for cluster_host in cluster_hosts.ManagedObjectReference:
if ds_host.key.value == cluster_host.value:
return cluster_host
def _find_datastore_for_migration(self, instance, vm_ref, cluster_ref,
datastore_regex):
"""Find datastore in the specified cluster where the instance will be
migrated to. Return the current datastore if it is already connected to
the specified cluster.
"""
vmdk = vm_util.get_vmdk_info(self._session, vm_ref, uuid=instance.uuid)
ds_ref = vmdk.device.backing.datastore
cluster_datastores = self._session._call_method(vutil,
'get_object_property',
cluster_ref,
'datastore')
if not cluster_datastores:
LOG.warning('No datastores found in the destination cluster')
return None
# check if the current datastore is connected to the destination
# cluster
for datastore in cluster_datastores.ManagedObjectReference:
if datastore.value == ds_ref.value:
ds = ds_obj.get_datastore_by_ref(self._session, ds_ref)
if (datastore_regex is None or
datastore_regex.match(ds.name)):
LOG.debug('Datastore "%s" is connected to the '
'destination cluster', ds.name)
return ds
# find the most suitable datastore on the destination cluster
return ds_util.get_datastore(self._session, cluster_ref,
datastore_regex)
def live_migration(self, context, instance, dest,
post_method, recover_method, block_migration,
migrate_data):
LOG.debug("Live migration data %s", migrate_data, instance=instance)
vm_ref = vm_util.get_vm_ref(self._session, instance)
cluster_name = migrate_data.cluster_name
cluster_ref = vm_util.get_cluster_ref_by_name(self._session,
cluster_name)
datastore_regex = re.compile(migrate_data.datastore_regex)
res_pool_ref = vm_util.get_res_pool_ref(self._session, cluster_ref)
# find a datastore where the instance will be migrated to
ds = self._find_datastore_for_migration(instance, vm_ref, cluster_ref,
datastore_regex)
if ds is None:
LOG.error("Cannot find datastore", instance=instance)
raise exception.HostNotFound(host=dest)
LOG.debug("Migrating instance to datastore %s", ds.name,
instance=instance)
# find ESX host in the destination cluster which is connected to the
# target datastore
esx_host = self._find_esx_host(cluster_ref, ds.ref)
if esx_host is None:
LOG.error("Cannot find ESX host for live migration, cluster: %s, "
"datastore: %s", migrate_data.cluster_name, ds.name,
instance=instance)
raise exception.HostNotFound(host=dest)
# Update networking backings
network_info = instance.get_network_info()
client_factory = self._session.vim.client.factory
devices = []
hardware_devices = self._session._call_method(
vutil, "get_object_property", vm_ref, "config.hardware.device")
vif_model = instance.image_meta.properties.get('hw_vif_model',
constants.DEFAULT_VIF_MODEL)
for vif in network_info:
vif_info = vmwarevif.get_vif_dict(
self._session, cluster_ref, vif_model, utils.is_neutron(), vif)
device = vmwarevif.get_network_device(hardware_devices,
vif['address'])
devices.append(vm_util.update_vif_spec(client_factory, vif_info,
device))
LOG.debug("Migrating instance to cluster '%s', datastore '%s' and "
"ESX host '%s'", cluster_name, ds.name, esx_host,
instance=instance)
try:
vm_util.relocate_vm(self._session, vm_ref, res_pool_ref,
ds.ref, esx_host, devices=devices)
LOG.info("Migrated instance to host %s", dest, instance=instance)
except Exception:
with excutils.save_and_reraise_exception():
recover_method(context, instance, dest, migrate_data)
post_method(context, instance, dest, block_migration, migrate_data)
def poll_rebooting_instances(self, timeout, instances):
"""Poll for rebooting instances."""
ctxt = nova_context.get_admin_context()

View File

@ -0,0 +1,6 @@
---
features:
- |
The VMware compute driver now supports live migration. Each compute node
must be managing a cluster in the same vCenter and ESX hosts must have
vMotion enabled.