openstack
/
nova
Code Issues Proposed changes

pci: Clarify SR-IOV ports vs direct passthrough ports 

This patch clarify for what type of ports
detach/attach pci device is needed and for what type of
port we just need pci request. To avoid confusion this
patch introduce 2 type of ports list VNIC_TYPES_SRIOV
and VNIC_TYPES_DIRECT_PASSTHROUGH. The VNIC_TYPES_SRIOV
are ports which require pci request, while
VNIC_TYPES_DIRECT_PASSTHROUGH  ports require pci device
attach/detach from libvirt dom. VNIC_TYPES_DIRECT_PASSTHROUGH
is subset of VNIC_TYPES_SRIOV.

Closes-Bug: #1563874

Change-Id: I3a45b1fb41e8e446d1f25d7a1d77991c8bf2a1ed
This commit is contained in:
Moshe Levi 2016-04-03 19:11:07 +03:00 committed by Stephen Finucane
parent ae7f78648d
commit b691125b62
3 changed files with 66 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
nova/network/model.py
View File

@ -96,9 +96,17 @@ VNIC_TYPE_MACVTAP = 'macvtap'
VNIC_TYPE_DIRECT_PHYSICAL = 'direct-physical'
VNIC_TYPE_BAREMETAL = 'baremetal'
# Define list of ports which needs pci request.
# Note: The macvtap port needs a PCI request as it is a tap interface
# with VF as the lower physical interface.
VNIC_TYPES_SRIOV = (VNIC_TYPE_DIRECT, VNIC_TYPE_MACVTAP,
VNIC_TYPE_DIRECT_PHYSICAL)
# Define list of ports which are passthrough to the guest
# and need a special treatment on snapshot and suspend/resume
VNIC_TYPES_DIRECT_PASSTHROUGH = (VNIC_TYPE_DIRECT,
VNIC_TYPE_DIRECT_PHYSICAL)
# Constants for the 'vif_model' values
VIF_MODEL_VIRTIO = 'virtio'
VIF_MODEL_NE2K_PCI = 'ne2k_pci'

59
nova/tests/unit/virt/libvirt/test_driver.py
View File

@ -11521,14 +11521,16 @@ class LibvirtConnTestCase(test.NoDBTestCase):
@mock.patch.object(dmcrypt, 'delete_volume')
@mock.patch.object(conn, '_get_instance_disk_info', return_value=[])
@mock.patch.object(conn, '_detach_sriov_ports')
@mock.patch.object(conn, '_detach_direct_passthrough_ports')
@mock.patch.object(conn, '_detach_pci_devices')
@mock.patch.object(pci_manager, 'get_instance_pci_devs',
return_value='pci devs')
@mock.patch.object(conn._host, 'get_guest', return_value=guest)
def suspend(mock_get_guest, mock_get_instance_pci_devs,
mock_detach_pci_devices, mock_detach_sriov_ports,
mock_get_instance_disk_info, mock_delete_volume):
mock_detach_pci_devices,
mock_detach_direct_passthrough_ports,
mock_get_instance_disk_info,
mock_delete_volume):
mock_managedSave = mock.Mock()
dom.managedSave = mock_managedSave
@ -11616,10 +11618,8 @@ class LibvirtConnTestCase(test.NoDBTestCase):
@mock.patch.object(FakeVirtDomain, 'ID', return_value=1)
@mock.patch.object(utils, 'get_image_from_system_metadata',
return_value=None)
def test_attach_sriov_ports(self,
mock_get_image_metadata,
mock_ID,
mock_attachDevice):
def test_attach_direct_passthrough_ports(self,
mock_get_image_metadata, mock_ID, mock_attachDevice):
instance = objects.Instance(**self.test_instance)
network_info = _fake_network_info(self, 1)
@ -11627,7 +11627,8 @@ class LibvirtConnTestCase(test.NoDBTestCase):
guest = libvirt_guest.Guest(FakeVirtDomain())
drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
drvr._attach_sriov_ports(self.context, instance, guest, network_info)
drvr._attach_direct_passthrough_ports(
self.context, instance, guest, network_info)
mock_get_image_metadata.assert_called_once_with(
instance.system_metadata)
self.assertTrue(mock_attachDevice.called)
@ -11636,10 +11637,8 @@ class LibvirtConnTestCase(test.NoDBTestCase):
@mock.patch.object(FakeVirtDomain, 'ID', return_value=1)
@mock.patch.object(utils, 'get_image_from_system_metadata',
return_value=None)
def test_attach_sriov_direct_physical_ports(self,
mock_get_image_metadata,
mock_ID,
mock_attachDevice):
def test_attach_direct_physical_passthrough_ports(self,
mock_get_image_metadata, mock_ID, mock_attachDevice):
instance = objects.Instance(**self.test_instance)
network_info = _fake_network_info(self, 1)
@ -11647,7 +11646,8 @@ class LibvirtConnTestCase(test.NoDBTestCase):
guest = libvirt_guest.Guest(FakeVirtDomain())
drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
drvr._attach_sriov_ports(self.context, instance, guest, network_info)
drvr._attach_direct_passthrough_ports(
self.context, instance, guest, network_info)
mock_get_image_metadata.assert_called_once_with(
instance.system_metadata)
self.assertTrue(mock_attachDevice.called)
@ -11656,10 +11656,8 @@ class LibvirtConnTestCase(test.NoDBTestCase):
@mock.patch.object(FakeVirtDomain, 'ID', return_value=1)
@mock.patch.object(utils, 'get_image_from_system_metadata',
return_value=None)
def test_attach_sriov_ports_with_info_cache(self,
mock_get_image_metadata,
mock_ID,
mock_attachDevice):
def test_attach_direct_passthrough_ports_with_info_cache(self,
mock_get_image_metadata, mock_ID, mock_attachDevice):
instance = objects.Instance(**self.test_instance)
network_info = _fake_network_info(self, 1)
@ -11669,14 +11667,15 @@ class LibvirtConnTestCase(test.NoDBTestCase):
guest = libvirt_guest.Guest(FakeVirtDomain())
drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
drvr._attach_sriov_ports(self.context, instance, guest, None)
drvr._attach_direct_passthrough_ports(
self.context, instance, guest, None)
mock_get_image_metadata.assert_called_once_with(
instance.system_metadata)
self.assertTrue(mock_attachDevice.called)
@mock.patch.object(host.Host,
'has_min_version', return_value=True)
def _test_detach_sriov_ports(self,
def _test_detach_direct_passthrough_ports(self,
mock_has_min_version, vif_type):
instance = objects.Instance(**self.test_instance)
@ -11707,25 +11706,25 @@ class LibvirtConnTestCase(test.NoDBTestCase):
guest = libvirt_guest.Guest(domain)
with mock.patch.object(drvr, '_detach_pci_devices') as mock_detach_pci:
drvr._detach_sriov_ports(self.context, instance, guest)
drvr._detach_direct_passthrough_ports(
self.context, instance, guest)
mock_detach_pci.assert_called_once_with(
guest, [expected_pci_device_obj])
def test_detach_sriov_ports_interface_interface_hostdev(self):
# Note: test detach_sriov_ports method for vif with config
def test_detach_direct_passthrough_ports_interface_interface_hostdev(self):
# Note: test detach_direct_passthrough_ports method for vif with config
# LibvirtConfigGuestInterface
self._test_detach_sriov_ports(vif_type="hw_veb")
self._test_detach_direct_passthrough_ports(vif_type="hw_veb")
def test_detach_sriov_ports_interface_pci_hostdev(self):
# Note: test detach_sriov_ports method for vif with config
def test_detach_direct_passthrough_ports_interface_pci_hostdev(self):
# Note: test detach_direct_passthrough_ports method for vif with config
# LibvirtConfigGuestHostdevPCI
self._test_detach_sriov_ports(vif_type="ib_hostdev")
self._test_detach_direct_passthrough_ports(vif_type="ib_hostdev")
@mock.patch.object(host.Host, 'has_min_version', return_value=True)
@mock.patch.object(FakeVirtDomain, 'detachDeviceFlags')
def test_detach_duplicate_mac_sriov_ports(self,
mock_detachDeviceFlags,
mock_has_min_version):
def test_detach_duplicate_mac_direct_passthrough_ports(
self, mock_detachDeviceFlags, mock_has_min_version):
instance = objects.Instance(**self.test_instance)
network_info = _fake_network_info(self, 2)
@ -11754,7 +11753,7 @@ class LibvirtConnTestCase(test.NoDBTestCase):
drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
guest = libvirt_guest.Guest(domain)
drvr._detach_sriov_ports(self.context, instance, guest)
drvr._detach_direct_passthrough_ports(self.context, instance, guest)
expected_xml = [
('<hostdev mode="subsystem" type="pci" managed="yes">\n'

51
nova/virt/libvirt/driver.py
View File

@ -1644,7 +1644,8 @@ class LibvirtDriver(driver.ComputeDriver):
if guest is not None:
self._attach_pci_devices(
guest, pci_manager.get_instance_pci_devs(instance))
self._attach_sriov_ports(context, instance, guest)
self._attach_direct_passthrough_ports(
context, instance, guest)
def _can_set_admin_password(self, image_meta):
@ -2519,7 +2520,7 @@ class LibvirtDriver(driver.ComputeDriver):
self._detach_pci_devices(guest,
pci_manager.get_instance_pci_devs(instance))
self._detach_sriov_ports(context, instance, guest)
self._detach_direct_passthrough_ports(context, instance, guest)
guest.save_memory_state()
def resume(self, context, instance, network_info, block_device_info=None):
@ -2536,7 +2537,8 @@ class LibvirtDriver(driver.ComputeDriver):
vifs_already_plugged=True)
self._attach_pci_devices(guest,
pci_manager.get_instance_pci_devs(instance))
self._attach_sriov_ports(context, instance, guest, network_info)
self._attach_direct_passthrough_ports(
context, instance, guest, network_info)
timer = loopingcall.FixedIntervalLoopingCall(self._wait_for_running,
instance)
timer.start(interval=0.5).wait()
@ -3305,56 +3307,61 @@ class LibvirtDriver(driver.ComputeDriver):
raise
@staticmethod
def _has_sriov_port(network_info):
def _has_direct_passthrough_port(network_info):
for vif in network_info:
if vif['vnic_type'] in [network_model.VNIC_TYPE_DIRECT,
network_model.VNIC_TYPE_DIRECT_PHYSICAL]:
if (vif['vnic_type'] in
network_model.VNIC_TYPES_DIRECT_PASSTHROUGH):
return True
return False
def _attach_sriov_ports(self, context, instance, guest, network_info=None):
def _attach_direct_passthrough_ports(
self, context, instance, guest, network_info=None):
if network_info is None:
network_info = instance.info_cache.network_info
if network_info is None:
return
if self._has_sriov_port(network_info):
if self._has_direct_passthrough_port(network_info):
for vif in network_info:
if vif['vnic_type'] in network_model.VNIC_TYPES_SRIOV:
if (vif['vnic_type'] in
network_model.VNIC_TYPES_DIRECT_PASSTHROUGH):
cfg = self.vif_driver.get_config(instance,
vif,
instance.image_meta,
instance.flavor,
CONF.libvirt.virt_type,
self._host)
LOG.debug('Attaching SR-IOV port %(port)s to %(dom)s',
{'port': vif, 'dom': guest.id},
LOG.debug('Attaching direct passthrough port %(port)s '
'to %(dom)s', {'port': vif, 'dom': guest.id},
instance=instance)
guest.attach_device(cfg)
def _detach_sriov_ports(self, context, instance, guest):
def _detach_direct_passthrough_ports(self, context, instance, guest):
network_info = instance.info_cache.network_info
if network_info is None:
return
if self._has_sriov_port(network_info):
# In case of SR-IOV vif types we create pci request per SR-IOV port
# Therefore we can trust that pci_slot value in the vif is correct.
sriov_pci_addresses = [
if self._has_direct_passthrough_port(network_info):
# In case of VNIC_TYPES_DIRECT_PASSTHROUGH ports we create
# pci request per direct passthrough port. Therefore we can trust
# that pci_slot value in the vif is correct.
direct_passthrough_pci_addresses = [
vif['profile']['pci_slot']
for vif in network_info
if vif['vnic_type'] in network_model.VNIC_TYPES_SRIOV and
vif['profile'].get('pci_slot') is not None
if (vif['vnic_type'] in
network_model.VNIC_TYPES_DIRECT_PASSTHROUGH and
vif['profile'].get('pci_slot') is not None)
]
# use detach_pci_devices to avoid failure in case of
# multiple guest SRIOV ports with the same MAC
# multiple guest direct passthrough ports with the same MAC
# (protection use-case, ports are on different physical
# interfaces)
pci_devs = pci_manager.get_instance_pci_devs(instance, 'all')
sriov_devs = [pci_dev for pci_dev in pci_devs
if pci_dev.address in sriov_pci_addresses]
self._detach_pci_devices(guest, sriov_devs)
direct_passthrough_pci_addresses = (
[pci_dev for pci_dev in pci_devs
if pci_dev.address in direct_passthrough_pci_addresses])
self._detach_pci_devices(guest, direct_passthrough_pci_addresses)
def _set_host_enabled(self, enabled,
disable_reason=DISABLE_REASON_UNDEFINED):