4176 lines
171 KiB
Python
4176 lines
171 KiB
Python
# Copyright (C) 2016 Red Hat, Inc
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import copy
|
|
import pprint
|
|
import typing as ty
|
|
from unittest import mock
|
|
from urllib import parse as urlparse
|
|
|
|
import ddt
|
|
import fixtures
|
|
from lxml import etree
|
|
from oslo_config import cfg
|
|
from oslo_log import log as logging
|
|
from oslo_serialization import jsonutils
|
|
from oslo_utils.fixture import uuidsentinel as uuids
|
|
from oslo_utils import units
|
|
|
|
import nova
|
|
from nova.compute import pci_placement_translator
|
|
from nova import context
|
|
from nova import exception
|
|
from nova.network import constants
|
|
from nova import objects
|
|
from nova.objects import fields
|
|
from nova.pci.utils import parse_address
|
|
from nova.tests import fixtures as nova_fixtures
|
|
from nova.tests.fixtures import libvirt as fakelibvirt
|
|
from nova.tests.functional.api import client
|
|
from nova.tests.functional.libvirt import base
|
|
|
|
CONF = cfg.CONF
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
class PciPlacementHealingFixture(fixtures.Fixture):
|
|
"""Allow asserting if the pci_placement_translator module needed to
|
|
heal PCI allocations. Such healing is only normal during upgrade. After
|
|
every compute is upgraded and the scheduling support of PCI tracking in
|
|
placement is enabled there should be no need to heal PCI allocations in
|
|
the resource tracker. We assert this as we eventually want to remove the
|
|
automatic healing logic from the resource tracker.
|
|
"""
|
|
|
|
def __init__(self):
|
|
super().__init__()
|
|
# a list of (nodename, result, allocation_before, allocation_after)
|
|
# tuples recoding the result of the calls to
|
|
# update_provider_tree_for_pci
|
|
self.calls = []
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
|
|
orig = pci_placement_translator.update_provider_tree_for_pci
|
|
|
|
def wrapped_update(
|
|
provider_tree, nodename, pci_tracker, allocations, same_host
|
|
):
|
|
alloc_before = copy.deepcopy(allocations)
|
|
updated = orig(
|
|
provider_tree, nodename, pci_tracker, allocations, same_host)
|
|
alloc_after = copy.deepcopy(allocations)
|
|
self.calls.append((nodename, updated, alloc_before, alloc_after))
|
|
return updated
|
|
|
|
self.useFixture(
|
|
fixtures.MonkeyPatch(
|
|
"nova.compute.pci_placement_translator."
|
|
"update_provider_tree_for_pci",
|
|
wrapped_update,
|
|
)
|
|
)
|
|
|
|
def last_healing(self, hostname: str) -> ty.Optional[ty.Tuple[dict, dict]]:
|
|
for h, updated, before, after in self.calls:
|
|
if h == hostname and updated:
|
|
return before, after
|
|
return None
|
|
|
|
|
|
class _PCIServersTestBase(base.ServersTestBase):
|
|
|
|
ADDITIONAL_FILTERS = ['NUMATopologyFilter', 'PciPassthroughFilter']
|
|
|
|
PCI_RC = f"CUSTOM_PCI_{fakelibvirt.PCI_VEND_ID}_{fakelibvirt.PCI_PROD_ID}"
|
|
|
|
def setUp(self):
|
|
self.ctxt = context.get_admin_context()
|
|
self.flags(
|
|
device_spec=self.PCI_DEVICE_SPEC,
|
|
alias=self.PCI_ALIAS,
|
|
group='pci'
|
|
)
|
|
|
|
super(_PCIServersTestBase, self).setUp()
|
|
|
|
# Mock the 'PciPassthroughFilter' filter, as most tests need to inspect
|
|
# this
|
|
host_manager = self.scheduler.manager.host_manager
|
|
pci_filter_class = host_manager.filter_cls_map['PciPassthroughFilter']
|
|
host_pass_mock = mock.Mock(wraps=pci_filter_class().host_passes)
|
|
self.mock_filter = self.useFixture(fixtures.MockPatch(
|
|
'nova.scheduler.filters.pci_passthrough_filter'
|
|
'.PciPassthroughFilter.host_passes',
|
|
side_effect=host_pass_mock)).mock
|
|
|
|
self.pci_healing_fixture = self.useFixture(
|
|
PciPlacementHealingFixture())
|
|
|
|
def assertPCIDeviceCounts(self, hostname, total, free):
|
|
"""Ensure $hostname has $total devices, $free of which are free."""
|
|
devices = objects.PciDeviceList.get_by_compute_node(
|
|
self.ctxt,
|
|
objects.ComputeNode.get_by_nodename(self.ctxt, hostname).id,
|
|
)
|
|
self.assertEqual(total, len(devices))
|
|
self.assertEqual(free, len([d for d in devices if d.is_available()]))
|
|
|
|
def assert_no_pci_healing(self, hostname):
|
|
last_healing = self.pci_healing_fixture.last_healing(hostname)
|
|
before = last_healing[0] if last_healing else None
|
|
after = last_healing[1] if last_healing else None
|
|
self.assertIsNone(
|
|
last_healing,
|
|
"The resource tracker needed to heal PCI allocation in placement "
|
|
"on host %s. This should not happen in normal operation as the "
|
|
"scheduler should create the proper allocation instead.\n"
|
|
"Allocations before healing:\n %s\n"
|
|
"Allocations after healing:\n %s\n"
|
|
% (
|
|
hostname,
|
|
pprint.pformat(before),
|
|
pprint.pformat(after),
|
|
),
|
|
)
|
|
|
|
def _get_rp_by_name(self, name, rps):
|
|
for rp in rps:
|
|
if rp["name"] == name:
|
|
return rp
|
|
self.fail(f'RP {name} is not found in Placement {rps}')
|
|
|
|
def assert_placement_pci_inventory(self, hostname, inventories, traits):
|
|
compute_rp_uuid = self.compute_rp_uuids[hostname]
|
|
rps = self._get_all_rps_in_a_tree(compute_rp_uuid)
|
|
|
|
# rps also contains the root provider so we subtract 1
|
|
self.assertEqual(
|
|
len(inventories),
|
|
len(rps) - 1,
|
|
f"Number of RPs on {hostname} doesn't match. "
|
|
f"Expected {list(inventories)} actual {[rp['name'] for rp in rps]}"
|
|
)
|
|
|
|
for rp_name, inv in inventories.items():
|
|
real_rp_name = f'{hostname}_{rp_name}'
|
|
rp = self._get_rp_by_name(real_rp_name, rps)
|
|
rp_inv = self._get_provider_inventory(rp['uuid'])
|
|
|
|
self.assertEqual(
|
|
len(inv),
|
|
len(rp_inv),
|
|
f"Number of inventories on {real_rp_name} are not as "
|
|
f"expected. Expected {inv}, actual {rp_inv}"
|
|
)
|
|
for rc, total in inv.items():
|
|
self.assertEqual(
|
|
total,
|
|
rp_inv[rc]["total"])
|
|
self.assertEqual(
|
|
total,
|
|
rp_inv[rc]["max_unit"])
|
|
|
|
rp_traits = self._get_provider_traits(rp['uuid'])
|
|
self.assertEqual(
|
|
# COMPUTE_MANAGED_PCI_DEVICE is automatically reported on
|
|
# PCI device RPs by nova
|
|
set(traits[rp_name]) | {"COMPUTE_MANAGED_PCI_DEVICE"},
|
|
set(rp_traits),
|
|
f"Traits on RP {real_rp_name} does not match with expectation"
|
|
)
|
|
|
|
def assert_placement_pci_usages(self, hostname, usages):
|
|
compute_rp_uuid = self.compute_rp_uuids[hostname]
|
|
rps = self._get_all_rps_in_a_tree(compute_rp_uuid)
|
|
|
|
for rp_name, usage in usages.items():
|
|
real_rp_name = f'{hostname}_{rp_name}'
|
|
rp = self._get_rp_by_name(real_rp_name, rps)
|
|
rp_usage = self._get_provider_usages(rp['uuid'])
|
|
self.assertEqual(
|
|
usage,
|
|
rp_usage,
|
|
f"Usage on RP {real_rp_name} does not match with expectation"
|
|
)
|
|
|
|
def assert_placement_pci_allocations(self, allocations):
|
|
for consumer, expected_allocations in allocations.items():
|
|
actual_allocations = self._get_allocations_by_server_uuid(consumer)
|
|
self.assertEqual(
|
|
len(expected_allocations),
|
|
len(actual_allocations),
|
|
f"The consumer {consumer} allocates from different number of "
|
|
f"RPs than expected. Expected: {expected_allocations}, "
|
|
f"Actual: {actual_allocations}"
|
|
)
|
|
for rp_name, expected_rp_allocs in expected_allocations.items():
|
|
rp_uuid = self._get_provider_uuid_by_name(rp_name)
|
|
self.assertIn(
|
|
rp_uuid,
|
|
actual_allocations,
|
|
f"The consumer {consumer} expected to allocate from "
|
|
f"{rp_name}. Expected: {expected_allocations}, "
|
|
f"Actual: {actual_allocations}"
|
|
)
|
|
actual_rp_allocs = actual_allocations[rp_uuid]['resources']
|
|
self.assertEqual(
|
|
expected_rp_allocs,
|
|
actual_rp_allocs,
|
|
f"The consumer {consumer} expected to have allocation "
|
|
f"{expected_rp_allocs} on {rp_name} but it has "
|
|
f"{actual_rp_allocs} instead."
|
|
)
|
|
|
|
def assert_placement_pci_allocations_on_host(self, hostname, allocations):
|
|
compute_rp_uuid = self.compute_rp_uuids[hostname]
|
|
rps = self._get_all_rps_in_a_tree(compute_rp_uuid)
|
|
|
|
for consumer, expected_allocations in allocations.items():
|
|
actual_allocations = self._get_allocations_by_server_uuid(consumer)
|
|
self.assertEqual(
|
|
len(expected_allocations),
|
|
# actual_allocations also contains allocations against the
|
|
# root provider for VCPU, MEMORY_MB, and DISK_GB so subtract
|
|
# one
|
|
len(actual_allocations) - 1,
|
|
f"The consumer {consumer} allocates from different number of "
|
|
f"RPs than expected. Expected: {expected_allocations}, "
|
|
f"Actual: {actual_allocations}"
|
|
)
|
|
for rp_name, expected_rp_allocs in expected_allocations.items():
|
|
real_rp_name = f'{hostname}_{rp_name}'
|
|
rp = self._get_rp_by_name(real_rp_name, rps)
|
|
self.assertIn(
|
|
rp['uuid'],
|
|
actual_allocations,
|
|
f"The consumer {consumer} expected to allocate from "
|
|
f"{rp['uuid']}. Expected: {expected_allocations}, "
|
|
f"Actual: {actual_allocations}"
|
|
)
|
|
actual_rp_allocs = actual_allocations[rp['uuid']]['resources']
|
|
self.assertEqual(
|
|
expected_rp_allocs,
|
|
actual_rp_allocs,
|
|
f"The consumer {consumer} expected to have allocation "
|
|
f"{expected_rp_allocs} on {rp_name} but it has "
|
|
f"{actual_rp_allocs} instead."
|
|
)
|
|
|
|
def assert_placement_pci_view(
|
|
self, hostname, inventories, traits, usages=None, allocations=None
|
|
):
|
|
if not usages:
|
|
usages = {}
|
|
|
|
if not allocations:
|
|
allocations = {}
|
|
|
|
self.assert_placement_pci_inventory(hostname, inventories, traits)
|
|
self.assert_placement_pci_usages(hostname, usages)
|
|
self.assert_placement_pci_allocations_on_host(hostname, allocations)
|
|
|
|
@staticmethod
|
|
def _to_list_of_json_str(list):
|
|
return [jsonutils.dumps(x) for x in list]
|
|
|
|
@staticmethod
|
|
def _move_allocation(allocations, from_uuid, to_uuid):
|
|
allocations[to_uuid] = allocations[from_uuid]
|
|
del allocations[from_uuid]
|
|
|
|
def _move_server_allocation(self, allocations, server_uuid, revert=False):
|
|
migration_uuid = self.get_migration_uuid_for_instance(server_uuid)
|
|
if revert:
|
|
self._move_allocation(allocations, migration_uuid, server_uuid)
|
|
else:
|
|
self._move_allocation(allocations, server_uuid, migration_uuid)
|
|
|
|
|
|
class _PCIServersWithMigrationTestBase(_PCIServersTestBase):
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
|
|
self.useFixture(fixtures.MonkeyPatch(
|
|
'nova.tests.fixtures.libvirt.Domain.migrateToURI3',
|
|
self._migrate_stub))
|
|
|
|
def _migrate_stub(self, domain, destination, params, flags):
|
|
"""Stub out migrateToURI3."""
|
|
|
|
src_hostname = domain._connection.hostname
|
|
dst_hostname = urlparse.urlparse(destination).netloc
|
|
|
|
# In a real live migration, libvirt and QEMU on the source and
|
|
# destination talk it out, resulting in the instance starting to exist
|
|
# on the destination. Fakelibvirt cannot do that, so we have to
|
|
# manually create the "incoming" instance on the destination
|
|
# fakelibvirt.
|
|
dst = self.computes[dst_hostname]
|
|
dst.driver._host.get_connection().createXML(
|
|
params['destination_xml'],
|
|
'fake-createXML-doesnt-care-about-flags')
|
|
|
|
src = self.computes[src_hostname]
|
|
conn = src.driver._host.get_connection()
|
|
|
|
# because migrateToURI3 is spawned in a background thread, this method
|
|
# does not block the upper nova layers. Because we don't want nova to
|
|
# think the live migration has finished until this method is done, the
|
|
# last thing we do is make fakelibvirt's Domain.jobStats() return
|
|
# VIR_DOMAIN_JOB_COMPLETED.
|
|
server = etree.fromstring(
|
|
params['destination_xml']
|
|
).find('./uuid').text
|
|
dom = conn.lookupByUUIDString(server)
|
|
dom.complete_job()
|
|
|
|
|
|
class SRIOVServersTest(_PCIServersWithMigrationTestBase):
|
|
|
|
# TODO(stephenfin): We're using this because we want to be able to force
|
|
# the host during scheduling. We should instead look at overriding policy
|
|
ADMIN_API = True
|
|
microversion = 'latest'
|
|
|
|
VFS_ALIAS_NAME = 'vfs'
|
|
PFS_ALIAS_NAME = 'pfs'
|
|
|
|
PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in (
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PF_PROD_ID,
|
|
'physical_network': 'physnet4',
|
|
},
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.VF_PROD_ID,
|
|
'physical_network': 'physnet4',
|
|
},
|
|
)]
|
|
# PFs will be removed from pools unless they are specifically
|
|
# requested, so we explicitly request them with the 'device_type'
|
|
# attribute
|
|
PCI_ALIAS = [jsonutils.dumps(x) for x in (
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PF_PROD_ID,
|
|
'device_type': fields.PciDeviceType.SRIOV_PF,
|
|
'name': PFS_ALIAS_NAME,
|
|
},
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.VF_PROD_ID,
|
|
'name': VFS_ALIAS_NAME,
|
|
},
|
|
)]
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
|
|
# The ultimate base class _IntegratedTestBase uses NeutronFixture but
|
|
# we need a bit more intelligent neutron for these tests. Applying the
|
|
# new fixture here means that we re-stub what the previous neutron
|
|
# fixture already stubbed.
|
|
self.neutron = self.useFixture(base.LibvirtNeutronFixture(self))
|
|
|
|
def _disable_sriov_in_pf(self, pci_info):
|
|
# Check for PF and change the capability from virt_functions
|
|
# Delete all the VFs
|
|
vfs_to_delete = []
|
|
|
|
for device_name, device in pci_info.devices.items():
|
|
if 'virt_functions' in device.pci_device:
|
|
device.generate_xml(skip_capability=True)
|
|
elif 'phys_function' in device.pci_device:
|
|
vfs_to_delete.append(device_name)
|
|
|
|
for device in vfs_to_delete:
|
|
del pci_info.devices[device]
|
|
|
|
def test_create_server_with_VF(self):
|
|
"""Create a server with an SR-IOV VF-type PCI device."""
|
|
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo()
|
|
self.start_compute(pci_info=pci_info)
|
|
|
|
# create a server
|
|
extra_spec = {"pci_passthrough:alias": "%s:1" % self.VFS_ALIAS_NAME}
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
self._create_server(flavor_id=flavor_id, networks='none')
|
|
|
|
# ensure the filter was called
|
|
self.assertTrue(self.mock_filter.called)
|
|
|
|
def test_create_server_with_PF(self):
|
|
"""Create a server with an SR-IOV PF-type PCI device."""
|
|
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo()
|
|
self.start_compute(pci_info=pci_info)
|
|
|
|
# create a server
|
|
extra_spec = {"pci_passthrough:alias": "%s:1" % self.PFS_ALIAS_NAME}
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
self._create_server(flavor_id=flavor_id, networks='none')
|
|
|
|
# ensure the filter was called
|
|
self.assertTrue(self.mock_filter.called)
|
|
|
|
def test_create_server_with_PF_no_VF(self):
|
|
"""Create a server with a PF and ensure the VFs are then reserved."""
|
|
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=4)
|
|
self.start_compute(pci_info=pci_info)
|
|
|
|
# create a server using the PF
|
|
extra_spec_pfs = {"pci_passthrough:alias": f"{self.PFS_ALIAS_NAME}:1"}
|
|
flavor_id_pfs = self._create_flavor(extra_spec=extra_spec_pfs)
|
|
self._create_server(flavor_id=flavor_id_pfs, networks='none')
|
|
|
|
# now attempt to build another server, this time using the VF; this
|
|
# should fail because the VF is used by an instance
|
|
extra_spec_vfs = {"pci_passthrough:alias": f"{self.VFS_ALIAS_NAME}:1"}
|
|
flavor_id_vfs = self._create_flavor(extra_spec=extra_spec_vfs)
|
|
self._create_server(
|
|
flavor_id=flavor_id_vfs, networks='none', expected_state='ERROR',
|
|
)
|
|
|
|
def test_create_server_with_VF_no_PF(self):
|
|
"""Create a server with a VF and ensure the PF is then reserved."""
|
|
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=4)
|
|
self.start_compute(pci_info=pci_info)
|
|
|
|
# create a server using the VF
|
|
extra_spec_vfs = {'pci_passthrough:alias': f'{self.VFS_ALIAS_NAME}:1'}
|
|
flavor_id_vfs = self._create_flavor(extra_spec=extra_spec_vfs)
|
|
self._create_server(flavor_id=flavor_id_vfs, networks='none')
|
|
|
|
# now attempt to build another server, this time using the PF; this
|
|
# should fail because the PF is used by an instance
|
|
extra_spec_pfs = {'pci_passthrough:alias': f'{self.PFS_ALIAS_NAME}:1'}
|
|
flavor_id_pfs = self._create_flavor(extra_spec=extra_spec_pfs)
|
|
self._create_server(
|
|
flavor_id=flavor_id_pfs, networks='none', expected_state='ERROR',
|
|
)
|
|
|
|
def test_create_server_with_neutron(self):
|
|
"""Create an instance using a neutron-provisioned SR-IOV VIF."""
|
|
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=2)
|
|
|
|
orig_create = nova.virt.libvirt.guest.Guest.create
|
|
|
|
def fake_create(cls, xml, host):
|
|
tree = etree.fromstring(xml)
|
|
elem = tree.find('./devices/interface/source/address')
|
|
|
|
# compare address
|
|
expected = ('0x81', '0x00', '0x2')
|
|
actual = (
|
|
elem.get('bus'), elem.get('slot'), elem.get('function'),
|
|
)
|
|
self.assertEqual(expected, actual)
|
|
|
|
return orig_create(xml, host)
|
|
|
|
self.stub_out(
|
|
'nova.virt.libvirt.guest.Guest.create',
|
|
fake_create,
|
|
)
|
|
|
|
self.start_compute(pci_info=pci_info)
|
|
|
|
# create the port
|
|
self.neutron.create_port({'port': self.neutron.network_4_port_1})
|
|
|
|
# ensure the binding details are currently unset
|
|
port = self.neutron.show_port(
|
|
base.LibvirtNeutronFixture.network_4_port_1['id'],
|
|
)['port']
|
|
self.assertNotIn('binding:profile', port)
|
|
|
|
# create a server using the VF via neutron
|
|
self._create_server(
|
|
networks=[
|
|
{'port': base.LibvirtNeutronFixture.network_4_port_1['id']},
|
|
],
|
|
)
|
|
|
|
# ensure the binding details sent to "neutron" were correct
|
|
port = self.neutron.show_port(
|
|
base.LibvirtNeutronFixture.network_4_port_1['id'],
|
|
)['port']
|
|
self.assertIn('binding:profile', port)
|
|
self.assertEqual(
|
|
{
|
|
'pci_vendor_info': '8086:1515',
|
|
'pci_slot': '0000:81:00.2',
|
|
'physical_network': 'physnet4',
|
|
},
|
|
port['binding:profile'],
|
|
)
|
|
|
|
def test_live_migrate_server_with_PF(self):
|
|
"""Live migrate an instance with a PCI PF.
|
|
|
|
This should fail because it's not possible to live migrate an instance
|
|
with a PCI passthrough device, even if it's a SR-IOV PF.
|
|
"""
|
|
|
|
# start two compute services
|
|
self.start_compute(
|
|
hostname='test_compute0',
|
|
pci_info=fakelibvirt.HostPCIDevicesInfo(num_pfs=2, num_vfs=4))
|
|
self.start_compute(
|
|
hostname='test_compute1',
|
|
pci_info=fakelibvirt.HostPCIDevicesInfo(num_pfs=2, num_vfs=4))
|
|
|
|
# create a server
|
|
extra_spec = {'pci_passthrough:alias': f'{self.PFS_ALIAS_NAME}:1'}
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
server = self._create_server(flavor_id=flavor_id, networks='none')
|
|
|
|
# now live migrate that server
|
|
ex = self.assertRaises(
|
|
client.OpenStackApiException,
|
|
self._live_migrate,
|
|
server, 'completed')
|
|
# NOTE(stephenfin): this wouldn't happen in a real deployment since
|
|
# live migration is a cast, but since we are using CastAsCallFixture
|
|
# this will bubble to the API
|
|
self.assertEqual(500, ex.response.status_code)
|
|
self.assertIn('NoValidHost', str(ex))
|
|
|
|
def test_live_migrate_server_with_VF(self):
|
|
"""Live migrate an instance with a PCI VF.
|
|
|
|
This should fail because it's not possible to live migrate an instance
|
|
with a PCI passthrough device, even if it's a SR-IOV VF.
|
|
"""
|
|
|
|
# start two compute services
|
|
self.start_compute(
|
|
hostname='test_compute0',
|
|
pci_info=fakelibvirt.HostPCIDevicesInfo(num_pfs=2, num_vfs=4))
|
|
self.start_compute(
|
|
hostname='test_compute1',
|
|
pci_info=fakelibvirt.HostPCIDevicesInfo(num_pfs=2, num_vfs=4))
|
|
|
|
# create a server
|
|
extra_spec = {'pci_passthrough:alias': f'{self.VFS_ALIAS_NAME}:1'}
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
server = self._create_server(flavor_id=flavor_id, networks='none')
|
|
|
|
# now live migrate that server
|
|
ex = self.assertRaises(
|
|
client.OpenStackApiException,
|
|
self._live_migrate,
|
|
server, 'completed')
|
|
# NOTE(stephenfin): this wouldn't happen in a real deployment since
|
|
# live migration is a cast, but since we are using CastAsCallFixture
|
|
# this will bubble to the API
|
|
self.assertEqual(500, ex.response.status_code)
|
|
self.assertIn('NoValidHost', str(ex))
|
|
|
|
def _test_move_operation_with_neutron(self, move_operation,
|
|
expect_fail=False):
|
|
# The purpose here is to force an observable PCI slot update when
|
|
# moving from source to dest. This is accomplished by having a single
|
|
# PCI VF device on the source, 2 PCI VF devices on the dest, and
|
|
# relying on the fact that our fake HostPCIDevicesInfo creates
|
|
# predictable PCI addresses. The PCI VF device on source and the first
|
|
# PCI VF device on dest will have identical PCI addresses. By sticking
|
|
# a "placeholder" instance on that first PCI VF device on the dest, the
|
|
# incoming instance from source will be forced to consume the second
|
|
# dest PCI VF device, with a different PCI address.
|
|
# We want to test server operations with SRIOV VFs and SRIOV PFs so
|
|
# the config of the compute hosts also have one extra PCI PF devices
|
|
# without any VF children. But the two compute has different PCI PF
|
|
# addresses and MAC so that the test can observe the slot update as
|
|
# well as the MAC updated during migration and after revert.
|
|
source_pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=1)
|
|
# add an extra PF without VF to be used by direct-physical ports
|
|
source_pci_info.add_device(
|
|
dev_type='PF',
|
|
bus=0x82, # the HostPCIDevicesInfo use the 0x81 by default
|
|
slot=0x0,
|
|
function=0,
|
|
iommu_group=42,
|
|
numa_node=0,
|
|
vf_ratio=0,
|
|
mac_address='b4:96:91:34:f4:aa',
|
|
)
|
|
self.start_compute(
|
|
hostname='source',
|
|
pci_info=source_pci_info)
|
|
|
|
dest_pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=2)
|
|
# add an extra PF without VF to be used by direct-physical ports
|
|
dest_pci_info.add_device(
|
|
dev_type='PF',
|
|
bus=0x82, # the HostPCIDevicesInfo use the 0x81 by default
|
|
slot=0x6, # make it different from the source host
|
|
function=0,
|
|
iommu_group=42,
|
|
numa_node=0,
|
|
vf_ratio=0,
|
|
mac_address='b4:96:91:34:f4:bb',
|
|
)
|
|
self.start_compute(
|
|
hostname='dest',
|
|
pci_info=dest_pci_info)
|
|
|
|
source_port = self.neutron.create_port(
|
|
{'port': self.neutron.network_4_port_1})
|
|
source_pf_port = self.neutron.create_port(
|
|
{'port': self.neutron.network_4_port_pf})
|
|
dest_port1 = self.neutron.create_port(
|
|
{'port': self.neutron.network_4_port_2})
|
|
dest_port2 = self.neutron.create_port(
|
|
{'port': self.neutron.network_4_port_3})
|
|
|
|
source_server = self._create_server(
|
|
networks=[
|
|
{'port': source_port['port']['id']},
|
|
{'port': source_pf_port['port']['id']}
|
|
],
|
|
host='source',
|
|
)
|
|
dest_server1 = self._create_server(
|
|
networks=[{'port': dest_port1['port']['id']}], host='dest')
|
|
dest_server2 = self._create_server(
|
|
networks=[{'port': dest_port2['port']['id']}], host='dest')
|
|
|
|
# Refresh the ports.
|
|
source_port = self.neutron.show_port(source_port['port']['id'])
|
|
source_pf_port = self.neutron.show_port(source_pf_port['port']['id'])
|
|
dest_port1 = self.neutron.show_port(dest_port1['port']['id'])
|
|
dest_port2 = self.neutron.show_port(dest_port2['port']['id'])
|
|
|
|
# Find the server on the dest compute that's using the same pci_slot as
|
|
# the server on the source compute, and delete the other one to make
|
|
# room for the incoming server from the source.
|
|
source_pci_slot = source_port['port']['binding:profile']['pci_slot']
|
|
dest_pci_slot1 = dest_port1['port']['binding:profile']['pci_slot']
|
|
if dest_pci_slot1 == source_pci_slot:
|
|
same_slot_port = dest_port1
|
|
self._delete_server(dest_server2)
|
|
else:
|
|
same_slot_port = dest_port2
|
|
self._delete_server(dest_server1)
|
|
|
|
# Before moving, explicitly assert that the servers on source and dest
|
|
# have the same pci_slot in their port's binding profile
|
|
self.assertEqual(source_port['port']['binding:profile']['pci_slot'],
|
|
same_slot_port['port']['binding:profile']['pci_slot'])
|
|
|
|
# Assert that the direct-physical port got the pci_slot information
|
|
# according to the source host PF PCI device.
|
|
self.assertEqual(
|
|
'0000:82:00.0', # which is in sync with the source host pci_info
|
|
source_pf_port['port']['binding:profile']['pci_slot']
|
|
)
|
|
# Assert that the direct-physical port is updated with the MAC address
|
|
# of the PF device from the source host
|
|
self.assertEqual(
|
|
'b4:96:91:34:f4:aa',
|
|
source_pf_port['port']['binding:profile']['device_mac_address']
|
|
)
|
|
|
|
# Before moving, assert that the servers on source and dest have the
|
|
# same PCI source address in their XML for their SRIOV nic.
|
|
source_conn = self.computes['source'].driver._host.get_connection()
|
|
dest_conn = self.computes['source'].driver._host.get_connection()
|
|
source_vms = [vm._def for vm in source_conn._vms.values()]
|
|
dest_vms = [vm._def for vm in dest_conn._vms.values()]
|
|
self.assertEqual(1, len(source_vms))
|
|
self.assertEqual(1, len(dest_vms))
|
|
self.assertEqual(1, len(source_vms[0]['devices']['nics']))
|
|
self.assertEqual(1, len(dest_vms[0]['devices']['nics']))
|
|
self.assertEqual(source_vms[0]['devices']['nics'][0]['source'],
|
|
dest_vms[0]['devices']['nics'][0]['source'])
|
|
|
|
move_operation(source_server)
|
|
|
|
# Refresh the ports again, keeping in mind the source_port is now bound
|
|
# on the dest after the move.
|
|
source_port = self.neutron.show_port(source_port['port']['id'])
|
|
same_slot_port = self.neutron.show_port(same_slot_port['port']['id'])
|
|
source_pf_port = self.neutron.show_port(source_pf_port['port']['id'])
|
|
|
|
self.assertNotEqual(
|
|
source_port['port']['binding:profile']['pci_slot'],
|
|
same_slot_port['port']['binding:profile']['pci_slot'])
|
|
|
|
# Assert that the direct-physical port got the pci_slot information
|
|
# according to the dest host PF PCI device.
|
|
self.assertEqual(
|
|
'0000:82:06.0', # which is in sync with the dest host pci_info
|
|
source_pf_port['port']['binding:profile']['pci_slot']
|
|
)
|
|
# Assert that the direct-physical port is updated with the MAC address
|
|
# of the PF device from the dest host
|
|
self.assertEqual(
|
|
'b4:96:91:34:f4:bb',
|
|
source_pf_port['port']['binding:profile']['device_mac_address']
|
|
)
|
|
|
|
conn = self.computes['dest'].driver._host.get_connection()
|
|
vms = [vm._def for vm in conn._vms.values()]
|
|
self.assertEqual(2, len(vms))
|
|
for vm in vms:
|
|
self.assertEqual(1, len(vm['devices']['nics']))
|
|
self.assertNotEqual(vms[0]['devices']['nics'][0]['source'],
|
|
vms[1]['devices']['nics'][0]['source'])
|
|
|
|
def test_unshelve_server_with_neutron(self):
|
|
def move_operation(source_server):
|
|
self._shelve_server(source_server)
|
|
# Disable the source compute, to force unshelving on the dest.
|
|
self.api.put_service(self.computes['source'].service_ref.uuid,
|
|
{'status': 'disabled'})
|
|
self._unshelve_server(source_server)
|
|
self._test_move_operation_with_neutron(move_operation)
|
|
|
|
def test_cold_migrate_server_with_neutron(self):
|
|
def move_operation(source_server):
|
|
# TODO(stephenfin): The mock of 'migrate_disk_and_power_off' should
|
|
# probably be less...dumb
|
|
with mock.patch('nova.virt.libvirt.driver.LibvirtDriver'
|
|
'.migrate_disk_and_power_off', return_value='{}'):
|
|
self._migrate_server(source_server)
|
|
self._confirm_resize(source_server)
|
|
self._test_move_operation_with_neutron(move_operation)
|
|
|
|
def test_cold_migrate_and_rever_server_with_neutron(self):
|
|
# The purpose here is to force an observable PCI slot update when
|
|
# moving from source to dest and the from dest to source after the
|
|
# revert. This is accomplished by having a single
|
|
# PCI VF device on the source, 2 PCI VF devices on the dest, and
|
|
# relying on the fact that our fake HostPCIDevicesInfo creates
|
|
# predictable PCI addresses. The PCI VF device on source and the first
|
|
# PCI VF device on dest will have identical PCI addresses. By sticking
|
|
# a "placeholder" instance on that first PCI VF device on the dest, the
|
|
# incoming instance from source will be forced to consume the second
|
|
# dest PCI VF device, with a different PCI address.
|
|
# We want to test server operations with SRIOV VFs and SRIOV PFs so
|
|
# the config of the compute hosts also have one extra PCI PF devices
|
|
# without any VF children. But the two compute has different PCI PF
|
|
# addresses and MAC so that the test can observe the slot update as
|
|
# well as the MAC updated during migration and after revert.
|
|
source_pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=1)
|
|
# add an extra PF without VF to be used by direct-physical ports
|
|
source_pci_info.add_device(
|
|
dev_type='PF',
|
|
bus=0x82, # the HostPCIDevicesInfo use the 0x81 by default
|
|
slot=0x0,
|
|
function=0,
|
|
iommu_group=42,
|
|
numa_node=0,
|
|
vf_ratio=0,
|
|
mac_address='b4:96:91:34:f4:aa',
|
|
)
|
|
self.start_compute(
|
|
hostname='source',
|
|
pci_info=source_pci_info)
|
|
dest_pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=2)
|
|
# add an extra PF without VF to be used by direct-physical ports
|
|
dest_pci_info.add_device(
|
|
dev_type='PF',
|
|
bus=0x82, # the HostPCIDevicesInfo use the 0x81 by default
|
|
slot=0x6, # make it different from the source host
|
|
function=0,
|
|
iommu_group=42,
|
|
numa_node=0,
|
|
vf_ratio=0,
|
|
mac_address='b4:96:91:34:f4:bb',
|
|
)
|
|
self.start_compute(
|
|
hostname='dest',
|
|
pci_info=dest_pci_info)
|
|
source_port = self.neutron.create_port(
|
|
{'port': self.neutron.network_4_port_1})
|
|
source_pf_port = self.neutron.create_port(
|
|
{'port': self.neutron.network_4_port_pf})
|
|
dest_port1 = self.neutron.create_port(
|
|
{'port': self.neutron.network_4_port_2})
|
|
dest_port2 = self.neutron.create_port(
|
|
{'port': self.neutron.network_4_port_3})
|
|
source_server = self._create_server(
|
|
networks=[
|
|
{'port': source_port['port']['id']},
|
|
{'port': source_pf_port['port']['id']}
|
|
],
|
|
host='source',
|
|
)
|
|
dest_server1 = self._create_server(
|
|
networks=[{'port': dest_port1['port']['id']}], host='dest')
|
|
dest_server2 = self._create_server(
|
|
networks=[{'port': dest_port2['port']['id']}], host='dest')
|
|
# Refresh the ports.
|
|
source_port = self.neutron.show_port(source_port['port']['id'])
|
|
source_pf_port = self.neutron.show_port(source_pf_port['port']['id'])
|
|
dest_port1 = self.neutron.show_port(dest_port1['port']['id'])
|
|
dest_port2 = self.neutron.show_port(dest_port2['port']['id'])
|
|
# Find the server on the dest compute that's using the same pci_slot as
|
|
# the server on the source compute, and delete the other one to make
|
|
# room for the incoming server from the source.
|
|
source_pci_slot = source_port['port']['binding:profile']['pci_slot']
|
|
dest_pci_slot1 = dest_port1['port']['binding:profile']['pci_slot']
|
|
if dest_pci_slot1 == source_pci_slot:
|
|
same_slot_port = dest_port1
|
|
self._delete_server(dest_server2)
|
|
else:
|
|
same_slot_port = dest_port2
|
|
self._delete_server(dest_server1)
|
|
# Before moving, explicitly assert that the servers on source and dest
|
|
# have the same pci_slot in their port's binding profile
|
|
self.assertEqual(source_port['port']['binding:profile']['pci_slot'],
|
|
same_slot_port['port']['binding:profile']['pci_slot'])
|
|
# Assert that the direct-physical port got the pci_slot information
|
|
# according to the source host PF PCI device.
|
|
self.assertEqual(
|
|
'0000:82:00.0', # which is in sync with the source host pci_info
|
|
source_pf_port['port']['binding:profile']['pci_slot']
|
|
)
|
|
# Assert that the direct-physical port is updated with the MAC address
|
|
# of the PF device from the source host
|
|
self.assertEqual(
|
|
'b4:96:91:34:f4:aa',
|
|
source_pf_port['port']['binding:profile']['device_mac_address']
|
|
)
|
|
# Before moving, assert that the servers on source and dest have the
|
|
# same PCI source address in their XML for their SRIOV nic.
|
|
source_conn = self.computes['source'].driver._host.get_connection()
|
|
dest_conn = self.computes['source'].driver._host.get_connection()
|
|
source_vms = [vm._def for vm in source_conn._vms.values()]
|
|
dest_vms = [vm._def for vm in dest_conn._vms.values()]
|
|
self.assertEqual(1, len(source_vms))
|
|
self.assertEqual(1, len(dest_vms))
|
|
self.assertEqual(1, len(source_vms[0]['devices']['nics']))
|
|
self.assertEqual(1, len(dest_vms[0]['devices']['nics']))
|
|
self.assertEqual(source_vms[0]['devices']['nics'][0]['source'],
|
|
dest_vms[0]['devices']['nics'][0]['source'])
|
|
|
|
# TODO(stephenfin): The mock of 'migrate_disk_and_power_off' should
|
|
# probably be less...dumb
|
|
with mock.patch('nova.virt.libvirt.driver.LibvirtDriver'
|
|
'.migrate_disk_and_power_off', return_value='{}'):
|
|
self._migrate_server(source_server)
|
|
|
|
# Refresh the ports again, keeping in mind the ports are now bound
|
|
# on the dest after migrating.
|
|
source_port = self.neutron.show_port(source_port['port']['id'])
|
|
same_slot_port = self.neutron.show_port(same_slot_port['port']['id'])
|
|
source_pf_port = self.neutron.show_port(source_pf_port['port']['id'])
|
|
self.assertNotEqual(
|
|
source_port['port']['binding:profile']['pci_slot'],
|
|
same_slot_port['port']['binding:profile']['pci_slot'])
|
|
# Assert that the direct-physical port got the pci_slot information
|
|
# according to the dest host PF PCI device.
|
|
self.assertEqual(
|
|
'0000:82:06.0', # which is in sync with the dest host pci_info
|
|
source_pf_port['port']['binding:profile']['pci_slot']
|
|
)
|
|
# Assert that the direct-physical port is updated with the MAC address
|
|
# of the PF device from the dest host
|
|
self.assertEqual(
|
|
'b4:96:91:34:f4:bb',
|
|
source_pf_port['port']['binding:profile']['device_mac_address']
|
|
)
|
|
conn = self.computes['dest'].driver._host.get_connection()
|
|
vms = [vm._def for vm in conn._vms.values()]
|
|
self.assertEqual(2, len(vms))
|
|
for vm in vms:
|
|
self.assertEqual(1, len(vm['devices']['nics']))
|
|
self.assertNotEqual(vms[0]['devices']['nics'][0]['source'],
|
|
vms[1]['devices']['nics'][0]['source'])
|
|
|
|
self._revert_resize(source_server)
|
|
|
|
# Refresh the ports again, keeping in mind the ports are now bound
|
|
# on the source as the migration is reverted
|
|
source_pf_port = self.neutron.show_port(source_pf_port['port']['id'])
|
|
|
|
# Assert that the direct-physical port got the pci_slot information
|
|
# according to the source host PF PCI device.
|
|
self.assertEqual(
|
|
'0000:82:00.0', # which is in sync with the source host pci_info
|
|
source_pf_port['port']['binding:profile']['pci_slot']
|
|
)
|
|
# Assert that the direct-physical port is updated with the MAC address
|
|
# of the PF device from the source host
|
|
self.assertEqual(
|
|
'b4:96:91:34:f4:aa',
|
|
source_pf_port['port']['binding:profile']['device_mac_address']
|
|
)
|
|
|
|
def test_evacuate_server_with_neutron(self):
|
|
def move_operation(source_server):
|
|
# Down the source compute to enable the evacuation
|
|
self.api.put_service(self.computes['source'].service_ref.uuid,
|
|
{'forced_down': True})
|
|
self.computes['source'].stop()
|
|
self._evacuate_server(source_server)
|
|
self._test_move_operation_with_neutron(move_operation)
|
|
|
|
def test_live_migrate_server_with_neutron(self):
|
|
"""Live migrate an instance using a neutron-provisioned SR-IOV VIF.
|
|
|
|
This should succeed since we support this, via detach and attach of the
|
|
PCI device.
|
|
"""
|
|
|
|
# start two compute services with differing PCI device inventory
|
|
source_pci_info = fakelibvirt.HostPCIDevicesInfo(
|
|
num_pfs=1, num_vfs=4, numa_node=0)
|
|
# add an extra PF without VF to be used by direct-physical ports
|
|
source_pci_info.add_device(
|
|
dev_type='PF',
|
|
bus=0x82, # the HostPCIDevicesInfo use the 0x81 by default
|
|
slot=0x0,
|
|
function=0,
|
|
iommu_group=42,
|
|
numa_node=0,
|
|
vf_ratio=0,
|
|
mac_address='b4:96:91:34:f4:aa',
|
|
)
|
|
self.start_compute(hostname='test_compute0', pci_info=source_pci_info)
|
|
|
|
dest_pci_info = fakelibvirt.HostPCIDevicesInfo(
|
|
num_pfs=1, num_vfs=2, numa_node=1)
|
|
# add an extra PF without VF to be used by direct-physical ports
|
|
dest_pci_info.add_device(
|
|
dev_type='PF',
|
|
bus=0x82, # the HostPCIDevicesInfo use the 0x81 by default
|
|
slot=0x6, # make it different from the source host
|
|
function=0,
|
|
iommu_group=42,
|
|
# numa node needs to be aligned with the other pci devices in this
|
|
# host as the instance needs to fit into a single host numa node
|
|
numa_node=1,
|
|
vf_ratio=0,
|
|
mac_address='b4:96:91:34:f4:bb',
|
|
)
|
|
|
|
self.start_compute(hostname='test_compute1', pci_info=dest_pci_info)
|
|
|
|
# create the ports
|
|
port = self.neutron.create_port(
|
|
{'port': self.neutron.network_4_port_1})['port']
|
|
pf_port = self.neutron.create_port(
|
|
{'port': self.neutron.network_4_port_pf})['port']
|
|
|
|
# create a server using the VF via neutron
|
|
extra_spec = {'hw:cpu_policy': 'dedicated'}
|
|
flavor_id = self._create_flavor(vcpu=4, extra_spec=extra_spec)
|
|
server = self._create_server(
|
|
flavor_id=flavor_id,
|
|
networks=[
|
|
{'port': port['id']},
|
|
{'port': pf_port['id']},
|
|
],
|
|
host='test_compute0',
|
|
)
|
|
|
|
# our source host should have marked two PCI devices as used, the VF
|
|
# and the parent PF, while the future destination is currently unused
|
|
self.assertEqual('test_compute0', server['OS-EXT-SRV-ATTR:host'])
|
|
self.assertPCIDeviceCounts('test_compute0', total=6, free=3)
|
|
self.assertPCIDeviceCounts('test_compute1', total=4, free=4)
|
|
|
|
# the instance should be on host NUMA node 0, since that's where our
|
|
# PCI devices are
|
|
host_numa = objects.NUMATopology.obj_from_db_obj(
|
|
objects.ComputeNode.get_by_nodename(
|
|
self.ctxt, 'test_compute0',
|
|
).numa_topology
|
|
)
|
|
self.assertEqual({0, 1, 2, 3}, host_numa.cells[0].pinned_cpus)
|
|
self.assertEqual(set(), host_numa.cells[1].pinned_cpus)
|
|
|
|
# ensure the binding details sent to "neutron" are correct
|
|
port = self.neutron.show_port(
|
|
base.LibvirtNeutronFixture.network_4_port_1['id'],
|
|
)['port']
|
|
self.assertIn('binding:profile', port)
|
|
self.assertEqual(
|
|
{
|
|
'pci_vendor_info': '8086:1515',
|
|
# TODO(stephenfin): Stop relying on a side-effect of how nova
|
|
# chooses from multiple PCI devices (apparently the last
|
|
# matching one)
|
|
'pci_slot': '0000:81:00.4',
|
|
'physical_network': 'physnet4',
|
|
},
|
|
port['binding:profile'],
|
|
)
|
|
|
|
# ensure the binding details sent to "neutron" are correct
|
|
pf_port = self.neutron.show_port(pf_port['id'],)['port']
|
|
self.assertIn('binding:profile', pf_port)
|
|
self.assertEqual(
|
|
{
|
|
'pci_vendor_info': '8086:1528',
|
|
'pci_slot': '0000:82:00.0',
|
|
'physical_network': 'physnet4',
|
|
'device_mac_address': 'b4:96:91:34:f4:aa',
|
|
},
|
|
pf_port['binding:profile'],
|
|
)
|
|
|
|
# now live migrate that server
|
|
self._live_migrate(server, 'completed')
|
|
|
|
# we should now have transitioned our usage to the destination, freeing
|
|
# up the source in the process
|
|
self.assertPCIDeviceCounts('test_compute0', total=6, free=6)
|
|
self.assertPCIDeviceCounts('test_compute1', total=4, free=1)
|
|
|
|
# the instance should now be on host NUMA node 1, since that's where
|
|
# our PCI devices are for this second host
|
|
host_numa = objects.NUMATopology.obj_from_db_obj(
|
|
objects.ComputeNode.get_by_nodename(
|
|
self.ctxt, 'test_compute1',
|
|
).numa_topology
|
|
)
|
|
self.assertEqual(set(), host_numa.cells[0].pinned_cpus)
|
|
self.assertEqual({4, 5, 6, 7}, host_numa.cells[1].pinned_cpus)
|
|
|
|
# ensure the binding details sent to "neutron" have been updated
|
|
port = self.neutron.show_port(
|
|
base.LibvirtNeutronFixture.network_4_port_1['id'],
|
|
)['port']
|
|
self.assertIn('binding:profile', port)
|
|
self.assertEqual(
|
|
{
|
|
'pci_vendor_info': '8086:1515',
|
|
'pci_slot': '0000:81:00.2',
|
|
'physical_network': 'physnet4',
|
|
},
|
|
port['binding:profile'],
|
|
)
|
|
# ensure the binding details sent to "neutron" are correct
|
|
pf_port = self.neutron.show_port(pf_port['id'],)['port']
|
|
self.assertIn('binding:profile', pf_port)
|
|
self.assertEqual(
|
|
{
|
|
'pci_vendor_info': '8086:1528',
|
|
'pci_slot': '0000:82:06.0',
|
|
'physical_network': 'physnet4',
|
|
'device_mac_address': 'b4:96:91:34:f4:bb',
|
|
},
|
|
pf_port['binding:profile'],
|
|
)
|
|
|
|
def test_get_server_diagnostics_server_with_VF(self):
|
|
"""Ensure server disagnostics include info on VF-type PCI devices."""
|
|
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo()
|
|
self.start_compute(pci_info=pci_info)
|
|
|
|
# create the SR-IOV port
|
|
port = self.neutron.create_port(
|
|
{'port': self.neutron.network_4_port_1})
|
|
|
|
flavor_id = self._create_flavor()
|
|
server = self._create_server(
|
|
flavor_id=flavor_id,
|
|
networks=[
|
|
{'uuid': base.LibvirtNeutronFixture.network_1['id']},
|
|
{'port': port['port']['id']},
|
|
],
|
|
)
|
|
|
|
# now check the server diagnostics to ensure the VF-type PCI device is
|
|
# attached
|
|
diagnostics = self.api.get_server_diagnostics(
|
|
server['id']
|
|
)
|
|
|
|
self.assertEqual(
|
|
base.LibvirtNeutronFixture.network_1_port_2['mac_address'],
|
|
diagnostics['nic_details'][0]['mac_address'],
|
|
)
|
|
|
|
for key in ('rx_packets', 'tx_packets'):
|
|
self.assertIn(key, diagnostics['nic_details'][0])
|
|
|
|
self.assertEqual(
|
|
base.LibvirtNeutronFixture.network_4_port_1['mac_address'],
|
|
diagnostics['nic_details'][1]['mac_address'],
|
|
)
|
|
for key in ('rx_packets', 'tx_packets'):
|
|
self.assertIn(key, diagnostics['nic_details'][1])
|
|
|
|
def test_create_server_after_change_in_nonsriov_pf_to_sriov_pf(self):
|
|
# Starts a compute with PF not configured with SRIOV capabilities
|
|
# Updates the PF with SRIOV capability and restart the compute service
|
|
# Then starts a VM with the sriov port. The VM should be in active
|
|
# state with sriov port attached.
|
|
|
|
# To emulate the device type changing, we first create a
|
|
# HostPCIDevicesInfo object with PFs and VFs. Then we make a copy
|
|
# and remove the VFs and the virt_function capability. This is
|
|
# done to ensure the physical function product id is same in both
|
|
# the versions.
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=1)
|
|
pci_info_no_sriov = copy.deepcopy(pci_info)
|
|
|
|
# Disable SRIOV capabilties in PF and delete the VFs
|
|
self._disable_sriov_in_pf(pci_info_no_sriov)
|
|
|
|
self.start_compute('test_compute0', pci_info=pci_info_no_sriov)
|
|
self.compute = self.computes['test_compute0']
|
|
|
|
ctxt = context.get_admin_context()
|
|
pci_devices = objects.PciDeviceList.get_by_compute_node(
|
|
ctxt,
|
|
objects.ComputeNode.get_by_nodename(
|
|
ctxt, 'test_compute0',
|
|
).id,
|
|
)
|
|
self.assertEqual(1, len(pci_devices))
|
|
self.assertEqual('type-PCI', pci_devices[0].dev_type)
|
|
|
|
# Restart the compute service with sriov PFs
|
|
self.restart_compute_service(
|
|
self.compute.host, pci_info=pci_info, keep_hypervisor_state=False)
|
|
|
|
# Verify if PCI devices are of type type-PF or type-VF
|
|
pci_devices = objects.PciDeviceList.get_by_compute_node(
|
|
ctxt,
|
|
objects.ComputeNode.get_by_nodename(
|
|
ctxt, 'test_compute0',
|
|
).id,
|
|
)
|
|
for pci_device in pci_devices:
|
|
self.assertIn(pci_device.dev_type, ['type-PF', 'type-VF'])
|
|
|
|
# create the port
|
|
self.neutron.create_port({'port': self.neutron.network_4_port_1})
|
|
|
|
# create a server using the VF via neutron
|
|
self._create_server(
|
|
networks=[
|
|
{'port': base.LibvirtNeutronFixture.network_4_port_1['id']},
|
|
],
|
|
)
|
|
|
|
def test_change_bound_port_vnic_type_kills_compute_at_restart(self):
|
|
"""Create a server with a direct port and change the vnic_type of the
|
|
bound port to macvtap. Then restart the compute service.
|
|
|
|
As the vnic_type is changed on the port but the vif_type is hwveb
|
|
instead of macvtap the vif plug logic will try to look up the netdev
|
|
of the parent VF. Howvere that VF consumed by the instance so the
|
|
netdev does not exists. This causes that the compute service will fail
|
|
with an exception during startup
|
|
"""
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=2)
|
|
self.start_compute(pci_info=pci_info)
|
|
|
|
# create a direct port
|
|
port = self.neutron.network_4_port_1
|
|
self.neutron.create_port({'port': port})
|
|
|
|
# create a server using the VF via neutron
|
|
server = self._create_server(networks=[{'port': port['id']}])
|
|
|
|
# update the vnic_type of the port in neutron
|
|
port = copy.deepcopy(port)
|
|
port['binding:vnic_type'] = 'macvtap'
|
|
self.neutron.update_port(port['id'], {"port": port})
|
|
|
|
compute = self.computes['compute1']
|
|
|
|
# Force an update on the instance info cache to ensure nova gets the
|
|
# information about the updated port
|
|
with context.target_cell(
|
|
context.get_admin_context(),
|
|
self.host_mappings['compute1'].cell_mapping
|
|
) as cctxt:
|
|
compute.manager._heal_instance_info_cache(cctxt)
|
|
self.assertIn(
|
|
'The vnic_type of the bound port %s has been changed in '
|
|
'neutron from "direct" to "macvtap". Changing vnic_type of a '
|
|
'bound port is not supported by Nova. To avoid breaking the '
|
|
'connectivity of the instance please change the port '
|
|
'vnic_type back to "direct".' % port['id'],
|
|
self.stdlog.logger.output,
|
|
)
|
|
|
|
def fake_get_ifname_by_pci_address(pci_addr: str, pf_interface=False):
|
|
# we want to fail the netdev lookup only if the pci_address is
|
|
# already consumed by our instance. So we look into the instance
|
|
# definition to see if the device is attached to the instance as VF
|
|
conn = compute.manager.driver._host.get_connection()
|
|
dom = conn.lookupByUUIDString(server['id'])
|
|
dev = dom._def['devices']['nics'][0]
|
|
lookup_addr = pci_addr.replace(':', '_').replace('.', '_')
|
|
if (
|
|
dev['type'] == 'hostdev' and
|
|
dev['source'] == 'pci_' + lookup_addr
|
|
):
|
|
# nova tried to look up the netdev of an already consumed VF.
|
|
# So we have to fail
|
|
raise exception.PciDeviceNotFoundById(id=pci_addr)
|
|
|
|
# We need to simulate the actual failure manually as in our functional
|
|
# environment all the PCI lookup is mocked. In reality nova tries to
|
|
# look up the netdev of the pci device on the host used by the port as
|
|
# the parent of the macvtap. However, as the originally direct port is
|
|
# bound to the instance, the VF pci device is already consumed by the
|
|
# instance and therefore there is no netdev for the VF.
|
|
self.libvirt.mock_get_ifname_by_pci_address.side_effect = (
|
|
fake_get_ifname_by_pci_address
|
|
)
|
|
# Nova cannot prevent the vnic_type change on a bound port. Neutron
|
|
# should prevent that instead. But the nova-compute should still
|
|
# be able to start up and only log an ERROR for this instance in
|
|
# inconsistent state.
|
|
self.restart_compute_service('compute1')
|
|
self.assertIn(
|
|
'Virtual interface plugging failed for instance. Probably the '
|
|
'vnic_type of the bound port has been changed. Nova does not '
|
|
'support such change.',
|
|
self.stdlog.logger.output,
|
|
)
|
|
|
|
|
|
class SRIOVAttachDetachTest(_PCIServersTestBase):
|
|
# no need for aliases as these test will request SRIOV via neutron
|
|
PCI_ALIAS = []
|
|
|
|
PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in (
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PF_PROD_ID,
|
|
"physical_network": "physnet2",
|
|
},
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.VF_PROD_ID,
|
|
"physical_network": "physnet2",
|
|
},
|
|
)]
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
|
|
self.neutron = self.useFixture(nova_fixtures.NeutronFixture(self))
|
|
|
|
# add extra ports and the related network to the neutron fixture
|
|
# specifically for these tests. It cannot be added globally in the
|
|
# fixture init as it adds a second network that makes auto allocation
|
|
# based test to fail due to ambiguous networks.
|
|
self.neutron._networks[
|
|
self.neutron.network_2['id']] = self.neutron.network_2
|
|
self.neutron._subnets[
|
|
self.neutron.subnet_2['id']] = self.neutron.subnet_2
|
|
for port in [self.neutron.sriov_port, self.neutron.sriov_port2,
|
|
self.neutron.sriov_pf_port, self.neutron.sriov_pf_port2,
|
|
self.neutron.macvtap_port, self.neutron.macvtap_port2]:
|
|
self.neutron._ports[port['id']] = copy.deepcopy(port)
|
|
|
|
def _get_attached_port_ids(self, instance_uuid):
|
|
return [
|
|
attachment['port_id']
|
|
for attachment in self.api.get_port_interfaces(instance_uuid)]
|
|
|
|
def _detach_port(self, instance_uuid, port_id):
|
|
self.api.detach_interface(instance_uuid, port_id)
|
|
self.notifier.wait_for_versioned_notifications(
|
|
'instance.interface_detach.end')
|
|
|
|
def _attach_port(self, instance_uuid, port_id):
|
|
self.api.attach_interface(
|
|
instance_uuid,
|
|
{'interfaceAttachment': {'port_id': port_id}})
|
|
self.notifier.wait_for_versioned_notifications(
|
|
'instance.interface_attach.end')
|
|
|
|
def _test_detach_attach(self, first_port_id, second_port_id):
|
|
# This test takes two ports that requires PCI claim.
|
|
# Starts a compute with one PF and one connected VF.
|
|
# Then starts a VM with the first port. Then detach it, then
|
|
# re-attach it. These expected to be successful. Then try to attach the
|
|
# second port and asserts that it fails as no free PCI device left on
|
|
# the host.
|
|
host_info = fakelibvirt.HostInfo(cpu_nodes=2, cpu_sockets=1,
|
|
cpu_cores=2, cpu_threads=2)
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=1)
|
|
self.start_compute(
|
|
'test_compute0', host_info=host_info, pci_info=pci_info)
|
|
self.compute = self.computes['test_compute0']
|
|
|
|
# Create server with a port
|
|
server = self._create_server(networks=[{'port': first_port_id}])
|
|
|
|
updated_port = self.neutron.show_port(first_port_id)['port']
|
|
self.assertEqual('test_compute0', updated_port['binding:host_id'])
|
|
self.assertIn(first_port_id, self._get_attached_port_ids(server['id']))
|
|
|
|
self._detach_port(server['id'], first_port_id)
|
|
|
|
updated_port = self.neutron.show_port(first_port_id)['port']
|
|
self.assertIsNone(updated_port['binding:host_id'])
|
|
self.assertNotIn(
|
|
first_port_id,
|
|
self._get_attached_port_ids(server['id']))
|
|
|
|
# Attach back the port
|
|
self._attach_port(server['id'], first_port_id)
|
|
|
|
updated_port = self.neutron.show_port(first_port_id)['port']
|
|
self.assertEqual('test_compute0', updated_port['binding:host_id'])
|
|
self.assertIn(first_port_id, self._get_attached_port_ids(server['id']))
|
|
|
|
# Try to attach the second port but no free PCI device left
|
|
ex = self.assertRaises(
|
|
client.OpenStackApiException, self._attach_port, server['id'],
|
|
second_port_id)
|
|
|
|
self.assertEqual(400, ex.response.status_code)
|
|
self.assertIn('Failed to claim PCI device', str(ex))
|
|
attached_ports = self._get_attached_port_ids(server['id'])
|
|
self.assertIn(first_port_id, attached_ports)
|
|
self.assertNotIn(second_port_id, attached_ports)
|
|
|
|
def test_detach_attach_direct(self):
|
|
self._test_detach_attach(
|
|
self.neutron.sriov_port['id'], self.neutron.sriov_port2['id'])
|
|
|
|
def test_detach_macvtap(self):
|
|
self._test_detach_attach(
|
|
self.neutron.macvtap_port['id'],
|
|
self.neutron.macvtap_port2['id'])
|
|
|
|
def test_detach_direct_physical(self):
|
|
self._test_detach_attach(
|
|
self.neutron.sriov_pf_port['id'],
|
|
self.neutron.sriov_pf_port2['id'])
|
|
|
|
|
|
class VDPAServersTest(_PCIServersWithMigrationTestBase):
|
|
|
|
# this is needed for os_compute_api:os-migrate-server:migrate policy
|
|
ADMIN_API = True
|
|
microversion = 'latest'
|
|
|
|
# Whitelist both the PF and VF; in reality, you probably wouldn't do this
|
|
# but we want to make sure that the PF is correctly taken off the table
|
|
# once any VF is used
|
|
PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in (
|
|
{
|
|
'vendor_id': '15b3',
|
|
'product_id': '101d',
|
|
'physical_network': 'physnet4',
|
|
},
|
|
{
|
|
'vendor_id': '15b3',
|
|
'product_id': '101e',
|
|
'physical_network': 'physnet4',
|
|
},
|
|
)]
|
|
# No need for aliases as these test will request SRIOV via neutron
|
|
PCI_ALIAS = []
|
|
|
|
NUM_PFS = 1
|
|
NUM_VFS = 4
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
# The ultimate base class _IntegratedTestBase uses NeutronFixture but
|
|
# we need a bit more intelligent neutron for these tests. Applying the
|
|
# new fixture here means that we re-stub what the previous neutron
|
|
# fixture already stubbed.
|
|
self.neutron = self.useFixture(base.LibvirtNeutronFixture(self))
|
|
|
|
def start_vdpa_compute(self, hostname='compute-0'):
|
|
vf_ratio = self.NUM_VFS // self.NUM_PFS
|
|
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(
|
|
num_pci=0, num_pfs=0, num_vfs=0)
|
|
vdpa_info = fakelibvirt.HostVDPADevicesInfo()
|
|
|
|
pci_info.add_device(
|
|
dev_type='PF',
|
|
bus=0x6,
|
|
slot=0x0,
|
|
function=0,
|
|
iommu_group=40, # totally arbitrary number
|
|
numa_node=0,
|
|
vf_ratio=vf_ratio,
|
|
vend_id='15b3',
|
|
vend_name='Mellanox Technologies',
|
|
prod_id='101d',
|
|
prod_name='MT2892 Family [ConnectX-6 Dx]',
|
|
driver_name='mlx5_core')
|
|
|
|
for idx in range(self.NUM_VFS):
|
|
vf = pci_info.add_device(
|
|
dev_type='VF',
|
|
bus=0x6,
|
|
slot=0x0,
|
|
function=idx + 1,
|
|
iommu_group=idx + 41, # totally arbitrary number + offset
|
|
numa_node=0,
|
|
vf_ratio=vf_ratio,
|
|
parent=(0x6, 0x0, 0),
|
|
vend_id='15b3',
|
|
vend_name='Mellanox Technologies',
|
|
prod_id='101e',
|
|
prod_name='ConnectX Family mlx5Gen Virtual Function',
|
|
driver_name='mlx5_core')
|
|
vdpa_info.add_device(f'vdpa_vdpa{idx}', idx, vf)
|
|
|
|
return super().start_compute(hostname=hostname,
|
|
pci_info=pci_info, vdpa_info=vdpa_info)
|
|
|
|
def create_vdpa_port(self):
|
|
vdpa_port = {
|
|
'id': uuids.vdpa_port,
|
|
'network_id': self.neutron.network_4['id'],
|
|
'status': 'ACTIVE',
|
|
'mac_address': 'b5:bc:2e:e7:51:ee',
|
|
'fixed_ips': [
|
|
{
|
|
'ip_address': '192.168.4.6',
|
|
'subnet_id': self.neutron.subnet_4['id']
|
|
}
|
|
],
|
|
'binding:vif_details': {},
|
|
'binding:vif_type': 'ovs',
|
|
'binding:vnic_type': 'vdpa',
|
|
}
|
|
|
|
# create the port
|
|
self.neutron.create_port({'port': vdpa_port})
|
|
return vdpa_port
|
|
|
|
def test_create_server(self):
|
|
"""Create an instance using a neutron-provisioned vDPA VIF."""
|
|
|
|
orig_create = nova.virt.libvirt.guest.Guest.create
|
|
|
|
def fake_create(cls, xml, host):
|
|
tree = etree.fromstring(xml)
|
|
elem = tree.find('./devices/interface/[@type="vdpa"]')
|
|
|
|
# compare source device
|
|
# the MAC address is derived from the neutron port, while the
|
|
# source dev path assumes we attach vDPA devs in order
|
|
expected = """
|
|
<interface type="vdpa">
|
|
<mac address="b5:bc:2e:e7:51:ee"/>
|
|
<source dev="/dev/vhost-vdpa-3"/>
|
|
</interface>"""
|
|
actual = etree.tostring(elem, encoding='unicode')
|
|
|
|
self.assertXmlEqual(expected, actual)
|
|
|
|
return orig_create(xml, host)
|
|
|
|
self.stub_out(
|
|
'nova.virt.libvirt.guest.Guest.create',
|
|
fake_create,
|
|
)
|
|
|
|
hostname = self.start_vdpa_compute()
|
|
num_pci = self.NUM_PFS + self.NUM_VFS
|
|
|
|
# both the PF and VF with vDPA capabilities (dev_type=vdpa) should have
|
|
# been counted
|
|
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci)
|
|
|
|
# create the port
|
|
vdpa_port = self.create_vdpa_port()
|
|
|
|
# ensure the binding details are currently unset
|
|
port = self.neutron.show_port(vdpa_port['id'])['port']
|
|
self.assertNotIn('binding:profile', port)
|
|
|
|
# create a server using the vDPA device via neutron
|
|
self._create_server(networks=[{'port': vdpa_port['id']}])
|
|
|
|
# ensure there is one less VF available and that the PF is no longer
|
|
# usable
|
|
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci - 2)
|
|
|
|
# ensure the binding details sent to "neutron" were correct
|
|
port = self.neutron.show_port(vdpa_port['id'])['port']
|
|
self.assertIn('binding:profile', port)
|
|
self.assertEqual(
|
|
{
|
|
'pci_vendor_info': '15b3:101e',
|
|
'pci_slot': '0000:06:00.4',
|
|
'physical_network': 'physnet4',
|
|
},
|
|
port['binding:profile'],
|
|
)
|
|
|
|
def _create_port_and_server(self):
|
|
# create the port and a server, with the port attached to the server
|
|
vdpa_port = self.create_vdpa_port()
|
|
server = self._create_server(networks=[{'port': vdpa_port['id']}])
|
|
return vdpa_port, server
|
|
|
|
def _test_common(self, op, *args, **kwargs):
|
|
self.start_vdpa_compute()
|
|
|
|
vdpa_port, server = self._create_port_and_server()
|
|
|
|
# attempt the unsupported action and ensure it fails
|
|
ex = self.assertRaises(
|
|
client.OpenStackApiException,
|
|
op, server, *args, **kwargs)
|
|
self.assertIn(
|
|
'not supported for instance with vDPA ports',
|
|
ex.response.text)
|
|
|
|
# NOTE(sbauza): Now we're post-Antelope release, we don't need to support
|
|
# this test
|
|
def test_attach_interface_service_version_61(self):
|
|
self.flags(disable_compute_service_check_for_ffu=True,
|
|
group='workarounds')
|
|
with mock.patch(
|
|
"nova.objects.service.get_minimum_version_all_cells",
|
|
return_value=61
|
|
):
|
|
self._test_common(self._attach_interface, uuids.vdpa_port)
|
|
|
|
def test_attach_interface(self):
|
|
hostname = self.start_vdpa_compute()
|
|
# create the port and a server, but don't attach the port to the server
|
|
# yet
|
|
server = self._create_server(networks='none')
|
|
vdpa_port = self.create_vdpa_port()
|
|
# attempt to attach the port to the server
|
|
self._attach_interface(server, vdpa_port['id'])
|
|
# ensure the binding details sent to "neutron" were correct
|
|
port = self.neutron.show_port(vdpa_port['id'])['port']
|
|
self.assertIn('binding:profile', port)
|
|
self.assertEqual(
|
|
{
|
|
'pci_vendor_info': '15b3:101e',
|
|
'pci_slot': '0000:06:00.4',
|
|
'physical_network': 'physnet4',
|
|
},
|
|
port['binding:profile'],
|
|
)
|
|
self.assertEqual(hostname, port['binding:host_id'])
|
|
self.assertEqual(server['id'], port['device_id'])
|
|
|
|
# NOTE(sbauza): Now we're post-Antelope release, we don't need to support
|
|
# this test
|
|
def test_detach_interface_service_version_61(self):
|
|
self.flags(disable_compute_service_check_for_ffu=True,
|
|
group='workarounds')
|
|
with mock.patch(
|
|
"nova.objects.service.get_minimum_version_all_cells",
|
|
return_value=61
|
|
):
|
|
self._test_common(self._detach_interface, uuids.vdpa_port)
|
|
|
|
def test_detach_interface(self):
|
|
self.start_vdpa_compute()
|
|
vdpa_port, server = self._create_port_and_server()
|
|
# ensure the binding details sent to "neutron" were correct
|
|
port = self.neutron.show_port(vdpa_port['id'])['port']
|
|
self.assertEqual(server['id'], port['device_id'])
|
|
self._detach_interface(server, vdpa_port['id'])
|
|
# ensure the port is no longer owned by the vm
|
|
port = self.neutron.show_port(vdpa_port['id'])['port']
|
|
self.assertEqual('', port['device_id'])
|
|
self.assertEqual({}, port['binding:profile'])
|
|
|
|
def test_shelve_offload(self):
|
|
hostname = self.start_vdpa_compute()
|
|
vdpa_port, server = self._create_port_and_server()
|
|
# assert the port is bound to the vm and the compute host
|
|
port = self.neutron.show_port(vdpa_port['id'])['port']
|
|
self.assertEqual(server['id'], port['device_id'])
|
|
self.assertEqual(hostname, port['binding:host_id'])
|
|
num_pci = self.NUM_PFS + self.NUM_VFS
|
|
# -2 we claim the vdpa device which make the parent PF unavailable
|
|
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci - 2)
|
|
server = self._shelve_server(server)
|
|
# now that the vm is shelve offloaded it should not be bound
|
|
# to any host but should still be owned by the vm
|
|
port = self.neutron.show_port(vdpa_port['id'])['port']
|
|
self.assertEqual(server['id'], port['device_id'])
|
|
self.assertIsNone(port['binding:host_id'])
|
|
self.assertIn('binding:profile', port)
|
|
self.assertIsNone(server['OS-EXT-SRV-ATTR:hypervisor_hostname'])
|
|
self.assertIsNone(server['OS-EXT-SRV-ATTR:host'])
|
|
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci)
|
|
|
|
def test_unshelve_to_same_host(self):
|
|
hostname = self.start_vdpa_compute()
|
|
num_pci = self.NUM_PFS + self.NUM_VFS
|
|
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci)
|
|
|
|
vdpa_port, server = self._create_port_and_server()
|
|
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci - 2)
|
|
self.assertEqual(
|
|
hostname, server['OS-EXT-SRV-ATTR:hypervisor_hostname'])
|
|
port = self.neutron.show_port(vdpa_port['id'])['port']
|
|
self.assertEqual(hostname, port['binding:host_id'])
|
|
|
|
server = self._shelve_server(server)
|
|
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci)
|
|
self.assertIsNone(server['OS-EXT-SRV-ATTR:hypervisor_hostname'])
|
|
port = self.neutron.show_port(vdpa_port['id'])['port']
|
|
self.assertIsNone(port['binding:host_id'])
|
|
|
|
server = self._unshelve_server(server)
|
|
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci - 2)
|
|
self.assertEqual(
|
|
hostname, server['OS-EXT-SRV-ATTR:hypervisor_hostname'])
|
|
port = self.neutron.show_port(vdpa_port['id'])['port']
|
|
self.assertEqual(hostname, port['binding:host_id'])
|
|
|
|
def test_unshelve_to_different_host(self):
|
|
source = self.start_vdpa_compute(hostname='source')
|
|
dest = self.start_vdpa_compute(hostname='dest')
|
|
|
|
num_pci = self.NUM_PFS + self.NUM_VFS
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci)
|
|
self.assertPCIDeviceCounts(dest, total=num_pci, free=num_pci)
|
|
|
|
# ensure we boot the vm on the "source" compute
|
|
self.api.put_service(
|
|
self.computes['dest'].service_ref.uuid, {'status': 'disabled'})
|
|
vdpa_port, server = self._create_port_and_server()
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci - 2)
|
|
self.assertEqual(
|
|
source, server['OS-EXT-SRV-ATTR:hypervisor_hostname'])
|
|
port = self.neutron.show_port(vdpa_port['id'])['port']
|
|
self.assertEqual(source, port['binding:host_id'])
|
|
|
|
server = self._shelve_server(server)
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci)
|
|
self.assertIsNone(server['OS-EXT-SRV-ATTR:hypervisor_hostname'])
|
|
port = self.neutron.show_port(vdpa_port['id'])['port']
|
|
self.assertIsNone(port['binding:host_id'])
|
|
|
|
# force the unshelve to the other host
|
|
self.api.put_service(
|
|
self.computes['source'].service_ref.uuid, {'status': 'disabled'})
|
|
self.api.put_service(
|
|
self.computes['dest'].service_ref.uuid, {'status': 'enabled'})
|
|
self.assertPCIDeviceCounts(dest, total=num_pci, free=num_pci)
|
|
server = self._unshelve_server(server)
|
|
# the dest devices should be claimed
|
|
self.assertPCIDeviceCounts(dest, total=num_pci, free=num_pci - 2)
|
|
# and the source host devices should still be free
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci)
|
|
self.assertEqual(
|
|
dest, server['OS-EXT-SRV-ATTR:hypervisor_hostname'])
|
|
port = self.neutron.show_port(vdpa_port['id'])['port']
|
|
self.assertEqual(dest, port['binding:host_id'])
|
|
|
|
def test_evacute(self):
|
|
source = self.start_vdpa_compute(hostname='source')
|
|
dest = self.start_vdpa_compute(hostname='dest')
|
|
|
|
num_pci = self.NUM_PFS + self.NUM_VFS
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci)
|
|
self.assertPCIDeviceCounts(dest, total=num_pci, free=num_pci)
|
|
|
|
# ensure we boot the vm on the "source" compute
|
|
self.api.put_service(
|
|
self.computes['dest'].service_ref.uuid, {'status': 'disabled'})
|
|
vdpa_port, server = self._create_port_and_server()
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci - 2)
|
|
self.assertEqual(
|
|
source, server['OS-EXT-SRV-ATTR:hypervisor_hostname'])
|
|
port = self.neutron.show_port(vdpa_port['id'])['port']
|
|
self.assertEqual(source, port['binding:host_id'])
|
|
|
|
# stop the source compute and enable the dest
|
|
self.api.put_service(
|
|
self.computes['dest'].service_ref.uuid, {'status': 'enabled'})
|
|
self.computes['source'].stop()
|
|
# Down the source compute to enable the evacuation
|
|
self.api.put_service(
|
|
self.computes['source'].service_ref.uuid, {'forced_down': True})
|
|
|
|
self.assertPCIDeviceCounts(dest, total=num_pci, free=num_pci)
|
|
server = self._evacuate_server(server)
|
|
self.assertPCIDeviceCounts(dest, total=num_pci, free=num_pci - 2)
|
|
self.assertEqual(
|
|
dest, server['OS-EXT-SRV-ATTR:hypervisor_hostname'])
|
|
port = self.neutron.show_port(vdpa_port['id'])['port']
|
|
self.assertEqual(dest, port['binding:host_id'])
|
|
|
|
# as the source compute is offline the pci claims will not be cleaned
|
|
# up on the source compute.
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci - 2)
|
|
# but if you fix/restart the source node the allocations for evacuated
|
|
# instances should be released.
|
|
self.restart_compute_service(source)
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci)
|
|
|
|
def test_resize_same_host(self):
|
|
self.flags(allow_resize_to_same_host=True)
|
|
num_pci = self.NUM_PFS + self.NUM_VFS
|
|
source = self.start_vdpa_compute()
|
|
vdpa_port, server = self._create_port_and_server()
|
|
# before we resize the vm should be using 1 VF but that will mark
|
|
# the PF as unavailable so we assert 2 devices are in use.
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci - 2)
|
|
flavor_id = self._create_flavor(name='new-flavor')
|
|
self.assertNotEqual(server['flavor']['original_name'], 'new-flavor')
|
|
with mock.patch(
|
|
'nova.virt.libvirt.driver.LibvirtDriver'
|
|
'.migrate_disk_and_power_off', return_value='{}',
|
|
):
|
|
server = self._resize_server(server, flavor_id)
|
|
self.assertEqual(
|
|
server['flavor']['original_name'], 'new-flavor')
|
|
# in resize verify the VF claims should be doubled even
|
|
# for same host resize so assert that 3 are in devices in use
|
|
# 1 PF and 2 VFs .
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci - 3)
|
|
server = self._confirm_resize(server)
|
|
# but once we confrim it should be reduced back to 1 PF and 1 VF
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci - 2)
|
|
# assert the hostname has not have changed as part
|
|
# of the resize.
|
|
self.assertEqual(
|
|
source, server['OS-EXT-SRV-ATTR:hypervisor_hostname'])
|
|
|
|
def test_resize_different_host(self):
|
|
self.flags(allow_resize_to_same_host=False)
|
|
source = self.start_vdpa_compute(hostname='source')
|
|
dest = self.start_vdpa_compute(hostname='dest')
|
|
|
|
num_pci = self.NUM_PFS + self.NUM_VFS
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci)
|
|
self.assertPCIDeviceCounts(dest, total=num_pci, free=num_pci)
|
|
|
|
# ensure we boot the vm on the "source" compute
|
|
self.api.put_service(
|
|
self.computes['dest'].service_ref.uuid, {'status': 'disabled'})
|
|
vdpa_port, server = self._create_port_and_server()
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci - 2)
|
|
flavor_id = self._create_flavor(name='new-flavor')
|
|
self.assertNotEqual(server['flavor']['original_name'], 'new-flavor')
|
|
# disable the source compute and enable the dest
|
|
self.api.put_service(
|
|
self.computes['source'].service_ref.uuid, {'status': 'disabled'})
|
|
self.api.put_service(
|
|
self.computes['dest'].service_ref.uuid, {'status': 'enabled'})
|
|
with mock.patch(
|
|
'nova.virt.libvirt.driver.LibvirtDriver'
|
|
'.migrate_disk_and_power_off', return_value='{}',
|
|
):
|
|
server = self._resize_server(server, flavor_id)
|
|
self.assertEqual(
|
|
server['flavor']['original_name'], 'new-flavor')
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci - 2)
|
|
self.assertPCIDeviceCounts(dest, total=num_pci, free=num_pci - 2)
|
|
server = self._confirm_resize(server)
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci)
|
|
self.assertPCIDeviceCounts(dest, total=num_pci, free=num_pci - 2)
|
|
self.assertEqual(
|
|
dest, server['OS-EXT-SRV-ATTR:hypervisor_hostname'])
|
|
|
|
def test_resize_revert(self):
|
|
self.flags(allow_resize_to_same_host=False)
|
|
source = self.start_vdpa_compute(hostname='source')
|
|
dest = self.start_vdpa_compute(hostname='dest')
|
|
|
|
num_pci = self.NUM_PFS + self.NUM_VFS
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci)
|
|
self.assertPCIDeviceCounts(dest, total=num_pci, free=num_pci)
|
|
|
|
# ensure we boot the vm on the "source" compute
|
|
self.api.put_service(
|
|
self.computes['dest'].service_ref.uuid, {'status': 'disabled'})
|
|
vdpa_port, server = self._create_port_and_server()
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci - 2)
|
|
flavor_id = self._create_flavor(name='new-flavor')
|
|
self.assertNotEqual(server['flavor']['original_name'], 'new-flavor')
|
|
# disable the source compute and enable the dest
|
|
self.api.put_service(
|
|
self.computes['source'].service_ref.uuid, {'status': 'disabled'})
|
|
self.api.put_service(
|
|
self.computes['dest'].service_ref.uuid, {'status': 'enabled'})
|
|
with mock.patch(
|
|
'nova.virt.libvirt.driver.LibvirtDriver'
|
|
'.migrate_disk_and_power_off', return_value='{}',
|
|
):
|
|
server = self._resize_server(server, flavor_id)
|
|
self.assertEqual(
|
|
server['flavor']['original_name'], 'new-flavor')
|
|
# in resize verify both the dest and source pci claims should be
|
|
# present.
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci - 2)
|
|
self.assertPCIDeviceCounts(dest, total=num_pci, free=num_pci - 2)
|
|
server = self._revert_resize(server)
|
|
# but once we revert the dest claims should be freed.
|
|
self.assertPCIDeviceCounts(dest, total=num_pci, free=num_pci)
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci - 2)
|
|
self.assertEqual(
|
|
source, server['OS-EXT-SRV-ATTR:hypervisor_hostname'])
|
|
|
|
def test_cold_migrate(self):
|
|
source = self.start_vdpa_compute(hostname='source')
|
|
dest = self.start_vdpa_compute(hostname='dest')
|
|
|
|
num_pci = self.NUM_PFS + self.NUM_VFS
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci)
|
|
self.assertPCIDeviceCounts(dest, total=num_pci, free=num_pci)
|
|
|
|
# ensure we boot the vm on the "source" compute
|
|
self.api.put_service(
|
|
self.computes['dest'].service_ref.uuid, {'status': 'disabled'})
|
|
vdpa_port, server = self._create_port_and_server()
|
|
self.assertEqual(
|
|
source, server['OS-EXT-SRV-ATTR:hypervisor_hostname'])
|
|
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci - 2)
|
|
# enable the dest we do not need to disable the source since cold
|
|
# migrate wont happen to the same host in the libvirt driver
|
|
self.api.put_service(
|
|
self.computes['dest'].service_ref.uuid, {'status': 'enabled'})
|
|
with mock.patch(
|
|
'nova.virt.libvirt.driver.LibvirtDriver'
|
|
'.migrate_disk_and_power_off', return_value='{}',
|
|
):
|
|
server = self._migrate_server(server)
|
|
self.assertEqual(
|
|
dest, server['OS-EXT-SRV-ATTR:hypervisor_hostname'])
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci - 2)
|
|
self.assertPCIDeviceCounts(dest, total=num_pci, free=num_pci - 2)
|
|
server = self._confirm_resize(server)
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci)
|
|
self.assertPCIDeviceCounts(dest, total=num_pci, free=num_pci - 2)
|
|
self.assertEqual(
|
|
dest, server['OS-EXT-SRV-ATTR:hypervisor_hostname'])
|
|
|
|
# NOTE(sbauza): Now we're post-Antelope release, we don't need to support
|
|
# this test
|
|
def test_suspend_and_resume_service_version_62(self):
|
|
self.flags(disable_compute_service_check_for_ffu=True,
|
|
group='workarounds')
|
|
with mock.patch(
|
|
"nova.objects.service.get_minimum_version_all_cells",
|
|
return_value=62
|
|
):
|
|
self._test_common(self._suspend_server)
|
|
|
|
def test_suspend_and_resume(self):
|
|
source = self.start_vdpa_compute(hostname='source')
|
|
vdpa_port, server = self._create_port_and_server()
|
|
num_pci = self.NUM_PFS + self.NUM_VFS
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci - 2)
|
|
server = self._suspend_server(server)
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci - 2)
|
|
self.assertEqual('SUSPENDED', server['status'])
|
|
server = self._resume_server(server)
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci - 2)
|
|
self.assertEqual('ACTIVE', server['status'])
|
|
|
|
# NOTE(sbauza): Now we're post-Antelope release, we don't need to support
|
|
# this test
|
|
def test_live_migrate_service_version_62(self):
|
|
self.flags(disable_compute_service_check_for_ffu=True,
|
|
group='workarounds')
|
|
with mock.patch(
|
|
"nova.objects.service.get_minimum_version_all_cells",
|
|
return_value=62
|
|
):
|
|
self._test_common(self._live_migrate)
|
|
|
|
def test_live_migrate(self):
|
|
source = self.start_vdpa_compute(hostname='source')
|
|
dest = self.start_vdpa_compute(hostname='dest')
|
|
|
|
num_pci = self.NUM_PFS + self.NUM_VFS
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci)
|
|
self.assertPCIDeviceCounts(dest, total=num_pci, free=num_pci)
|
|
|
|
# ensure we boot the vm on the "source" compute
|
|
self.api.put_service(
|
|
self.computes['dest'].service_ref.uuid, {'status': 'disabled'})
|
|
vdpa_port, server = self._create_port_and_server()
|
|
self.assertEqual(
|
|
source, server['OS-EXT-SRV-ATTR:hypervisor_hostname'])
|
|
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci - 2)
|
|
# enable the dest we do not need to disable the source since cold
|
|
# migrate wont happen to the same host in the libvirt driver
|
|
self.api.put_service(
|
|
self.computes['dest'].service_ref.uuid, {'status': 'enabled'})
|
|
|
|
with mock.patch(
|
|
'nova.virt.libvirt.LibvirtDriver.'
|
|
'_detach_direct_passthrough_vifs'
|
|
):
|
|
server = self._live_migrate(server)
|
|
self.assertPCIDeviceCounts(source, total=num_pci, free=num_pci)
|
|
self.assertPCIDeviceCounts(dest, total=num_pci, free=num_pci - 2)
|
|
self.assertEqual(
|
|
dest, server['OS-EXT-SRV-ATTR:hypervisor_hostname'])
|
|
|
|
|
|
class PCIServersTest(_PCIServersTestBase):
|
|
|
|
ADMIN_API = True
|
|
microversion = 'latest'
|
|
|
|
ALIAS_NAME = 'a1'
|
|
PCI_DEVICE_SPEC = [jsonutils.dumps(
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PCI_PROD_ID,
|
|
}
|
|
)]
|
|
PCI_ALIAS = [jsonutils.dumps(
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PCI_PROD_ID,
|
|
'name': ALIAS_NAME,
|
|
}
|
|
)]
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
self.flags(group="pci", report_in_placement=True)
|
|
self.flags(group='filter_scheduler', pci_in_placement=True)
|
|
|
|
def test_create_server_with_pci_dev_and_numa(self):
|
|
"""Verifies that an instance can be booted with cpu pinning and with an
|
|
assigned pci device with legacy policy and numa info for the pci
|
|
device.
|
|
"""
|
|
|
|
self.flags(cpu_dedicated_set='0-7', group='compute')
|
|
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(num_pci=1, numa_node=1)
|
|
self.start_compute(pci_info=pci_info)
|
|
|
|
self.assert_placement_pci_view(
|
|
"compute1",
|
|
inventories={"0000:81:00.0": {self.PCI_RC: 1}},
|
|
traits={"0000:81:00.0": []},
|
|
usages={"0000:81:00.0": {self.PCI_RC: 0}},
|
|
)
|
|
|
|
# create a flavor
|
|
extra_spec = {
|
|
'hw:cpu_policy': 'dedicated',
|
|
'pci_passthrough:alias': '%s:1' % self.ALIAS_NAME,
|
|
}
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
|
|
server = self._create_server(flavor_id=flavor_id, networks='none')
|
|
|
|
self.assert_placement_pci_view(
|
|
"compute1",
|
|
inventories={"0000:81:00.0": {self.PCI_RC: 1}},
|
|
traits={"0000:81:00.0": []},
|
|
usages={"0000:81:00.0": {self.PCI_RC: 1}},
|
|
allocations={server['id']: {"0000:81:00.0": {self.PCI_RC: 1}}},
|
|
)
|
|
self.assert_no_pci_healing("compute1")
|
|
|
|
def test_create_server_with_pci_dev_and_numa_fails(self):
|
|
"""This test ensures that it is not possible to allocated CPU and
|
|
memory resources from one NUMA node and a PCI device from another
|
|
if we use the legacy policy and the pci device reports numa info.
|
|
"""
|
|
self.flags(cpu_dedicated_set='0-7', group='compute')
|
|
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(num_pci=1, numa_node=0)
|
|
self.start_compute(pci_info=pci_info)
|
|
|
|
compute1_placement_pci_view = {
|
|
"inventories": {"0000:81:00.0": {self.PCI_RC: 1}},
|
|
"traits": {"0000:81:00.0": []},
|
|
"usages": {"0000:81:00.0": {self.PCI_RC: 0}},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"compute1", **compute1_placement_pci_view)
|
|
|
|
# boot one instance with no PCI device to "fill up" NUMA node 0
|
|
extra_spec = {'hw:cpu_policy': 'dedicated'}
|
|
flavor_id = self._create_flavor(vcpu=4, extra_spec=extra_spec)
|
|
self._create_server(flavor_id=flavor_id, networks='none')
|
|
|
|
# now boot one with a PCI device, which should fail to boot
|
|
extra_spec['pci_passthrough:alias'] = '%s:1' % self.ALIAS_NAME
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
self._create_server(
|
|
flavor_id=flavor_id, networks='none', expected_state='ERROR')
|
|
|
|
self.assert_placement_pci_view(
|
|
"compute1", **compute1_placement_pci_view)
|
|
self.assert_no_pci_healing("compute1")
|
|
|
|
def test_live_migrate_server_with_pci(self):
|
|
"""Live migrate an instance with a PCI passthrough device.
|
|
|
|
This should fail because it's not possible to live migrate an instance
|
|
with a PCI passthrough device, even if it's a SR-IOV VF.
|
|
"""
|
|
|
|
# start two compute services
|
|
self.start_compute(
|
|
hostname='test_compute0',
|
|
pci_info=fakelibvirt.HostPCIDevicesInfo(num_pci=1))
|
|
|
|
test_compute0_placement_pci_view = {
|
|
"inventories": {"0000:81:00.0": {self.PCI_RC: 1}},
|
|
"traits": {"0000:81:00.0": []},
|
|
"usages": {"0000:81:00.0": {self.PCI_RC: 0}},
|
|
"allocations": {},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
self.start_compute(
|
|
hostname='test_compute1',
|
|
pci_info=fakelibvirt.HostPCIDevicesInfo(num_pci=1))
|
|
|
|
test_compute1_placement_pci_view = {
|
|
"inventories": {"0000:81:00.0": {self.PCI_RC: 1}},
|
|
"traits": {"0000:81:00.0": []},
|
|
"usages": {"0000:81:00.0": {self.PCI_RC: 0}},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
# create a server
|
|
extra_spec = {'pci_passthrough:alias': f'{self.ALIAS_NAME}:1'}
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
server = self._create_server(
|
|
flavor_id=flavor_id, networks='none', host="test_compute0")
|
|
|
|
test_compute0_placement_pci_view[
|
|
"usages"]["0000:81:00.0"][self.PCI_RC] = 1
|
|
test_compute0_placement_pci_view[
|
|
"allocations"][server['id']] = {"0000:81:00.0": {self.PCI_RC: 1}}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
# now live migrate that server
|
|
ex = self.assertRaises(
|
|
client.OpenStackApiException,
|
|
self._live_migrate,
|
|
server, 'completed')
|
|
# NOTE(stephenfin): this wouldn't happen in a real deployment since
|
|
# live migration is a cast, but since we are using CastAsCallFixture
|
|
# this will bubble to the API
|
|
self.assertEqual(500, ex.response.status_code)
|
|
self.assertIn('NoValidHost', str(ex))
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
self.assert_no_pci_healing("test_compute0")
|
|
self.assert_no_pci_healing("test_compute1")
|
|
|
|
def test_resize_pci_to_vanilla(self):
|
|
# Start two computes, one with PCI and one without.
|
|
self.start_compute(
|
|
hostname='test_compute0',
|
|
pci_info=fakelibvirt.HostPCIDevicesInfo(num_pci=1))
|
|
test_compute0_placement_pci_view = {
|
|
"inventories": {"0000:81:00.0": {self.PCI_RC: 1}},
|
|
"traits": {"0000:81:00.0": []},
|
|
"usages": {"0000:81:00.0": {self.PCI_RC: 0}},
|
|
"allocations": {},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
self.start_compute(hostname='test_compute1')
|
|
test_compute1_placement_pci_view = {
|
|
"inventories": {},
|
|
"traits": {},
|
|
"usages": {},
|
|
"allocations": {},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
# Boot a server with a single PCI device.
|
|
extra_spec = {'pci_passthrough:alias': f'{self.ALIAS_NAME}:1'}
|
|
pci_flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
server = self._create_server(flavor_id=pci_flavor_id, networks='none')
|
|
|
|
test_compute0_placement_pci_view[
|
|
"usages"]["0000:81:00.0"][self.PCI_RC] = 1
|
|
test_compute0_placement_pci_view[
|
|
"allocations"][server['id']] = {"0000:81:00.0": {self.PCI_RC: 1}}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
# Resize it to a flavor without PCI devices. We expect this to work, as
|
|
# test_compute1 is available.
|
|
flavor_id = self._create_flavor()
|
|
with mock.patch(
|
|
'nova.virt.libvirt.driver.LibvirtDriver'
|
|
'.migrate_disk_and_power_off',
|
|
return_value='{}',
|
|
):
|
|
self._resize_server(server, flavor_id)
|
|
self._confirm_resize(server)
|
|
self.assertPCIDeviceCounts('test_compute0', total=1, free=1)
|
|
self.assertPCIDeviceCounts('test_compute1', total=0, free=0)
|
|
test_compute0_placement_pci_view[
|
|
"usages"]["0000:81:00.0"][self.PCI_RC] = 0
|
|
del test_compute0_placement_pci_view["allocations"][server['id']]
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
self.assert_no_pci_healing("test_compute0")
|
|
self.assert_no_pci_healing("test_compute1")
|
|
|
|
def test_resize_vanilla_to_pci(self):
|
|
"""Resize an instance from a non PCI flavor to a PCI flavor"""
|
|
# Start two computes, one with PCI and one without.
|
|
self.start_compute(
|
|
hostname='test_compute0',
|
|
pci_info=fakelibvirt.HostPCIDevicesInfo(num_pci=1))
|
|
test_compute0_placement_pci_view = {
|
|
"inventories": {"0000:81:00.0": {self.PCI_RC: 1}},
|
|
"traits": {"0000:81:00.0": []},
|
|
"usages": {"0000:81:00.0": {self.PCI_RC: 0}},
|
|
"allocations": {},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
self.start_compute(hostname='test_compute1')
|
|
test_compute1_placement_pci_view = {
|
|
"inventories": {},
|
|
"traits": {},
|
|
"usages": {},
|
|
"allocations": {},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
# Boot a server without PCI device and make sure it lands on the
|
|
# compute that has no device, so we can resize it later to the other
|
|
# host having PCI device.
|
|
extra_spec = {}
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
server = self._create_server(
|
|
flavor_id=flavor_id, networks='none', host="test_compute1")
|
|
|
|
self.assertPCIDeviceCounts('test_compute0', total=1, free=1)
|
|
self.assertPCIDeviceCounts('test_compute1', total=0, free=0)
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
# Resize it to a flavor with a PCI devices. We expect this to work, as
|
|
# test_compute0 is available and having PCI devices.
|
|
extra_spec = {'pci_passthrough:alias': f'{self.ALIAS_NAME}:1'}
|
|
pci_flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
with mock.patch(
|
|
'nova.virt.libvirt.driver.LibvirtDriver'
|
|
'.migrate_disk_and_power_off',
|
|
return_value='{}',
|
|
):
|
|
self._resize_server(server, pci_flavor_id)
|
|
self._confirm_resize(server)
|
|
self.assertPCIDeviceCounts('test_compute0', total=1, free=0)
|
|
self.assertPCIDeviceCounts('test_compute1', total=0, free=0)
|
|
test_compute0_placement_pci_view[
|
|
"usages"]["0000:81:00.0"][self.PCI_RC] = 1
|
|
test_compute0_placement_pci_view[
|
|
"allocations"][server['id']] = {"0000:81:00.0": {self.PCI_RC: 1}}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
self.assert_no_pci_healing("test_compute0")
|
|
self.assert_no_pci_healing("test_compute1")
|
|
|
|
def test_resize_from_one_dev_to_two(self):
|
|
self.start_compute(
|
|
hostname='test_compute0',
|
|
pci_info=fakelibvirt.HostPCIDevicesInfo(num_pci=1))
|
|
self.assertPCIDeviceCounts('test_compute0', total=1, free=1)
|
|
test_compute0_placement_pci_view = {
|
|
"inventories": {"0000:81:00.0": {self.PCI_RC: 1}},
|
|
"traits": {"0000:81:00.0": []},
|
|
"usages": {"0000:81:00.0": {self.PCI_RC: 0}},
|
|
"allocations": {},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
self.start_compute(
|
|
hostname='test_compute1',
|
|
pci_info=fakelibvirt.HostPCIDevicesInfo(num_pci=2),
|
|
)
|
|
self.assertPCIDeviceCounts('test_compute1', total=2, free=2)
|
|
test_compute1_placement_pci_view = {
|
|
"inventories": {
|
|
"0000:81:00.0": {self.PCI_RC: 1},
|
|
"0000:81:01.0": {self.PCI_RC: 1},
|
|
},
|
|
"traits": {
|
|
"0000:81:00.0": [],
|
|
"0000:81:01.0": [],
|
|
},
|
|
"usages": {
|
|
"0000:81:00.0": {self.PCI_RC: 0},
|
|
"0000:81:01.0": {self.PCI_RC: 0},
|
|
},
|
|
"allocations": {},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
# boot a VM on test_compute0 with a single PCI dev
|
|
extra_spec = {'pci_passthrough:alias': f'{self.ALIAS_NAME}:1'}
|
|
pci_flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
server = self._create_server(
|
|
flavor_id=pci_flavor_id, networks='none', host="test_compute0")
|
|
|
|
self.assertPCIDeviceCounts('test_compute0', total=1, free=0)
|
|
test_compute0_placement_pci_view["usages"][
|
|
"0000:81:00.0"][self.PCI_RC] = 1
|
|
test_compute0_placement_pci_view["allocations"][
|
|
server['id']] = {"0000:81:00.0": {self.PCI_RC: 1}}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
# resize the server to a flavor requesting two devices
|
|
extra_spec = {'pci_passthrough:alias': f'{self.ALIAS_NAME}:2'}
|
|
pci_flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
with mock.patch(
|
|
'nova.virt.libvirt.driver.LibvirtDriver'
|
|
'.migrate_disk_and_power_off',
|
|
return_value='{}',
|
|
):
|
|
self._resize_server(server, pci_flavor_id)
|
|
|
|
self.assertPCIDeviceCounts('test_compute0', total=1, free=0)
|
|
# one the source host the PCI allocation is now held by the migration
|
|
self._move_server_allocation(
|
|
test_compute0_placement_pci_view['allocations'], server['id'])
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
# on the dest we have now two device allocated
|
|
self.assertPCIDeviceCounts('test_compute1', total=2, free=0)
|
|
test_compute1_placement_pci_view["usages"] = {
|
|
"0000:81:00.0": {self.PCI_RC: 1},
|
|
"0000:81:01.0": {self.PCI_RC: 1},
|
|
}
|
|
test_compute1_placement_pci_view["allocations"][
|
|
server['id']] = {
|
|
"0000:81:00.0": {self.PCI_RC: 1},
|
|
"0000:81:01.0": {self.PCI_RC: 1},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
# now revert the resize
|
|
self._revert_resize(server)
|
|
|
|
self.assertPCIDeviceCounts('test_compute0', total=1, free=0)
|
|
# on the host the allocation should move back to the instance UUID
|
|
self._move_server_allocation(
|
|
test_compute0_placement_pci_view["allocations"],
|
|
server["id"],
|
|
revert=True,
|
|
)
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
# so the dest should be freed
|
|
self.assertPCIDeviceCounts('test_compute1', total=2, free=2)
|
|
test_compute1_placement_pci_view["usages"] = {
|
|
"0000:81:00.0": {self.PCI_RC: 0},
|
|
"0000:81:01.0": {self.PCI_RC: 0},
|
|
}
|
|
del test_compute1_placement_pci_view["allocations"][server['id']]
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
# now resize again and confirm it
|
|
with mock.patch(
|
|
'nova.virt.libvirt.driver.LibvirtDriver'
|
|
'.migrate_disk_and_power_off',
|
|
return_value='{}',
|
|
):
|
|
self._resize_server(server, pci_flavor_id)
|
|
self._confirm_resize(server)
|
|
|
|
# the source host now need to be freed up
|
|
self.assertPCIDeviceCounts('test_compute0', total=1, free=1)
|
|
test_compute0_placement_pci_view["usages"] = {
|
|
"0000:81:00.0": {self.PCI_RC: 0},
|
|
}
|
|
test_compute0_placement_pci_view["allocations"] = {}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
# and dest allocated
|
|
self.assertPCIDeviceCounts('test_compute1', total=2, free=0)
|
|
test_compute1_placement_pci_view["usages"] = {
|
|
"0000:81:00.0": {self.PCI_RC: 1},
|
|
"0000:81:01.0": {self.PCI_RC: 1},
|
|
}
|
|
test_compute1_placement_pci_view["allocations"][
|
|
server['id']] = {
|
|
"0000:81:00.0": {self.PCI_RC: 1},
|
|
"0000:81:01.0": {self.PCI_RC: 1},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
self.assert_no_pci_healing("test_compute0")
|
|
self.assert_no_pci_healing("test_compute1")
|
|
|
|
def test_same_host_resize_with_pci(self):
|
|
"""Start a single compute with 3 PCI devs and resize and instance
|
|
from one dev to two devs
|
|
"""
|
|
self.flags(allow_resize_to_same_host=True)
|
|
self.start_compute(
|
|
hostname='test_compute0',
|
|
pci_info=fakelibvirt.HostPCIDevicesInfo(num_pci=3))
|
|
self.assertPCIDeviceCounts('test_compute0', total=3, free=3)
|
|
test_compute0_placement_pci_view = {
|
|
"inventories": {
|
|
"0000:81:00.0": {self.PCI_RC: 1},
|
|
"0000:81:01.0": {self.PCI_RC: 1},
|
|
"0000:81:02.0": {self.PCI_RC: 1},
|
|
},
|
|
"traits": {
|
|
"0000:81:00.0": [],
|
|
"0000:81:01.0": [],
|
|
"0000:81:02.0": [],
|
|
},
|
|
"usages": {
|
|
"0000:81:00.0": {self.PCI_RC: 0},
|
|
"0000:81:01.0": {self.PCI_RC: 0},
|
|
"0000:81:02.0": {self.PCI_RC: 0},
|
|
},
|
|
"allocations": {},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
# Boot a server with a single PCI device.
|
|
# To stabilize the test we reserve 81.01 and 81.02 in placement so
|
|
# we can be sure that the instance will use 81.00, otherwise the
|
|
# allocation will be random between 00, 01, and 02
|
|
self._reserve_placement_resource(
|
|
"test_compute0_0000:81:01.0", self.PCI_RC, 1)
|
|
self._reserve_placement_resource(
|
|
"test_compute0_0000:81:02.0", self.PCI_RC, 1)
|
|
extra_spec = {'pci_passthrough:alias': f'{self.ALIAS_NAME}:1'}
|
|
pci_flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
server = self._create_server(flavor_id=pci_flavor_id, networks='none')
|
|
|
|
self.assertPCIDeviceCounts('test_compute0', total=3, free=2)
|
|
test_compute0_placement_pci_view[
|
|
"usages"]["0000:81:00.0"][self.PCI_RC] = 1
|
|
test_compute0_placement_pci_view[
|
|
"allocations"][server['id']] = {"0000:81:00.0": {self.PCI_RC: 1}}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
# remove the reservations, so we can resize on the same host and
|
|
# consume 01 and 02
|
|
self._reserve_placement_resource(
|
|
"test_compute0_0000:81:01.0", self.PCI_RC, 0)
|
|
self._reserve_placement_resource(
|
|
"test_compute0_0000:81:02.0", self.PCI_RC, 0)
|
|
|
|
# Resize the server to use 2 PCI devices
|
|
extra_spec = {'pci_passthrough:alias': f'{self.ALIAS_NAME}:2'}
|
|
pci_flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
with mock.patch(
|
|
'nova.virt.libvirt.driver.LibvirtDriver'
|
|
'.migrate_disk_and_power_off',
|
|
return_value='{}',
|
|
):
|
|
self._resize_server(server, pci_flavor_id)
|
|
|
|
self.assertPCIDeviceCounts('test_compute0', total=3, free=0)
|
|
# the source host side of the allocation is now held by the migration
|
|
# UUID
|
|
self._move_server_allocation(
|
|
test_compute0_placement_pci_view["allocations"], server['id'])
|
|
# but we have the dest host side of the allocations on the same host
|
|
test_compute0_placement_pci_view[
|
|
"usages"]["0000:81:01.0"][self.PCI_RC] = 1
|
|
test_compute0_placement_pci_view[
|
|
"usages"]["0000:81:02.0"][self.PCI_RC] = 1
|
|
test_compute0_placement_pci_view["allocations"][server['id']] = {
|
|
"0000:81:01.0": {self.PCI_RC: 1},
|
|
"0000:81:02.0": {self.PCI_RC: 1},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
# revert the resize so the instance should go back to use a single
|
|
# device
|
|
self._revert_resize(server)
|
|
self.assertPCIDeviceCounts('test_compute0', total=3, free=2)
|
|
# the migration allocation is moved back to the instance UUID
|
|
self._move_server_allocation(
|
|
test_compute0_placement_pci_view["allocations"],
|
|
server["id"],
|
|
revert=True,
|
|
)
|
|
# and the "dest" side of the allocation is dropped
|
|
test_compute0_placement_pci_view[
|
|
"usages"]["0000:81:01.0"][self.PCI_RC] = 0
|
|
test_compute0_placement_pci_view[
|
|
"usages"]["0000:81:02.0"][self.PCI_RC] = 0
|
|
test_compute0_placement_pci_view["allocations"][server['id']] = {
|
|
"0000:81:00.0": {self.PCI_RC: 1},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
# resize again but now confirm the same host resize and assert that
|
|
# only the new flavor usage remains
|
|
with mock.patch(
|
|
'nova.virt.libvirt.driver.LibvirtDriver'
|
|
'.migrate_disk_and_power_off',
|
|
return_value='{}',
|
|
):
|
|
self._resize_server(server, pci_flavor_id)
|
|
self._confirm_resize(server)
|
|
|
|
self.assertPCIDeviceCounts('test_compute0', total=3, free=1)
|
|
test_compute0_placement_pci_view["usages"] = {
|
|
"0000:81:01.0": {self.PCI_RC: 1},
|
|
"0000:81:02.0": {self.PCI_RC: 1},
|
|
}
|
|
test_compute0_placement_pci_view["allocations"][
|
|
server['id']] = {self.PCI_RC: 1}
|
|
test_compute0_placement_pci_view["allocations"][server['id']] = {
|
|
"0000:81:01.0": {self.PCI_RC: 1},
|
|
"0000:81:02.0": {self.PCI_RC: 1},
|
|
}
|
|
self.assert_no_pci_healing("test_compute0")
|
|
|
|
def _confirm_resize(self, server, host='host1'):
|
|
# NOTE(sbauza): Unfortunately, _cleanup_resize() in libvirt checks the
|
|
# host option to know the source hostname but given we have a global
|
|
# CONF, the value will be the hostname of the last compute service that
|
|
# was created, so we need to change it here.
|
|
# TODO(sbauza): Remove the below once we stop using CONF.host in
|
|
# libvirt and rather looking at the compute host value.
|
|
orig_host = CONF.host
|
|
self.flags(host=host)
|
|
super()._confirm_resize(server)
|
|
self.flags(host=orig_host)
|
|
|
|
def test_cold_migrate_server_with_pci(self):
|
|
host_devices = {}
|
|
orig_create = nova.virt.libvirt.guest.Guest.create
|
|
|
|
def fake_create(cls, xml, host):
|
|
tree = etree.fromstring(xml)
|
|
elem = tree.find('./devices/hostdev/source/address')
|
|
|
|
hostname = host.get_hostname()
|
|
address = (
|
|
elem.get('bus'), elem.get('slot'), elem.get('function'),
|
|
)
|
|
if hostname in host_devices:
|
|
self.assertNotIn(address, host_devices[hostname])
|
|
else:
|
|
host_devices[hostname] = []
|
|
host_devices[host.get_hostname()].append(address)
|
|
|
|
return orig_create(xml, host)
|
|
|
|
self.stub_out(
|
|
'nova.virt.libvirt.guest.Guest.create',
|
|
fake_create,
|
|
)
|
|
|
|
# start two compute services
|
|
for hostname in ('test_compute0', 'test_compute1'):
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(num_pci=2)
|
|
self.start_compute(hostname=hostname, pci_info=pci_info)
|
|
test_compute0_placement_pci_view = {
|
|
"inventories": {
|
|
"0000:81:00.0": {self.PCI_RC: 1},
|
|
"0000:81:01.0": {self.PCI_RC: 1},
|
|
},
|
|
"traits": {
|
|
"0000:81:00.0": [],
|
|
"0000:81:01.0": [],
|
|
},
|
|
"usages": {
|
|
"0000:81:00.0": {self.PCI_RC: 0},
|
|
"0000:81:01.0": {self.PCI_RC: 0},
|
|
},
|
|
"allocations": {},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
test_compute1_placement_pci_view = {
|
|
"inventories": {
|
|
"0000:81:00.0": {self.PCI_RC: 1},
|
|
"0000:81:01.0": {self.PCI_RC: 1},
|
|
},
|
|
"traits": {
|
|
"0000:81:00.0": [],
|
|
"0000:81:01.0": [],
|
|
},
|
|
"usages": {
|
|
"0000:81:00.0": {self.PCI_RC: 0},
|
|
"0000:81:01.0": {self.PCI_RC: 0},
|
|
},
|
|
"allocations": {},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
# boot an instance with a PCI device on each host
|
|
extra_spec = {
|
|
'pci_passthrough:alias': '%s:1' % self.ALIAS_NAME,
|
|
}
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
|
|
# force the allocation on test_compute0 to 81:00 to make it easy
|
|
# to assert the placement allocation
|
|
self._reserve_placement_resource(
|
|
"test_compute0_0000:81:01.0", self.PCI_RC, 1)
|
|
server_a = self._create_server(
|
|
flavor_id=flavor_id, networks='none', host='test_compute0')
|
|
# force the allocation on test_compute1 to 81:00 to make it easy
|
|
# to assert the placement allocation
|
|
self._reserve_placement_resource(
|
|
"test_compute1_0000:81:01.0", self.PCI_RC, 1)
|
|
server_b = self._create_server(
|
|
flavor_id=flavor_id, networks='none', host='test_compute1')
|
|
|
|
# the instances should have landed on separate hosts; ensure both hosts
|
|
# have one used PCI device and one free PCI device
|
|
self.assertNotEqual(
|
|
server_a['OS-EXT-SRV-ATTR:host'], server_b['OS-EXT-SRV-ATTR:host'],
|
|
)
|
|
for hostname in ('test_compute0', 'test_compute1'):
|
|
self.assertPCIDeviceCounts(hostname, total=2, free=1)
|
|
|
|
test_compute0_placement_pci_view["usages"][
|
|
"0000:81:00.0"][self.PCI_RC] = 1
|
|
test_compute0_placement_pci_view["allocations"][
|
|
server_a['id']] = {"0000:81:00.0": {self.PCI_RC: 1}}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
test_compute1_placement_pci_view[
|
|
"usages"]["0000:81:00.0"][self.PCI_RC] = 1
|
|
test_compute1_placement_pci_view["allocations"][
|
|
server_b['id']] = {"0000:81:00.0": {self.PCI_RC: 1}}
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
# remove the resource reservation from test_compute1 to be able to
|
|
# migrate server_a there
|
|
self._reserve_placement_resource(
|
|
"test_compute1_0000:81:01.0", self.PCI_RC, 0)
|
|
|
|
# TODO(stephenfin): The mock of 'migrate_disk_and_power_off' should
|
|
# probably be less...dumb
|
|
with mock.patch(
|
|
'nova.virt.libvirt.driver.LibvirtDriver'
|
|
'.migrate_disk_and_power_off', return_value='{}',
|
|
):
|
|
# TODO(stephenfin): Use a helper
|
|
self.api.post_server_action(server_a['id'], {'migrate': None})
|
|
server_a = self._wait_for_state_change(server_a, 'VERIFY_RESIZE')
|
|
|
|
# the instances should now be on the same host; ensure the source host
|
|
# still has one used PCI device while the destination now has two used
|
|
# test_compute0 initially
|
|
self.assertEqual(
|
|
server_a['OS-EXT-SRV-ATTR:host'], server_b['OS-EXT-SRV-ATTR:host'],
|
|
)
|
|
self.assertPCIDeviceCounts('test_compute0', total=2, free=1)
|
|
# on the source host the allocation is now held by the migration UUID
|
|
self._move_server_allocation(
|
|
test_compute0_placement_pci_view["allocations"], server_a['id'])
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
self.assertPCIDeviceCounts('test_compute1', total=2, free=0)
|
|
# sever_a now have allocation on test_compute1 on 81:01
|
|
test_compute1_placement_pci_view["usages"][
|
|
"0000:81:01.0"][self.PCI_RC] = 1
|
|
test_compute1_placement_pci_view["allocations"][
|
|
server_a['id']] = {"0000:81:01.0": {self.PCI_RC: 1}}
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
# now, confirm the migration and check our counts once again
|
|
self._confirm_resize(server_a)
|
|
|
|
self.assertPCIDeviceCounts('test_compute0', total=2, free=2)
|
|
# the source host now has no allocations as the migration allocation
|
|
# is removed by confirm resize
|
|
test_compute0_placement_pci_view["usages"] = {
|
|
"0000:81:00.0": {self.PCI_RC: 0},
|
|
"0000:81:01.0": {self.PCI_RC: 0},
|
|
}
|
|
test_compute0_placement_pci_view["allocations"] = {}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
self.assertPCIDeviceCounts('test_compute1', total=2, free=0)
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
self.assert_no_pci_healing("test_compute0")
|
|
self.assert_no_pci_healing("test_compute1")
|
|
|
|
def test_request_two_pci_but_host_has_one(self):
|
|
# simulate a single type-PCI device on the host
|
|
self.start_compute(pci_info=fakelibvirt.HostPCIDevicesInfo(num_pci=1))
|
|
self.assertPCIDeviceCounts('compute1', total=1, free=1)
|
|
|
|
alias = [jsonutils.dumps(x) for x in (
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PCI_PROD_ID,
|
|
'name': 'a1',
|
|
},
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PCI_PROD_ID,
|
|
'name': 'a2',
|
|
},
|
|
)]
|
|
self.flags(group='pci', alias=alias)
|
|
# request two PCI devices both are individually matching with the
|
|
# single available device on the host
|
|
extra_spec = {'pci_passthrough:alias': 'a1:1,a2:1'}
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
# so we expect that the boot fails with no valid host error as only
|
|
# one of the requested PCI device can be allocated
|
|
server = self._create_server(
|
|
flavor_id=flavor_id, networks="none", expected_state='ERROR')
|
|
self.assertIn('fault', server)
|
|
self.assertIn('No valid host', server['fault']['message'])
|
|
|
|
def _create_two_computes(self):
|
|
self.start_compute(
|
|
hostname='test_compute0',
|
|
pci_info=fakelibvirt.HostPCIDevicesInfo(num_pci=1))
|
|
self.assertPCIDeviceCounts('test_compute0', total=1, free=1)
|
|
test_compute0_placement_pci_view = {
|
|
"inventories": {"0000:81:00.0": {self.PCI_RC: 1}},
|
|
"traits": {"0000:81:00.0": []},
|
|
"usages": {"0000:81:00.0": {self.PCI_RC: 0}},
|
|
"allocations": {},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
self.start_compute(
|
|
hostname='test_compute1',
|
|
pci_info=fakelibvirt.HostPCIDevicesInfo(num_pci=1),
|
|
)
|
|
self.assertPCIDeviceCounts('test_compute1', total=1, free=1)
|
|
test_compute1_placement_pci_view = {
|
|
"inventories": {"0000:81:00.0": {self.PCI_RC: 1}},
|
|
"traits": {"0000:81:00.0": []},
|
|
"usages": {"0000:81:00.0": {self.PCI_RC: 0}},
|
|
"allocations": {},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
return (
|
|
test_compute0_placement_pci_view,
|
|
test_compute1_placement_pci_view,
|
|
)
|
|
|
|
def _create_two_computes_and_an_instance_on_the_first(self):
|
|
(
|
|
test_compute0_placement_pci_view,
|
|
test_compute1_placement_pci_view,
|
|
) = self._create_two_computes()
|
|
|
|
# boot a VM on test_compute0 with a single PCI dev
|
|
extra_spec = {'pci_passthrough:alias': f'{self.ALIAS_NAME}:1'}
|
|
pci_flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
server = self._create_server(
|
|
flavor_id=pci_flavor_id, networks='none', host="test_compute0")
|
|
|
|
self.assertPCIDeviceCounts('test_compute0', total=1, free=0)
|
|
test_compute0_placement_pci_view["usages"][
|
|
"0000:81:00.0"][self.PCI_RC] = 1
|
|
test_compute0_placement_pci_view["allocations"][
|
|
server['id']] = {"0000:81:00.0": {self.PCI_RC: 1}}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
return (
|
|
server,
|
|
test_compute0_placement_pci_view,
|
|
test_compute1_placement_pci_view,
|
|
)
|
|
|
|
def test_evacuate(self):
|
|
(
|
|
server,
|
|
test_compute0_placement_pci_view,
|
|
test_compute1_placement_pci_view,
|
|
) = self._create_two_computes_and_an_instance_on_the_first()
|
|
|
|
# kill test_compute0 and evacuate the instance
|
|
self.computes['test_compute0'].stop()
|
|
self.api.put_service(
|
|
self.computes["test_compute0"].service_ref.uuid,
|
|
{"forced_down": True},
|
|
)
|
|
self._evacuate_server(server)
|
|
# source allocation should be kept as source is dead but the server
|
|
# now has allocation on both hosts as evacuation does not use migration
|
|
# allocations.
|
|
self.assertPCIDeviceCounts('test_compute0', total=1, free=0)
|
|
self.assert_placement_pci_inventory(
|
|
"test_compute0",
|
|
test_compute0_placement_pci_view["inventories"],
|
|
test_compute0_placement_pci_view["traits"]
|
|
)
|
|
self.assert_placement_pci_usages(
|
|
"test_compute0", test_compute0_placement_pci_view["usages"]
|
|
)
|
|
self.assert_placement_pci_allocations(
|
|
{
|
|
server['id']: {
|
|
"test_compute0": {
|
|
"VCPU": 2,
|
|
"MEMORY_MB": 2048,
|
|
"DISK_GB": 20,
|
|
},
|
|
"test_compute0_0000:81:00.0": {self.PCI_RC: 1},
|
|
"test_compute1": {
|
|
"VCPU": 2,
|
|
"MEMORY_MB": 2048,
|
|
"DISK_GB": 20,
|
|
},
|
|
"test_compute1_0000:81:00.0": {self.PCI_RC: 1},
|
|
},
|
|
}
|
|
)
|
|
|
|
# dest allocation should be created
|
|
self.assertPCIDeviceCounts('test_compute1', total=1, free=0)
|
|
test_compute1_placement_pci_view["usages"][
|
|
"0000:81:00.0"][self.PCI_RC] = 1
|
|
test_compute1_placement_pci_view["allocations"][
|
|
server['id']] = {"0000:81:00.0": {self.PCI_RC: 1}}
|
|
self.assert_placement_pci_inventory(
|
|
"test_compute1",
|
|
test_compute1_placement_pci_view["inventories"],
|
|
test_compute1_placement_pci_view["traits"]
|
|
)
|
|
self.assert_placement_pci_usages(
|
|
"test_compute1", test_compute0_placement_pci_view["usages"]
|
|
)
|
|
|
|
# recover test_compute0 and check that it is cleaned
|
|
self.restart_compute_service('test_compute0')
|
|
|
|
self.assertPCIDeviceCounts('test_compute0', total=1, free=1)
|
|
test_compute0_placement_pci_view = {
|
|
"inventories": {"0000:81:00.0": {self.PCI_RC: 1}},
|
|
"traits": {"0000:81:00.0": []},
|
|
"usages": {"0000:81:00.0": {self.PCI_RC: 0}},
|
|
"allocations": {},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
# and test_compute1 is not changes (expect that the instance now has
|
|
# only allocation on this compute)
|
|
self.assertPCIDeviceCounts('test_compute1', total=1, free=0)
|
|
test_compute1_placement_pci_view["usages"][
|
|
"0000:81:00.0"][self.PCI_RC] = 1
|
|
test_compute1_placement_pci_view["allocations"][
|
|
server['id']] = {"0000:81:00.0": {self.PCI_RC: 1}}
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
self.assert_no_pci_healing("test_compute0")
|
|
self.assert_no_pci_healing("test_compute1")
|
|
|
|
def test_unshelve_after_offload(self):
|
|
(
|
|
server,
|
|
test_compute0_placement_pci_view,
|
|
test_compute1_placement_pci_view,
|
|
) = self._create_two_computes_and_an_instance_on_the_first()
|
|
|
|
# shelve offload the server
|
|
self._shelve_server(server)
|
|
|
|
# source allocation should be freed
|
|
self.assertPCIDeviceCounts('test_compute0', total=1, free=1)
|
|
test_compute0_placement_pci_view["usages"][
|
|
"0000:81:00.0"][self.PCI_RC] = 0
|
|
del test_compute0_placement_pci_view["allocations"][server['id']]
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
# test_compute1 should not be touched
|
|
self.assertPCIDeviceCounts('test_compute1', total=1, free=1)
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
# disable test_compute0 and unshelve the instance
|
|
self.api.put_service(
|
|
self.computes["test_compute0"].service_ref.uuid,
|
|
{"status": "disabled"},
|
|
)
|
|
self._unshelve_server(server)
|
|
|
|
# test_compute0 should be unchanged
|
|
self.assertPCIDeviceCounts('test_compute0', total=1, free=1)
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
# test_compute1 should be allocated
|
|
self.assertPCIDeviceCounts('test_compute1', total=1, free=0)
|
|
test_compute1_placement_pci_view["usages"][
|
|
"0000:81:00.0"][self.PCI_RC] = 1
|
|
test_compute1_placement_pci_view["allocations"][
|
|
server['id']] = {"0000:81:00.0": {self.PCI_RC: 1}}
|
|
self.assert_placement_pci_view(
|
|
"test_compute1", **test_compute1_placement_pci_view)
|
|
|
|
self.assert_no_pci_healing("test_compute0")
|
|
self.assert_no_pci_healing("test_compute1")
|
|
|
|
def test_reschedule(self):
|
|
(
|
|
test_compute0_placement_pci_view,
|
|
test_compute1_placement_pci_view,
|
|
) = self._create_two_computes()
|
|
|
|
# try to boot a VM with a single device but inject fault on the first
|
|
# compute so that the VM is re-scheduled to the other
|
|
extra_spec = {'pci_passthrough:alias': f'{self.ALIAS_NAME}:1'}
|
|
pci_flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
|
|
calls = []
|
|
orig_guest_create = (
|
|
nova.virt.libvirt.driver.LibvirtDriver._create_guest)
|
|
|
|
def fake_guest_create(*args, **kwargs):
|
|
if not calls:
|
|
calls.append(1)
|
|
raise fakelibvirt.make_libvirtError(
|
|
fakelibvirt.libvirtError,
|
|
"internal error",
|
|
error_code=fakelibvirt.VIR_ERR_INTERNAL_ERROR,
|
|
)
|
|
else:
|
|
return orig_guest_create(*args, **kwargs)
|
|
|
|
with mock.patch(
|
|
'nova.virt.libvirt.driver.LibvirtDriver._create_guest',
|
|
new=fake_guest_create
|
|
):
|
|
server = self._create_server(
|
|
flavor_id=pci_flavor_id, networks='none')
|
|
|
|
compute_pci_view_map = {
|
|
'test_compute0': test_compute0_placement_pci_view,
|
|
'test_compute1': test_compute1_placement_pci_view,
|
|
}
|
|
allocated_compute = server['OS-EXT-SRV-ATTR:host']
|
|
not_allocated_compute = (
|
|
"test_compute0"
|
|
if allocated_compute == "test_compute1"
|
|
else "test_compute1"
|
|
)
|
|
|
|
allocated_pci_view = compute_pci_view_map.pop(
|
|
server['OS-EXT-SRV-ATTR:host'])
|
|
not_allocated_pci_view = list(compute_pci_view_map.values())[0]
|
|
|
|
self.assertPCIDeviceCounts(allocated_compute, total=1, free=0)
|
|
allocated_pci_view["usages"][
|
|
"0000:81:00.0"][self.PCI_RC] = 1
|
|
allocated_pci_view["allocations"][
|
|
server['id']] = {"0000:81:00.0": {self.PCI_RC: 1}}
|
|
self.assert_placement_pci_view(allocated_compute, **allocated_pci_view)
|
|
|
|
self.assertPCIDeviceCounts(not_allocated_compute, total=1, free=1)
|
|
self.assert_placement_pci_view(
|
|
not_allocated_compute, **not_allocated_pci_view)
|
|
self.assert_no_pci_healing("test_compute0")
|
|
self.assert_no_pci_healing("test_compute1")
|
|
|
|
def test_multi_create(self):
|
|
self.start_compute(
|
|
hostname='test_compute0',
|
|
pci_info=fakelibvirt.HostPCIDevicesInfo(num_pci=3))
|
|
self.assertPCIDeviceCounts('test_compute0', total=3, free=3)
|
|
test_compute0_placement_pci_view = {
|
|
"inventories": {
|
|
"0000:81:00.0": {self.PCI_RC: 1},
|
|
"0000:81:01.0": {self.PCI_RC: 1},
|
|
"0000:81:02.0": {self.PCI_RC: 1},
|
|
},
|
|
"traits": {
|
|
"0000:81:00.0": [],
|
|
"0000:81:01.0": [],
|
|
"0000:81:02.0": [],
|
|
},
|
|
"usages": {
|
|
"0000:81:00.0": {self.PCI_RC: 0},
|
|
"0000:81:01.0": {self.PCI_RC: 0},
|
|
"0000:81:02.0": {self.PCI_RC: 0},
|
|
},
|
|
"allocations": {},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
extra_spec = {'pci_passthrough:alias': f'{self.ALIAS_NAME}:1'}
|
|
pci_flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
body = self._build_server(flavor_id=pci_flavor_id, networks='none')
|
|
body.update(
|
|
{
|
|
"min_count": "2",
|
|
}
|
|
)
|
|
self.api.post_server({'server': body})
|
|
|
|
servers = self.api.get_servers(detail=False)
|
|
for server in servers:
|
|
self._wait_for_state_change(server, 'ACTIVE')
|
|
|
|
self.assertEqual(2, len(servers))
|
|
self.assertPCIDeviceCounts('test_compute0', total=3, free=1)
|
|
# we have no way to influence which instance takes which device, so
|
|
# we need to look at the nova DB to properly assert the placement
|
|
# allocation
|
|
devices = objects.PciDeviceList.get_by_compute_node(
|
|
self.ctxt,
|
|
objects.ComputeNode.get_by_nodename(self.ctxt, 'test_compute0').id,
|
|
)
|
|
for dev in devices:
|
|
if dev.instance_uuid:
|
|
test_compute0_placement_pci_view["usages"][
|
|
dev.address][self.PCI_RC] = 1
|
|
test_compute0_placement_pci_view["allocations"][
|
|
dev.instance_uuid] = {dev.address: {self.PCI_RC: 1}}
|
|
|
|
self.assert_placement_pci_view(
|
|
"test_compute0", **test_compute0_placement_pci_view)
|
|
|
|
self.assert_no_pci_healing("test_compute0")
|
|
|
|
|
|
class PCIServersWithPreferredNUMATest(_PCIServersTestBase):
|
|
|
|
ALIAS_NAME = 'a1'
|
|
PCI_DEVICE_SPEC = [jsonutils.dumps(
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PCI_PROD_ID,
|
|
}
|
|
)]
|
|
PCI_ALIAS = [jsonutils.dumps(
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PCI_PROD_ID,
|
|
'name': ALIAS_NAME,
|
|
'device_type': fields.PciDeviceType.STANDARD,
|
|
'numa_policy': fields.PCINUMAAffinityPolicy.PREFERRED,
|
|
}
|
|
)]
|
|
expected_state = 'ACTIVE'
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
self.flags(group="pci", report_in_placement=True)
|
|
self.flags(group='filter_scheduler', pci_in_placement=True)
|
|
|
|
def test_create_server_with_pci_dev_and_numa(self):
|
|
"""Validate behavior of 'preferred' PCI NUMA policy.
|
|
|
|
This test ensures that it *is* possible to allocate CPU and memory
|
|
resources from one NUMA node and a PCI device from another *if* PCI
|
|
NUMA policies are in use.
|
|
"""
|
|
|
|
self.flags(cpu_dedicated_set='0-7', group='compute')
|
|
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(num_pci=1, numa_node=0)
|
|
self.start_compute(pci_info=pci_info)
|
|
compute1_placement_pci_view = {
|
|
"inventories": {
|
|
"0000:81:00.0": {self.PCI_RC: 1},
|
|
},
|
|
"traits": {
|
|
"0000:81:00.0": [],
|
|
},
|
|
"usages": {
|
|
"0000:81:00.0": {self.PCI_RC: 0},
|
|
},
|
|
"allocations": {},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"compute1", **compute1_placement_pci_view)
|
|
|
|
# boot one instance with no PCI device to "fill up" NUMA node 0
|
|
extra_spec = {
|
|
'hw:cpu_policy': 'dedicated',
|
|
}
|
|
flavor_id = self._create_flavor(vcpu=4, extra_spec=extra_spec)
|
|
self._create_server(flavor_id=flavor_id)
|
|
|
|
self.assert_placement_pci_view(
|
|
"compute1", **compute1_placement_pci_view)
|
|
|
|
# now boot one with a PCI device, which should succeed thanks to the
|
|
# use of the PCI policy
|
|
extra_spec['pci_passthrough:alias'] = '%s:1' % self.ALIAS_NAME
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
server_with_pci = self._create_server(
|
|
flavor_id=flavor_id, expected_state=self.expected_state)
|
|
|
|
if self.expected_state == 'ACTIVE':
|
|
compute1_placement_pci_view["usages"][
|
|
"0000:81:00.0"][self.PCI_RC] = 1
|
|
compute1_placement_pci_view["allocations"][
|
|
server_with_pci['id']] = {"0000:81:00.0": {self.PCI_RC: 1}}
|
|
|
|
self.assert_placement_pci_view(
|
|
"compute1", **compute1_placement_pci_view)
|
|
self.assert_no_pci_healing("compute1")
|
|
|
|
|
|
class PCIServersWithRequiredNUMATest(PCIServersWithPreferredNUMATest):
|
|
|
|
ALIAS_NAME = 'a1'
|
|
PCI_ALIAS = [jsonutils.dumps(
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PCI_PROD_ID,
|
|
'name': ALIAS_NAME,
|
|
'device_type': fields.PciDeviceType.STANDARD,
|
|
'numa_policy': fields.PCINUMAAffinityPolicy.REQUIRED,
|
|
}
|
|
)]
|
|
expected_state = 'ERROR'
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
self.useFixture(
|
|
fixtures.MockPatch(
|
|
'nova.pci.utils.is_physical_function', return_value=False
|
|
)
|
|
)
|
|
|
|
def test_create_server_with_pci_dev_and_numa_placement_conflict(self):
|
|
# fakelibvirt will simulate the devices:
|
|
# * one type-PCI in 81.00 on numa 0
|
|
# * one type-PCI in 81.01 on numa 1
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(num_pci=2)
|
|
# the device_spec will assign different traits to 81.00 than 81.01
|
|
# so the two devices become different from placement perspective
|
|
device_spec = self._to_list_of_json_str(
|
|
[
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PCI_PROD_ID,
|
|
"address": "0000:81:00.0",
|
|
"traits": "green",
|
|
},
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PCI_PROD_ID,
|
|
"address": "0000:81:01.0",
|
|
"traits": "red",
|
|
},
|
|
]
|
|
)
|
|
self.flags(group='pci', device_spec=device_spec)
|
|
# both numa 0 and numa 1 has 4 PCPUs
|
|
self.flags(cpu_dedicated_set='0-7', group='compute')
|
|
self.start_compute(pci_info=pci_info)
|
|
compute1_placement_pci_view = {
|
|
"inventories": {
|
|
"0000:81:00.0": {self.PCI_RC: 1},
|
|
"0000:81:01.0": {self.PCI_RC: 1},
|
|
},
|
|
"traits": {
|
|
"0000:81:00.0": ["CUSTOM_GREEN"],
|
|
"0000:81:01.0": ["CUSTOM_RED"],
|
|
},
|
|
"usages": {
|
|
"0000:81:00.0": {self.PCI_RC: 0},
|
|
"0000:81:01.0": {self.PCI_RC: 0},
|
|
},
|
|
"allocations": {},
|
|
}
|
|
self.assert_placement_pci_view(
|
|
"compute1", **compute1_placement_pci_view)
|
|
|
|
# boot one instance with no PCI device to "fill up" NUMA node 0
|
|
# so we will have PCPUs on numa 0 and we have PCI on both nodes
|
|
extra_spec = {
|
|
'hw:cpu_policy': 'dedicated',
|
|
}
|
|
flavor_id = self._create_flavor(vcpu=4, extra_spec=extra_spec)
|
|
self._create_server(flavor_id=flavor_id)
|
|
|
|
pci_alias = {
|
|
"resource_class": self.PCI_RC,
|
|
# this means only 81.00 will match in placement which is on numa 0
|
|
"traits": "green",
|
|
"name": "pci-dev",
|
|
# this forces the scheduler to only accept a solution where the
|
|
# PCI device is on the same numa node as the pinned CPUs
|
|
'numa_policy': fields.PCINUMAAffinityPolicy.REQUIRED,
|
|
}
|
|
self.flags(
|
|
group="pci",
|
|
alias=self._to_list_of_json_str([pci_alias]),
|
|
)
|
|
|
|
# Ask for dedicated CPUs, that can only be fulfilled on numa 1.
|
|
# And ask for a PCI alias that can only be fulfilled on numa 0 due to
|
|
# trait request.
|
|
# We expect that this makes the scheduling fail.
|
|
extra_spec = {
|
|
"hw:cpu_policy": "dedicated",
|
|
"pci_passthrough:alias": "pci-dev:1",
|
|
}
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
server = self._create_server(
|
|
flavor_id=flavor_id, expected_state="ERROR")
|
|
|
|
self.assertIn('fault', server)
|
|
self.assertIn('No valid host', server['fault']['message'])
|
|
self.assert_placement_pci_view(
|
|
"compute1", **compute1_placement_pci_view)
|
|
self.assert_no_pci_healing("compute1")
|
|
|
|
|
|
@ddt.ddt
|
|
class PCIServersWithSRIOVAffinityPoliciesTest(_PCIServersTestBase):
|
|
|
|
ALIAS_NAME = 'a1'
|
|
PCI_DEVICE_SPEC = [jsonutils.dumps(
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PCI_PROD_ID,
|
|
}
|
|
)]
|
|
# we set the numa_affinity policy to required to ensure strict affinity
|
|
# between pci devices and the guest cpu and memory will be enforced.
|
|
PCI_ALIAS = [jsonutils.dumps(
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PCI_PROD_ID,
|
|
'name': ALIAS_NAME,
|
|
'device_type': fields.PciDeviceType.STANDARD,
|
|
'numa_policy': fields.PCINUMAAffinityPolicy.REQUIRED,
|
|
}
|
|
)]
|
|
|
|
# NOTE(sean-k-mooney): i could just apply the ddt decorators
|
|
# to this function for the most part but i have chosen to
|
|
# keep one top level function per policy to make documenting
|
|
# the test cases simpler.
|
|
def _test_policy(self, pci_numa_node, status, policy):
|
|
# only allow cpus on numa node 1 to be used for pinning
|
|
self.flags(cpu_dedicated_set='4-7', group='compute')
|
|
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(
|
|
num_pci=1, numa_node=pci_numa_node)
|
|
self.start_compute(pci_info=pci_info)
|
|
|
|
# request cpu pinning to create a numa topology and allow the test to
|
|
# force which numa node the vm would have to be pinned too.
|
|
extra_spec = {
|
|
'hw:cpu_policy': 'dedicated',
|
|
'pci_passthrough:alias': '%s:1' % self.ALIAS_NAME,
|
|
'hw:pci_numa_affinity_policy': policy
|
|
}
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
self._create_server(flavor_id=flavor_id, expected_state=status)
|
|
|
|
if status == 'ACTIVE':
|
|
self.assertTrue(self.mock_filter.called)
|
|
else:
|
|
# the PciPassthroughFilter should not have been called, since the
|
|
# NUMATopologyFilter should have eliminated the filter first
|
|
self.assertFalse(self.mock_filter.called)
|
|
|
|
@ddt.unpack # unpacks each sub-tuple e.g. *(pci_numa_node, status)
|
|
# the preferred policy should always pass regardless of numa affinity
|
|
@ddt.data((-1, 'ACTIVE'), (0, 'ACTIVE'), (1, 'ACTIVE'))
|
|
def test_create_server_with_sriov_numa_affinity_policy_preferred(
|
|
self, pci_numa_node, status):
|
|
"""Validate behavior of 'preferred' PCI NUMA affinity policy.
|
|
|
|
This test ensures that it *is* possible to allocate CPU and memory
|
|
resources from one NUMA node and a PCI device from another *if*
|
|
the SR-IOV NUMA affinity policy is set to preferred.
|
|
"""
|
|
self._test_policy(pci_numa_node, status, 'preferred')
|
|
|
|
@ddt.unpack # unpacks each sub-tuple e.g. *(pci_numa_node, status)
|
|
# the legacy policy allow a PCI device to be used if it has NUMA
|
|
# affinity or if no NUMA info is available so we set the NUMA
|
|
# node for this device to -1 which is the sentinel value use by the
|
|
# Linux kernel for a device with no NUMA affinity.
|
|
@ddt.data((-1, 'ACTIVE'), (0, 'ERROR'), (1, 'ACTIVE'))
|
|
def test_create_server_with_sriov_numa_affinity_policy_legacy(
|
|
self, pci_numa_node, status):
|
|
"""Validate behavior of 'legacy' PCI NUMA affinity policy.
|
|
|
|
This test ensures that it *is* possible to allocate CPU and memory
|
|
resources from one NUMA node and a PCI device from another *if*
|
|
the SR-IOV NUMA affinity policy is set to legacy and the device
|
|
does not report NUMA information.
|
|
"""
|
|
self._test_policy(pci_numa_node, status, 'legacy')
|
|
|
|
@ddt.unpack # unpacks each sub-tuple e.g. *(pci_numa_node, status)
|
|
# The required policy requires a PCI device to both report a NUMA
|
|
# and for the guest cpus and ram to be affinitized to the same
|
|
# NUMA node so we create 1 pci device in the first NUMA node.
|
|
@ddt.data((-1, 'ERROR'), (0, 'ERROR'), (1, 'ACTIVE'))
|
|
def test_create_server_with_sriov_numa_affinity_policy_required(
|
|
self, pci_numa_node, status):
|
|
"""Validate behavior of 'required' PCI NUMA affinity policy.
|
|
|
|
This test ensures that it *is not* possible to allocate CPU and memory
|
|
resources from one NUMA node and a PCI device from another *if*
|
|
the SR-IOV NUMA affinity policy is set to required and the device
|
|
does reports NUMA information.
|
|
"""
|
|
|
|
# we set the numa_affinity policy to preferred to allow the PCI device
|
|
# to be selected from any numa node so we can prove the flavor
|
|
# overrides the alias.
|
|
alias = [jsonutils.dumps(
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PCI_PROD_ID,
|
|
'name': self.ALIAS_NAME,
|
|
'device_type': fields.PciDeviceType.STANDARD,
|
|
'numa_policy': fields.PCINUMAAffinityPolicy.PREFERRED,
|
|
}
|
|
)]
|
|
|
|
self.flags(
|
|
device_spec=self.PCI_DEVICE_SPEC,
|
|
alias=alias,
|
|
group='pci'
|
|
)
|
|
|
|
self._test_policy(pci_numa_node, status, 'required')
|
|
|
|
def test_socket_policy_pass(self):
|
|
# With 1 socket containing 2 NUMA nodes, make the first node's CPU
|
|
# available for pinning, but affine the PCI device to the second node.
|
|
# This should pass.
|
|
host_info = fakelibvirt.HostInfo(
|
|
cpu_nodes=2, cpu_sockets=1, cpu_cores=2, cpu_threads=2,
|
|
kB_mem=(16 * units.Gi) // units.Ki)
|
|
self.flags(cpu_dedicated_set='0-3', group='compute')
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(num_pci=1, numa_node=1)
|
|
|
|
self.start_compute(host_info=host_info, pci_info=pci_info)
|
|
|
|
extra_spec = {
|
|
'hw:cpu_policy': 'dedicated',
|
|
'pci_passthrough:alias': '%s:1' % self.ALIAS_NAME,
|
|
'hw:pci_numa_affinity_policy': 'socket'
|
|
}
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
self._create_server(flavor_id=flavor_id)
|
|
self.assertTrue(self.mock_filter.called)
|
|
|
|
def test_socket_policy_fail(self):
|
|
# With 2 sockets containing 1 NUMA node each, make the first socket's
|
|
# CPUs available for pinning, but affine the PCI device to the second
|
|
# NUMA node in the second socket. This should fail.
|
|
host_info = fakelibvirt.HostInfo(
|
|
cpu_nodes=1, cpu_sockets=2, cpu_cores=2, cpu_threads=2,
|
|
kB_mem=(16 * units.Gi) // units.Ki)
|
|
self.flags(cpu_dedicated_set='0-3', group='compute')
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(num_pci=1, numa_node=1)
|
|
self.start_compute(host_info=host_info, pci_info=pci_info)
|
|
|
|
extra_spec = {
|
|
'hw:cpu_policy': 'dedicated',
|
|
'pci_passthrough:alias': '%s:1' % self.ALIAS_NAME,
|
|
'hw:pci_numa_affinity_policy': 'socket'
|
|
}
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
server = self._create_server(
|
|
flavor_id=flavor_id, expected_state='ERROR')
|
|
self.assertIn('fault', server)
|
|
self.assertIn('No valid host', server['fault']['message'])
|
|
|
|
def test_socket_policy_multi_numa_pass(self):
|
|
# 2 sockets, 2 NUMA nodes each, with the PCI device on NUMA 0 and
|
|
# socket 0. If we restrict cpu_dedicated_set to NUMA 1, 2 and 3, we
|
|
# should still be able to boot an instance with hw:numa_nodes=3 and the
|
|
# `socket` policy, because one of the instance's NUMA nodes will be on
|
|
# the same socket as the PCI device (even if there is no direct NUMA
|
|
# node affinity).
|
|
host_info = fakelibvirt.HostInfo(
|
|
cpu_nodes=2, cpu_sockets=2, cpu_cores=2, cpu_threads=1,
|
|
kB_mem=(16 * units.Gi) // units.Ki)
|
|
self.flags(cpu_dedicated_set='2-7', group='compute')
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(num_pci=1, numa_node=0)
|
|
|
|
self.start_compute(host_info=host_info, pci_info=pci_info)
|
|
|
|
extra_spec = {
|
|
'hw:numa_nodes': '3',
|
|
'hw:cpu_policy': 'dedicated',
|
|
'pci_passthrough:alias': '%s:1' % self.ALIAS_NAME,
|
|
'hw:pci_numa_affinity_policy': 'socket'
|
|
}
|
|
flavor_id = self._create_flavor(vcpu=6, memory_mb=3144,
|
|
extra_spec=extra_spec)
|
|
self._create_server(flavor_id=flavor_id)
|
|
self.assertTrue(self.mock_filter.called)
|
|
|
|
|
|
@ddt.ddt
|
|
class PCIServersWithPortNUMAPoliciesTest(_PCIServersTestBase):
|
|
|
|
ALIAS_NAME = 'a1'
|
|
PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in (
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PF_PROD_ID,
|
|
'physical_network': 'physnet4',
|
|
},
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.VF_PROD_ID,
|
|
'physical_network': 'physnet4',
|
|
},
|
|
)]
|
|
# we set the numa_affinity policy to required to ensure strict affinity
|
|
# between pci devices and the guest cpu and memory will be enforced.
|
|
PCI_ALIAS = [jsonutils.dumps(
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PCI_PROD_ID,
|
|
'name': ALIAS_NAME,
|
|
'device_type': fields.PciDeviceType.STANDARD,
|
|
'numa_policy': fields.PCINUMAAffinityPolicy.REQUIRED,
|
|
}
|
|
)]
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
|
|
# The ultimate base class _IntegratedTestBase uses NeutronFixture but
|
|
# we need a bit more intelligent neutron for these tests. Applying the
|
|
# new fixture here means that we re-stub what the previous neutron
|
|
# fixture already stubbed.
|
|
self.neutron = self.useFixture(base.LibvirtNeutronFixture(self))
|
|
self.flags(disable_fallback_pcpu_query=True, group='workarounds')
|
|
|
|
def _create_port_with_policy(self, policy):
|
|
port_data = copy.deepcopy(
|
|
base.LibvirtNeutronFixture.network_4_port_1)
|
|
port_data[constants.NUMA_POLICY] = policy
|
|
# create the port
|
|
new_port = self.neutron.create_port({'port': port_data})
|
|
port_id = new_port['port']['id']
|
|
port = self.neutron.show_port(port_id)['port']
|
|
self.assertEqual(port[constants.NUMA_POLICY], policy)
|
|
return port_id
|
|
|
|
# NOTE(sean-k-mooney): i could just apply the ddt decorators
|
|
# to this function for the most part but i have chosen to
|
|
# keep one top level function per policy to make documenting
|
|
# the test cases simpler.
|
|
def _test_policy(self, pci_numa_node, status, policy):
|
|
# only allow cpus on numa node 1 to be used for pinning
|
|
self.flags(cpu_dedicated_set='4-7', group='compute')
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(
|
|
num_pfs=1, num_vfs=2, numa_node=pci_numa_node)
|
|
self.start_compute(pci_info=pci_info)
|
|
|
|
# request cpu pinning to create a numa topology and allow the test to
|
|
# force which numa node the vm would have to be pinned too.
|
|
extra_spec = {
|
|
'hw:cpu_policy': 'dedicated',
|
|
}
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
|
|
port_id = self._create_port_with_policy(policy)
|
|
# create a server using the VF via neutron
|
|
self._create_server(
|
|
flavor_id=flavor_id,
|
|
networks=[
|
|
{'port': port_id},
|
|
],
|
|
expected_state=status
|
|
)
|
|
|
|
if status == 'ACTIVE':
|
|
self.assertTrue(self.mock_filter.called)
|
|
else:
|
|
# the PciPassthroughFilter should not have been called, since the
|
|
# NUMATopologyFilter should have eliminated the filter first
|
|
self.assertFalse(self.mock_filter.called)
|
|
|
|
@ddt.unpack # unpacks each sub-tuple e.g. *(pci_numa_node, status)
|
|
# the preferred policy should always pass regardless of numa affinity
|
|
@ddt.data((-1, 'ACTIVE'), (0, 'ACTIVE'), (1, 'ACTIVE'))
|
|
def test_create_server_with_sriov_numa_affinity_policy_preferred(
|
|
self, pci_numa_node, status):
|
|
"""Validate behavior of 'preferred' PCI NUMA affinity policy.
|
|
|
|
This test ensures that it *is* possible to allocate CPU and memory
|
|
resources from one NUMA node and a PCI device from another *if*
|
|
the port NUMA affinity policy is set to preferred.
|
|
"""
|
|
self._test_policy(pci_numa_node, status, 'preferred')
|
|
|
|
@ddt.unpack # unpacks each sub-tuple e.g. *(pci_numa_node, status)
|
|
# the legacy policy allow a PCI device to be used if it has NUMA
|
|
# affinity or if no NUMA info is available so we set the NUMA
|
|
# node for this device to -1 which is the sentinel value use by the
|
|
# Linux kernel for a device with no NUMA affinity.
|
|
@ddt.data((-1, 'ACTIVE'), (0, 'ERROR'), (1, 'ACTIVE'))
|
|
def test_create_server_with_sriov_numa_affinity_policy_legacy(
|
|
self, pci_numa_node, status):
|
|
"""Validate behavior of 'legacy' PCI NUMA affinity policy.
|
|
|
|
This test ensures that it *is* possible to allocate CPU and memory
|
|
resources from one NUMA node and a PCI device from another *if*
|
|
the port NUMA affinity policy is set to legacy and the device
|
|
does not report NUMA information.
|
|
"""
|
|
self._test_policy(pci_numa_node, status, 'legacy')
|
|
|
|
@ddt.unpack # unpacks each sub-tuple e.g. *(pci_numa_node, status)
|
|
# The required policy requires a PCI device to both report a NUMA
|
|
# and for the guest cpus and ram to be affinitized to the same
|
|
# NUMA node so we create 1 pci device in the first NUMA node.
|
|
@ddt.data((-1, 'ERROR'), (0, 'ERROR'), (1, 'ACTIVE'))
|
|
def test_create_server_with_sriov_numa_affinity_policy_required(
|
|
self, pci_numa_node, status):
|
|
"""Validate behavior of 'required' PCI NUMA affinity policy.
|
|
|
|
This test ensures that it *is not* possible to allocate CPU and memory
|
|
resources from one NUMA node and a PCI device from another *if*
|
|
the port NUMA affinity policy is set to required and the device
|
|
does reports NUMA information.
|
|
"""
|
|
|
|
# we set the numa_affinity policy to preferred to allow the PCI device
|
|
# to be selected from any numa node so we can prove the flavor
|
|
# overrides the alias.
|
|
alias = [jsonutils.dumps(
|
|
{
|
|
'vendor_id': fakelibvirt.PCI_VEND_ID,
|
|
'product_id': fakelibvirt.PCI_PROD_ID,
|
|
'name': self.ALIAS_NAME,
|
|
'device_type': fields.PciDeviceType.STANDARD,
|
|
'numa_policy': fields.PCINUMAAffinityPolicy.PREFERRED,
|
|
}
|
|
)]
|
|
|
|
self.flags(
|
|
device_spec=self.PCI_DEVICE_SPEC,
|
|
alias=alias,
|
|
group='pci'
|
|
)
|
|
|
|
self._test_policy(pci_numa_node, status, 'required')
|
|
|
|
def test_socket_policy_pass(self):
|
|
# With 1 socket containing 2 NUMA nodes, make the first node's CPU
|
|
# available for pinning, but affine the PCI device to the second node.
|
|
# This should pass.
|
|
host_info = fakelibvirt.HostInfo(
|
|
cpu_nodes=2, cpu_sockets=1, cpu_cores=2, cpu_threads=2,
|
|
kB_mem=(16 * units.Gi) // units.Ki)
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(
|
|
num_pfs=1, num_vfs=1, numa_node=1)
|
|
self.flags(cpu_dedicated_set='0-3', group='compute')
|
|
self.start_compute(host_info=host_info, pci_info=pci_info)
|
|
|
|
extra_spec = {
|
|
'hw:cpu_policy': 'dedicated',
|
|
}
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
port_id = self._create_port_with_policy('socket')
|
|
# create a server using the VF via neutron
|
|
self._create_server(
|
|
flavor_id=flavor_id,
|
|
networks=[
|
|
{'port': port_id},
|
|
],
|
|
)
|
|
self.assertTrue(self.mock_filter.called)
|
|
|
|
def test_socket_policy_fail(self):
|
|
# With 2 sockets containing 1 NUMA node each, make the first socket's
|
|
# CPUs available for pinning, but affine the PCI device to the second
|
|
# NUMA node in the second socket. This should fail.
|
|
host_info = fakelibvirt.HostInfo(
|
|
cpu_nodes=1, cpu_sockets=2, cpu_cores=2, cpu_threads=2,
|
|
kB_mem=(16 * units.Gi) // units.Ki)
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(
|
|
num_pfs=1, num_vfs=1, numa_node=1)
|
|
self.flags(cpu_dedicated_set='0-3', group='compute')
|
|
self.start_compute(host_info=host_info, pci_info=pci_info)
|
|
|
|
extra_spec = {
|
|
'hw:cpu_policy': 'dedicated',
|
|
}
|
|
flavor_id = self._create_flavor(extra_spec=extra_spec)
|
|
port_id = self._create_port_with_policy('socket')
|
|
# create a server using the VF via neutron
|
|
server = self._create_server(
|
|
flavor_id=flavor_id,
|
|
networks=[
|
|
{'port': port_id},
|
|
],
|
|
expected_state='ERROR'
|
|
)
|
|
self.assertIn('fault', server)
|
|
self.assertIn('No valid host', server['fault']['message'])
|
|
self.assertFalse(self.mock_filter.called)
|
|
|
|
def test_socket_policy_multi_numa_pass(self):
|
|
# 2 sockets, 2 NUMA nodes each, with the PCI device on NUMA 0 and
|
|
# socket 0. If we restrict cpu_dedicated_set to NUMA 1, 2 and 3, we
|
|
# should still be able to boot an instance with hw:numa_nodes=3 and the
|
|
# `socket` policy, because one of the instance's NUMA nodes will be on
|
|
# the same socket as the PCI device (even if there is no direct NUMA
|
|
# node affinity).
|
|
host_info = fakelibvirt.HostInfo(
|
|
cpu_nodes=2, cpu_sockets=2, cpu_cores=2, cpu_threads=1,
|
|
kB_mem=(16 * units.Gi) // units.Ki)
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(
|
|
num_pfs=1, num_vfs=1, numa_node=0)
|
|
self.flags(cpu_dedicated_set='2-7', group='compute')
|
|
self.start_compute(host_info=host_info, pci_info=pci_info)
|
|
|
|
extra_spec = {
|
|
'hw:numa_nodes': '3',
|
|
'hw:cpu_policy': 'dedicated',
|
|
}
|
|
flavor_id = self._create_flavor(vcpu=6, memory_mb=3144,
|
|
extra_spec=extra_spec)
|
|
port_id = self._create_port_with_policy('socket')
|
|
# create a server using the VF via neutron
|
|
self._create_server(
|
|
flavor_id=flavor_id,
|
|
networks=[
|
|
{'port': port_id},
|
|
],
|
|
)
|
|
self.assertTrue(self.mock_filter.called)
|
|
|
|
|
|
class RemoteManagedServersTest(_PCIServersWithMigrationTestBase):
|
|
|
|
ADMIN_API = True
|
|
microversion = 'latest'
|
|
|
|
PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in (
|
|
# A PF with access to physnet4.
|
|
{
|
|
'vendor_id': '15b3',
|
|
'product_id': 'a2dc',
|
|
'physical_network': 'physnet4',
|
|
'remote_managed': 'false',
|
|
},
|
|
# A VF with access to physnet4.
|
|
{
|
|
'vendor_id': '15b3',
|
|
'product_id': '1021',
|
|
'physical_network': 'physnet4',
|
|
'remote_managed': 'true',
|
|
},
|
|
# A PF programmed to forward traffic to an overlay network.
|
|
{
|
|
'vendor_id': '15b3',
|
|
'product_id': 'a2d6',
|
|
'physical_network': None,
|
|
'remote_managed': 'false',
|
|
},
|
|
# A VF programmed to forward traffic to an overlay network.
|
|
{
|
|
'vendor_id': '15b3',
|
|
'product_id': '101e',
|
|
'physical_network': None,
|
|
'remote_managed': 'true',
|
|
},
|
|
)]
|
|
|
|
PCI_ALIAS = []
|
|
|
|
NUM_PFS = 1
|
|
NUM_VFS = 4
|
|
vf_ratio = NUM_VFS // NUM_PFS
|
|
|
|
# Min Libvirt version that supports working with PCI VPD.
|
|
FAKE_LIBVIRT_VERSION = 7_009_000 # 7.9.0
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
self.neutron = self.useFixture(base.LibvirtNeutronFixture(self))
|
|
|
|
self.useFixture(fixtures.MockPatch(
|
|
'nova.pci.utils.get_vf_num_by_pci_address',
|
|
new=mock.MagicMock(
|
|
side_effect=lambda addr: self._get_pci_function_number(addr))))
|
|
|
|
self.useFixture(fixtures.MockPatch(
|
|
'nova.pci.utils.get_mac_by_pci_address',
|
|
new=mock.MagicMock(
|
|
side_effect=(
|
|
lambda addr: {
|
|
"0000:80:00.0": "52:54:00:1e:59:42",
|
|
"0000:81:00.0": "52:54:00:1e:59:01",
|
|
"0000:82:00.0": "52:54:00:1e:59:02",
|
|
}.get(addr)
|
|
)
|
|
)
|
|
))
|
|
|
|
@classmethod
|
|
def _get_pci_function_number(cls, pci_addr: str):
|
|
"""Get a VF function number based on a PCI address.
|
|
|
|
Assume that the PCI ARI capability is enabled (slot bits become a part
|
|
of a function number).
|
|
"""
|
|
_, _, slot, function = parse_address(pci_addr)
|
|
# The number of PFs is extracted to get a VF number.
|
|
return int(slot, 16) + int(function, 16) - cls.NUM_PFS
|
|
|
|
def start_compute(
|
|
self, hostname='test_compute0', host_info=None, pci_info=None,
|
|
mdev_info=None, vdpa_info=None,
|
|
libvirt_version=None,
|
|
qemu_version=None):
|
|
|
|
if not pci_info:
|
|
pci_info = fakelibvirt.HostPCIDevicesInfo(
|
|
num_pci=0, num_pfs=0, num_vfs=0)
|
|
|
|
pci_info.add_device(
|
|
dev_type='PF',
|
|
bus=0x81,
|
|
slot=0x0,
|
|
function=0,
|
|
iommu_group=42,
|
|
numa_node=0,
|
|
vf_ratio=self.vf_ratio,
|
|
vend_id='15b3',
|
|
vend_name='Mellanox Technologies',
|
|
prod_id='a2dc',
|
|
prod_name='BlueField-3 integrated ConnectX-7 controller',
|
|
driver_name='mlx5_core',
|
|
vpd_fields={
|
|
'name': 'MT43244 BlueField-3 integrated ConnectX-7',
|
|
'readonly': {
|
|
'serial_number': 'MT0000X00001',
|
|
},
|
|
}
|
|
)
|
|
|
|
for idx in range(self.NUM_VFS):
|
|
pci_info.add_device(
|
|
dev_type='VF',
|
|
bus=0x81,
|
|
slot=0x0,
|
|
function=idx + 1,
|
|
iommu_group=idx + 43,
|
|
numa_node=0,
|
|
vf_ratio=self.vf_ratio,
|
|
parent=(0x81, 0x0, 0),
|
|
vend_id='15b3',
|
|
vend_name='Mellanox Technologies',
|
|
prod_id='1021',
|
|
prod_name='MT2910 Family [ConnectX-7]',
|
|
driver_name='mlx5_core',
|
|
vpd_fields={
|
|
'name': 'MT2910 Family [ConnectX-7]',
|
|
'readonly': {
|
|
'serial_number': 'MT0000X00001',
|
|
},
|
|
}
|
|
)
|
|
|
|
pci_info.add_device(
|
|
dev_type='PF',
|
|
bus=0x82,
|
|
slot=0x0,
|
|
function=0,
|
|
iommu_group=84,
|
|
numa_node=0,
|
|
vf_ratio=self.vf_ratio,
|
|
vend_id='15b3',
|
|
vend_name='Mellanox Technologies',
|
|
prod_id='a2d6',
|
|
prod_name='MT42822 BlueField-2 integrated ConnectX-6',
|
|
driver_name='mlx5_core',
|
|
vpd_fields={
|
|
'name': 'MT42822 BlueField-2 integrated ConnectX-6',
|
|
'readonly': {
|
|
'serial_number': 'MT0000X00002',
|
|
},
|
|
}
|
|
)
|
|
|
|
for idx in range(self.NUM_VFS):
|
|
pci_info.add_device(
|
|
dev_type='VF',
|
|
bus=0x82,
|
|
slot=0x0,
|
|
function=idx + 1,
|
|
iommu_group=idx + 85,
|
|
numa_node=0,
|
|
vf_ratio=self.vf_ratio,
|
|
parent=(0x82, 0x0, 0),
|
|
vend_id='15b3',
|
|
vend_name='Mellanox Technologies',
|
|
prod_id='101e',
|
|
prod_name='ConnectX Family mlx5Gen Virtual Function',
|
|
driver_name='mlx5_core')
|
|
|
|
return super().start_compute(
|
|
hostname=hostname, host_info=host_info, pci_info=pci_info,
|
|
mdev_info=mdev_info, vdpa_info=vdpa_info,
|
|
libvirt_version=libvirt_version or self.FAKE_LIBVIRT_VERSION)
|
|
|
|
def create_remote_managed_tunnel_port(self):
|
|
dpu_tunnel_port = {
|
|
'id': uuids.dpu_tunnel_port,
|
|
'network_id': self.neutron.network_3['id'],
|
|
'status': 'ACTIVE',
|
|
'mac_address': 'fa:16:3e:f0:a4:bb',
|
|
'fixed_ips': [
|
|
{
|
|
'ip_address': '192.168.2.8',
|
|
'subnet_id': self.neutron.subnet_3['id']
|
|
}
|
|
],
|
|
'binding:vif_details': {},
|
|
'binding:vif_type': 'ovs',
|
|
'binding:vnic_type': 'remote-managed',
|
|
}
|
|
|
|
self.neutron.create_port({'port': dpu_tunnel_port})
|
|
return dpu_tunnel_port
|
|
|
|
def create_remote_managed_physnet_port(self):
|
|
dpu_physnet_port = {
|
|
'id': uuids.dpu_physnet_port,
|
|
'network_id': self.neutron.network_4['id'],
|
|
'status': 'ACTIVE',
|
|
'mac_address': 'd2:0b:fd:99:89:8b',
|
|
'fixed_ips': [
|
|
{
|
|
'ip_address': '192.168.4.10',
|
|
'subnet_id': self.neutron.subnet_4['id']
|
|
}
|
|
],
|
|
'binding:vif_details': {},
|
|
'binding:vif_type': 'ovs',
|
|
'binding:vnic_type': 'remote-managed',
|
|
}
|
|
|
|
self.neutron.create_port({'port': dpu_physnet_port})
|
|
return dpu_physnet_port
|
|
|
|
def test_create_server_physnet(self):
|
|
"""Create an instance with a tunnel remote-managed port."""
|
|
|
|
hostname = self.start_compute()
|
|
num_pci = (self.NUM_PFS + self.NUM_VFS) * 2
|
|
|
|
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci)
|
|
|
|
dpu_port = self.create_remote_managed_physnet_port()
|
|
|
|
port = self.neutron.show_port(dpu_port['id'])['port']
|
|
self.assertNotIn('binding:profile', port)
|
|
|
|
self._create_server(networks=[{'port': dpu_port['id']}])
|
|
|
|
# Ensure there is one less VF available and that the PF
|
|
# is no longer usable.
|
|
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci - 2)
|
|
|
|
# Ensure the binding:profile details sent to Neutron are correct after
|
|
# a port update.
|
|
port = self.neutron.show_port(dpu_port['id'])['port']
|
|
self.assertIn('binding:profile', port)
|
|
self.assertEqual({
|
|
'card_serial_number': 'MT0000X00001',
|
|
'pci_slot': '0000:81:00.4',
|
|
'pci_vendor_info': '15b3:1021',
|
|
'pf_mac_address': '52:54:00:1e:59:01',
|
|
'physical_network': 'physnet4',
|
|
'vf_num': 3
|
|
}, port['binding:profile'])
|
|
|
|
def test_create_server_tunnel(self):
|
|
"""Create an instance with a tunnel remote-managed port."""
|
|
|
|
hostname = self.start_compute()
|
|
num_pci = (self.NUM_PFS + self.NUM_VFS) * 2
|
|
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci)
|
|
|
|
dpu_port = self.create_remote_managed_tunnel_port()
|
|
port = self.neutron.show_port(dpu_port['id'])['port']
|
|
self.assertNotIn('binding:profile', port)
|
|
|
|
self._create_server(networks=[{'port': dpu_port['id']}])
|
|
|
|
# Ensure there is one less VF available and that the PF
|
|
# is no longer usable.
|
|
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci - 2)
|
|
|
|
# Ensure the binding:profile details sent to Neutron are correct after
|
|
# a port update.
|
|
port = self.neutron.show_port(dpu_port['id'])['port']
|
|
self.assertIn('binding:profile', port)
|
|
self.assertEqual({
|
|
'card_serial_number': 'MT0000X00002',
|
|
'pci_slot': '0000:82:00.4',
|
|
'pci_vendor_info': '15b3:101e',
|
|
'pf_mac_address': '52:54:00:1e:59:02',
|
|
'physical_network': None,
|
|
'vf_num': 3
|
|
}, port['binding:profile'])
|
|
|
|
def _test_common(self, op, *args, **kwargs):
|
|
self.start_compute()
|
|
dpu_port = self.create_remote_managed_tunnel_port()
|
|
server = self._create_server(networks=[{'port': dpu_port['id']}])
|
|
op(server, *args, **kwargs)
|
|
|
|
def test_attach_interface(self):
|
|
self.start_compute()
|
|
|
|
dpu_port = self.create_remote_managed_tunnel_port()
|
|
server = self._create_server(networks='none')
|
|
|
|
self._attach_interface(server, dpu_port['id'])
|
|
|
|
port = self.neutron.show_port(dpu_port['id'])['port']
|
|
self.assertIn('binding:profile', port)
|
|
self.assertEqual(
|
|
{
|
|
'pci_vendor_info': '15b3:101e',
|
|
'pci_slot': '0000:82:00.4',
|
|
'physical_network': None,
|
|
'pf_mac_address': '52:54:00:1e:59:02',
|
|
'vf_num': 3,
|
|
'card_serial_number': 'MT0000X00002',
|
|
},
|
|
port['binding:profile'],
|
|
)
|
|
|
|
def test_detach_interface(self):
|
|
self._test_common(self._detach_interface, uuids.dpu_tunnel_port)
|
|
|
|
port = self.neutron.show_port(uuids.dpu_tunnel_port)['port']
|
|
self.assertIn('binding:profile', port)
|
|
self.assertEqual({}, port['binding:profile'])
|
|
|
|
def test_shelve(self):
|
|
self._test_common(self._shelve_server)
|
|
|
|
port = self.neutron.show_port(uuids.dpu_tunnel_port)['port']
|
|
self.assertIn('binding:profile', port)
|
|
self.assertEqual({}, port['binding:profile'])
|
|
|
|
def test_suspend(self):
|
|
self.start_compute()
|
|
dpu_port = self.create_remote_managed_tunnel_port()
|
|
server = self._create_server(networks=[{'port': dpu_port['id']}])
|
|
self._suspend_server(server)
|
|
# TODO(dmitriis): detachDevice does not properly handle hostdevs
|
|
# so full suspend/resume testing is problematic.
|
|
|
|
def _test_move_operation_with_neutron(self, move_operation, dpu_port):
|
|
"""Test a move operation with a remote-managed port.
|
|
"""
|
|
compute1_pci_info = fakelibvirt.HostPCIDevicesInfo(
|
|
num_pfs=0, num_vfs=0)
|
|
|
|
compute1_pci_info.add_device(
|
|
dev_type='PF',
|
|
bus=0x80,
|
|
slot=0x0,
|
|
function=0,
|
|
iommu_group=84,
|
|
numa_node=1,
|
|
vf_ratio=self.vf_ratio,
|
|
vend_id='15b3',
|
|
vend_name='Mellanox Technologies',
|
|
prod_id='a2d6',
|
|
prod_name='MT42822 BlueField-2 integrated ConnectX-6',
|
|
driver_name='mlx5_core',
|
|
vpd_fields={
|
|
'name': 'MT42822 BlueField-2 integrated ConnectX-6',
|
|
'readonly': {
|
|
'serial_number': 'MT0000X00042',
|
|
},
|
|
}
|
|
)
|
|
for idx in range(self.NUM_VFS):
|
|
compute1_pci_info.add_device(
|
|
dev_type='VF',
|
|
bus=0x80,
|
|
slot=0x0,
|
|
function=idx + 1,
|
|
iommu_group=idx + 85,
|
|
numa_node=1,
|
|
vf_ratio=self.vf_ratio,
|
|
parent=(0x80, 0x0, 0),
|
|
vend_id='15b3',
|
|
vend_name='Mellanox Technologies',
|
|
prod_id='101e',
|
|
prod_name='ConnectX Family mlx5Gen Virtual Function',
|
|
driver_name='mlx5_core',
|
|
vpd_fields={
|
|
'name': 'MT42822 BlueField-2 integrated ConnectX-6',
|
|
'readonly': {
|
|
'serial_number': 'MT0000X00042',
|
|
},
|
|
}
|
|
)
|
|
|
|
self.start_compute(hostname='test_compute0')
|
|
self.start_compute(hostname='test_compute1',
|
|
pci_info=compute1_pci_info)
|
|
|
|
port = self.neutron.show_port(dpu_port['id'])['port']
|
|
self.assertNotIn('binding:profile', port)
|
|
|
|
flavor_id = self._create_flavor(vcpu=4)
|
|
server = self._create_server(
|
|
flavor_id=flavor_id,
|
|
networks=[{'port': dpu_port['id']}],
|
|
host='test_compute0',
|
|
)
|
|
|
|
self.assertEqual('test_compute0', server['OS-EXT-SRV-ATTR:host'])
|
|
self.assertPCIDeviceCounts('test_compute0', total=10, free=8)
|
|
self.assertPCIDeviceCounts('test_compute1', total=5, free=5)
|
|
|
|
port = self.neutron.show_port(dpu_port['id'])['port']
|
|
self.assertIn('binding:profile', port)
|
|
self.assertEqual(
|
|
{
|
|
'pci_vendor_info': '15b3:101e',
|
|
'pci_slot': '0000:82:00.4',
|
|
'physical_network': None,
|
|
'pf_mac_address': '52:54:00:1e:59:02',
|
|
'vf_num': 3,
|
|
'card_serial_number': 'MT0000X00002',
|
|
},
|
|
port['binding:profile'],
|
|
)
|
|
|
|
move_operation(server)
|
|
|
|
def test_unshelve_server_with_neutron(self):
|
|
def move_operation(source_server):
|
|
self._shelve_server(source_server)
|
|
# Disable the source compute, to force unshelving on the dest.
|
|
self.api.put_service(
|
|
self.computes['test_compute0'].service_ref.uuid,
|
|
{'status': 'disabled'})
|
|
self._unshelve_server(source_server)
|
|
|
|
dpu_port = self.create_remote_managed_tunnel_port()
|
|
self._test_move_operation_with_neutron(move_operation, dpu_port)
|
|
|
|
self.assertPCIDeviceCounts('test_compute0', total=10, free=10)
|
|
self.assertPCIDeviceCounts('test_compute1', total=5, free=3)
|
|
|
|
# Ensure the binding:profile details got updated, including the
|
|
# fields relevant to remote-managed ports.
|
|
port = self.neutron.show_port(dpu_port['id'])['port']
|
|
self.assertIn('binding:profile', port)
|
|
self.assertEqual(
|
|
{
|
|
'pci_vendor_info': '15b3:101e',
|
|
'pci_slot': '0000:80:00.4',
|
|
'physical_network': None,
|
|
'pf_mac_address': '52:54:00:1e:59:42',
|
|
'vf_num': 3,
|
|
'card_serial_number': 'MT0000X00042',
|
|
},
|
|
port['binding:profile'],
|
|
)
|
|
|
|
def test_cold_migrate_server_with_neutron(self):
|
|
def move_operation(source_server):
|
|
with mock.patch('nova.virt.libvirt.driver.LibvirtDriver'
|
|
'.migrate_disk_and_power_off', return_value='{}'):
|
|
server = self._migrate_server(source_server)
|
|
self._confirm_resize(server)
|
|
|
|
self.assertPCIDeviceCounts('test_compute0', total=10, free=10)
|
|
self.assertPCIDeviceCounts('test_compute1', total=5, free=3)
|
|
|
|
# Ensure the binding:profile details got updated, including the
|
|
# fields relevant to remote-managed ports.
|
|
port = self.neutron.show_port(dpu_port['id'])['port']
|
|
self.assertIn('binding:profile', port)
|
|
self.assertEqual(
|
|
{
|
|
'pci_vendor_info': '15b3:101e',
|
|
'pci_slot': '0000:80:00.4',
|
|
'physical_network': None,
|
|
'pf_mac_address': '52:54:00:1e:59:42',
|
|
'vf_num': 3,
|
|
'card_serial_number': 'MT0000X00042',
|
|
},
|
|
port['binding:profile'],
|
|
)
|
|
|
|
dpu_port = self.create_remote_managed_tunnel_port()
|
|
self._test_move_operation_with_neutron(move_operation, dpu_port)
|
|
|
|
def test_cold_migrate_server_with_neutron_revert(self):
|
|
def move_operation(source_server):
|
|
with mock.patch('nova.virt.libvirt.driver.LibvirtDriver'
|
|
'.migrate_disk_and_power_off', return_value='{}'):
|
|
server = self._migrate_server(source_server)
|
|
|
|
self.assertPCIDeviceCounts('test_compute0', total=10, free=8)
|
|
self.assertPCIDeviceCounts('test_compute1', total=5, free=3)
|
|
|
|
self._revert_resize(server)
|
|
|
|
self.assertPCIDeviceCounts('test_compute0', total=10, free=8)
|
|
self.assertPCIDeviceCounts('test_compute1', total=5, free=5)
|
|
|
|
port = self.neutron.show_port(dpu_port['id'])['port']
|
|
self.assertIn('binding:profile', port)
|
|
self.assertEqual(
|
|
{
|
|
'pci_vendor_info': '15b3:101e',
|
|
'pci_slot': '0000:82:00.4',
|
|
'physical_network': None,
|
|
'pf_mac_address': '52:54:00:1e:59:02',
|
|
'vf_num': 3,
|
|
'card_serial_number': 'MT0000X00002',
|
|
},
|
|
port['binding:profile'],
|
|
)
|
|
|
|
dpu_port = self.create_remote_managed_tunnel_port()
|
|
self._test_move_operation_with_neutron(move_operation, dpu_port)
|
|
|
|
def test_evacuate_server_with_neutron(self):
|
|
def move_operation(source_server):
|
|
# Down the source compute to enable the evacuation
|
|
self.api.put_service(
|
|
self.computes['test_compute0'].service_ref.uuid,
|
|
{'forced_down': True})
|
|
self.computes['test_compute0'].stop()
|
|
self._evacuate_server(source_server)
|
|
|
|
dpu_port = self.create_remote_managed_tunnel_port()
|
|
self._test_move_operation_with_neutron(move_operation, dpu_port)
|
|
|
|
self.assertPCIDeviceCounts('test_compute0', total=10, free=8)
|
|
self.assertPCIDeviceCounts('test_compute1', total=5, free=3)
|
|
|
|
# Ensure the binding:profile details got updated, including the
|
|
# fields relevant to remote-managed ports.
|
|
port = self.neutron.show_port(dpu_port['id'])['port']
|
|
self.assertIn('binding:profile', port)
|
|
self.assertEqual(
|
|
{
|
|
'pci_vendor_info': '15b3:101e',
|
|
'pci_slot': '0000:80:00.4',
|
|
'physical_network': None,
|
|
'pf_mac_address': '52:54:00:1e:59:42',
|
|
'vf_num': 3,
|
|
'card_serial_number': 'MT0000X00042',
|
|
},
|
|
port['binding:profile'],
|
|
)
|
|
|
|
def test_live_migrate_server_with_neutron(self):
|
|
"""Live migrate an instance using a remote-managed port.
|
|
|
|
This should succeed since we support this via detach and attach of the
|
|
PCI device similar to how this is done for SR-IOV ports.
|
|
"""
|
|
def move_operation(source_server):
|
|
self._live_migrate(source_server, 'completed')
|
|
|
|
dpu_port = self.create_remote_managed_tunnel_port()
|
|
self._test_move_operation_with_neutron(move_operation, dpu_port)
|
|
|
|
self.assertPCIDeviceCounts('test_compute0', total=10, free=10)
|
|
self.assertPCIDeviceCounts('test_compute1', total=5, free=3)
|
|
|
|
# Ensure the binding:profile details got updated, including the
|
|
# fields relevant to remote-managed ports.
|
|
port = self.neutron.show_port(dpu_port['id'])['port']
|
|
self.assertIn('binding:profile', port)
|
|
self.assertEqual(
|
|
{
|
|
'pci_vendor_info': '15b3:101e',
|
|
'pci_slot': '0000:80:00.4',
|
|
'physical_network': None,
|
|
'pf_mac_address': '52:54:00:1e:59:42',
|
|
'vf_num': 3,
|
|
'card_serial_number': 'MT0000X00042',
|
|
},
|
|
port['binding:profile'],
|
|
)
|