From 14e3b352c24b2a1fe54ba13a733cf6e7989215cc Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Wed, 23 Feb 2022 16:14:03 +0000 Subject: [PATCH] libvirt: Add vIOMMU device to guest Implementation for BP/libvirt-viommu-device. With provide `hw:viommu_model` property to extra_specs or `hw_viommu_model` to image property. will enable viommu to libvirt guest. [1] https://www.berrange.com/posts/2017/02/16/setting-up-a-nested-kvm-guest-for-developing-testing-pci-device-assignment-with-numa/ [2] https://review.opendev.org/c/openstack/nova-specs/+/840310 Implements: blueprint libvirt-viommu-device Change-Id: Ief9c550292788160433a28a7a1c36ba38a6bc849 Signed-off-by: Stephen Finucane --- .../ImageMetaPropsPayload.json | 2 +- doc/source/admin/pci-passthrough.rst | 58 ++ nova/api/validation/extra_specs/hw.py | 16 + nova/exception.py | 10 + nova/notifications/objects/image.py | 3 +- nova/objects/fields.py | 14 + nova/objects/image_meta.py | 8 +- .../test_instance.py | 4 +- .../validation/extra_specs/test_validators.py | 5 + .../objects/test_notification.py | 2 +- nova/tests/unit/objects/test_image_meta.py | 16 + nova/tests/unit/objects/test_objects.py | 2 +- nova/tests/unit/virt/libvirt/test_config.py | 47 ++ nova/tests/unit/virt/libvirt/test_driver.py | 572 +++++++++++++----- nova/virt/libvirt/config.py | 93 ++- nova/virt/libvirt/driver.py | 118 +++- .../guest-iommu-device-4795c3a060aca424.yaml | 21 + 17 files changed, 824 insertions(+), 167 deletions(-) create mode 100644 releasenotes/notes/guest-iommu-device-4795c3a060aca424.yaml diff --git a/doc/notification_samples/common_payloads/ImageMetaPropsPayload.json b/doc/notification_samples/common_payloads/ImageMetaPropsPayload.json index 6aa4d9cbe55f..cdde7d3097af 100644 --- a/doc/notification_samples/common_payloads/ImageMetaPropsPayload.json +++ b/doc/notification_samples/common_payloads/ImageMetaPropsPayload.json @@ -4,5 +4,5 @@ "hw_architecture": "x86_64" }, "nova_object.name": "ImageMetaPropsPayload", - "nova_object.version": "1.11" + "nova_object.version": "1.12" } diff --git a/doc/source/admin/pci-passthrough.rst b/doc/source/admin/pci-passthrough.rst index aecf0c9e3f4a..32287fd7897a 100644 --- a/doc/source/admin/pci-passthrough.rst +++ b/doc/source/admin/pci-passthrough.rst @@ -395,3 +395,61 @@ be added to the resource provider representing the matching PCI devices. It is suggested to use the PCI address of the device instead. For deeper technical details please read the `nova specification. `_ + + +Virtual IOMMU support +--------------------- + +With provided :nova:extra-spec:`hw:viommu_model` flavor extra spec or equivalent +image metadata property ``hw_viommu_model`` and with the guest CPU architecture +and OS allows, we can enable vIOMMU in libvirt driver. + +.. note:: + + Enable vIOMMU might introduce significant performance overhead. + You can see performance comparision table from + `AMD vIOMMU session on KVM Forum 2021`_. + For the above reason, vIOMMU should only be enabled for workflow that + require it. + +.. _`AMD vIOMMU session on KVM Forum 2021`: https://static.sched.com/hosted_files/kvmforum2021/da/vIOMMU%20KVM%20Forum%202021%20-%20v4.pdf + +Here are four possible values allowed for ``hw:viommu_model`` +(and ``hw_viommu_model``): + +**virtio** + Supported on Libvirt since 8.3.0, for Q35 and ARM virt guests. + +**smmuv3** + Supported on Libvirt since 5.5.0, for ARM virt guests. +**intel** + Supported for for Q35 guests. + +**auto** + This option will translate to ``virtio`` if Libvirt supported, + else ``intel`` on X86 (Q35) and ``smmuv3`` on AArch64. + +For the viommu attributes: + +* ``intremap``, ``caching_mode``, and ``iotlb`` + options for viommu (These attributes are driver attributes defined in + `Libvirt IOMMU Domain`_) will direcly enabled. + +* ``eim`` will directly enabled if machine type is Q35. + ``eim`` is driver attribute defined in `Libvirt IOMMU Domain`_. + +.. note:: + + eim(Extended Interrupt Mode) attribute (with possible values on and off) + can be used to configure Extended Interrupt Mode. + A q35 domain with split I/O APIC (as described in hypervisor features), + and both interrupt remapping and EIM turned on for the IOMMU, will be + able to use more than 255 vCPUs. Since 3.4.0 (QEMU/KVM only). + +* ``aw_bits`` attribute can used to set the address width to allow mapping + larger iova addresses in the guest. Since Qemu current supported + values are 39 and 48, we directly set this to larger width (48) + if Libvirt supported. + ``aw_bits`` is driver attribute defined in `Libvirt IOMMU Domain`_. + +.. _`Libvirt IOMMU Domain`: https://libvirt.org/formatdomain.html#iommu-devices diff --git a/nova/api/validation/extra_specs/hw.py b/nova/api/validation/extra_specs/hw.py index 02e8de9cf29f..c0c8f02809ed 100644 --- a/nova/api/validation/extra_specs/hw.py +++ b/nova/api/validation/extra_specs/hw.py @@ -511,6 +511,22 @@ feature_flag_validators = [ ], }, ), + base.ExtraSpecValidator( + name='hw:viommu_model', + description=( + 'This can be used to set model for virtual IOMMU device.' + ), + value={ + 'type': str, + 'enum': [ + 'intel', + 'smmuv3', + 'virtio', + 'auto' + ], + 'description': 'model for vIOMMU', + }, + ), ] ephemeral_encryption_validators = [ diff --git a/nova/exception.py b/nova/exception.py index fe3c12d87072..077c3ef50271 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -207,6 +207,16 @@ class Invalid(NovaException): code = 400 +class InvalidVIOMMUMachineType(Invalid): + msg_fmt = _("vIOMMU is not supported by Current machine type %(mtype)s " + "(Architecture: %(arch)s).") + + +class InvalidVIOMMUArchitecture(Invalid): + msg_fmt = _("vIOMMU required either x86 or AArch64 architecture, " + "but given architecture %(arch)s.") + + class InvalidConfiguration(Invalid): msg_fmt = _("Configuration is Invalid.") diff --git a/nova/notifications/objects/image.py b/nova/notifications/objects/image.py index a408b27eab50..01c86d1cb04e 100644 --- a/nova/notifications/objects/image.py +++ b/nova/notifications/objects/image.py @@ -129,7 +129,8 @@ class ImageMetaPropsPayload(base.NotificationPayloadBase): # Version 1.10: Added 'hw_ephemeral_encryption' and # 'hw_ephemeral_encryption_format' fields # Version 1.11: Added 'hw_locked_memory' field - VERSION = '1.11' + # Version 1.12: Added 'hw_viommu_model' field + VERSION = '1.12' SCHEMA = { k: ('image_meta_props', k) for k in image_meta.ImageMetaProps.fields} diff --git a/nova/objects/fields.py b/nova/objects/fields.py index 2946d2bd990a..cae1ea4a4d73 100644 --- a/nova/objects/fields.py +++ b/nova/objects/fields.py @@ -616,6 +616,16 @@ class VIFModel(BaseNovaEnum): return super(VIFModel, self).coerce(obj, attr, value) +class VIOMMUModel(BaseNovaEnum): + + INTEL = 'intel' + SMMUV3 = 'smmuv3' + VIRTIO = 'virtio' + AUTO = 'auto' + + ALL = (INTEL, SMMUV3, VIRTIO, AUTO) + + class VMMode(BaseNovaEnum): """Represents possible vm modes for instances. @@ -1301,6 +1311,10 @@ class VIFModelField(BaseEnumField): AUTO_TYPE = VIFModel() +class VIOMMUModelField(BaseEnumField): + AUTO_TYPE = VIOMMUModel() + + class VMModeField(BaseEnumField): AUTO_TYPE = VMMode() diff --git a/nova/objects/image_meta.py b/nova/objects/image_meta.py index 0ca8ed571fd4..7927ad257526 100644 --- a/nova/objects/image_meta.py +++ b/nova/objects/image_meta.py @@ -191,14 +191,17 @@ class ImageMetaProps(base.NovaObject): # Version 1.32: Added 'hw_ephemeral_encryption' and # 'hw_ephemeral_encryption_format' fields # Version 1.33: Added 'hw_locked_memory' field + # Version 1.34: Added 'hw_viommu_model' field # NOTE(efried): When bumping this version, the version of # ImageMetaPropsPayload must also be bumped. See its docstring for details. - VERSION = '1.33' + VERSION = '1.34' def obj_make_compatible(self, primitive, target_version): super(ImageMetaProps, self).obj_make_compatible(primitive, target_version) target_version = versionutils.convert_version_to_tuple(target_version) + if target_version < (1, 34): + primitive.pop('hw_viommu_model', None) if target_version < (1, 33): primitive.pop('hw_locked_memory', None) if target_version < (1, 32): @@ -446,6 +449,9 @@ class ImageMetaProps(base.NovaObject): # name of a NIC device model eg virtio, e1000, rtl8139 'hw_vif_model': fields.VIFModelField(), + # name of IOMMU device model eg virtio, intel, smmuv3, or auto + 'hw_viommu_model': fields.VIOMMUModelField(), + # "xen" vs "hvm" 'hw_vm_mode': fields.VMModeField(), diff --git a/nova/tests/functional/notification_sample_tests/test_instance.py b/nova/tests/functional/notification_sample_tests/test_instance.py index e6c88be239d2..f671a8abcabd 100644 --- a/nova/tests/functional/notification_sample_tests/test_instance.py +++ b/nova/tests/functional/notification_sample_tests/test_instance.py @@ -1231,7 +1231,7 @@ class TestInstanceNotificationSample( 'nova_object.data': {}, 'nova_object.name': 'ImageMetaPropsPayload', 'nova_object.namespace': 'nova', - 'nova_object.version': '1.11', + 'nova_object.version': '1.12', }, 'image.size': 58145823, 'image.tags': [], @@ -1327,7 +1327,7 @@ class TestInstanceNotificationSample( 'nova_object.data': {}, 'nova_object.name': 'ImageMetaPropsPayload', 'nova_object.namespace': 'nova', - 'nova_object.version': '1.11', + 'nova_object.version': '1.12', }, 'image.size': 58145823, 'image.tags': [], diff --git a/nova/tests/unit/api/validation/extra_specs/test_validators.py b/nova/tests/unit/api/validation/extra_specs/test_validators.py index dd45f85ff12b..a8911aadadbd 100644 --- a/nova/tests/unit/api/validation/extra_specs/test_validators.py +++ b/nova/tests/unit/api/validation/extra_specs/test_validators.py @@ -74,6 +74,10 @@ class TestValidators(test.NoDBTestCase): ('hw:pci_numa_affinity_policy', 'preferred'), ('hw:pci_numa_affinity_policy', 'socket'), ('hw:cpu_policy', 'mixed'), + ('hw:viommu_model', 'auto'), + ('hw:viommu_model', 'intel'), + ('hw:viommu_model', 'smmuv3'), + ('hw:viommu_model', 'virtio'), ) for key, value in valid_specs: validators.validate(key, value) @@ -92,6 +96,7 @@ class TestValidators(test.NoDBTestCase): ('hw:pci_numa_affinity_policy', 'requird'), ('hw:pci_numa_affinity_policy', 'prefrred'), ('hw:pci_numa_affinity_policy', 'socet'), + ('hw:viommu_model', 'autt'), ) for key, value in invalid_specs: with testtools.ExpectedException(exception.ValidationError): diff --git a/nova/tests/unit/notifications/objects/test_notification.py b/nova/tests/unit/notifications/objects/test_notification.py index 41352f2e48fc..de9e6f276232 100644 --- a/nova/tests/unit/notifications/objects/test_notification.py +++ b/nova/tests/unit/notifications/objects/test_notification.py @@ -386,7 +386,7 @@ notification_object_data = { # ImageMetaProps, so when you see a fail here for that reason, you must # *also* bump the version of ImageMetaPropsPayload. See its docstring for # more information. - 'ImageMetaPropsPayload': '1.11-938809cd33367c52cbc814fb9b6783dc', + 'ImageMetaPropsPayload': '1.12-b9c64832d7772c1973e913bacbe0e8f9', 'InstanceActionNotification': '1.0-a73147b93b520ff0061865849d3dfa56', 'InstanceActionPayload': '1.8-4fa3da9cbf0761f1f700ae578f36dc2f', 'InstanceActionRebuildNotification': diff --git a/nova/tests/unit/objects/test_image_meta.py b/nova/tests/unit/objects/test_image_meta.py index 27d91290ad6f..371f7b101a55 100644 --- a/nova/tests/unit/objects/test_image_meta.py +++ b/nova/tests/unit/objects/test_image_meta.py @@ -538,3 +538,19 @@ class TestImageMetaProps(test.NoDBTestCase): hw_pci_numa_affinity_policy=fields.PCINUMAAffinityPolicy.SOCKET) self.assertRaises(exception.ObjectActionError, obj.obj_to_primitive, '1.27') + + def test_obj_make_compatible_viommu_model(self): + """Check 'hw_viommu_model' compatibility.""" + # assert that 'hw_viommu_model' is supported on a suitably new version + obj = objects.ImageMetaProps( + hw_viommu_model=objects.fields.VIOMMUModel.VIRTIO, + ) + primitive = obj.obj_to_primitive('1.34') + self.assertIn('hw_viommu_model', primitive['nova_object.data']) + self.assertEqual( + objects.fields.VIOMMUModel.VIRTIO, + primitive['nova_object.data']['hw_viommu_model']) + + # and is absent on older versions + primitive = obj.obj_to_primitive('1.33') + self.assertNotIn('hw_viommu_model', primitive['nova_object.data']) diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py index f42b74e13593..951ae00f7dae 100644 --- a/nova/tests/unit/objects/test_objects.py +++ b/nova/tests/unit/objects/test_objects.py @@ -1072,7 +1072,7 @@ object_data = { 'HyperVLiveMigrateData': '1.4-e265780e6acfa631476c8170e8d6fce0', 'IDEDeviceBus': '1.0-29d4c9f27ac44197f01b6ac1b7e16502', 'ImageMeta': '1.8-642d1b2eb3e880a367f37d72dd76162d', - 'ImageMetaProps': '1.33-6b7a29f769e6b8eee3f05832d78c85a2', + 'ImageMetaProps': '1.34-29b3a6b7fe703f36bfd240d914f16c21', 'Instance': '2.7-d187aec68cad2e4d8b8a03a68e4739ce', 'InstanceAction': '1.2-9a5abc87fdd3af46f45731960651efb5', 'InstanceActionEvent': '1.4-5b1f361bd81989f8bb2c20bb7e8a4cb4', diff --git a/nova/tests/unit/virt/libvirt/test_config.py b/nova/tests/unit/virt/libvirt/test_config.py index 4b258a2464a8..6cc67fba54b0 100644 --- a/nova/tests/unit/virt/libvirt/test_config.py +++ b/nova/tests/unit/virt/libvirt/test_config.py @@ -16,6 +16,7 @@ from lxml import etree from oslo_utils.fixture import uuidsentinel as uuids from oslo_utils import units +from nova import exception from nova.objects import fields as obj_fields from nova import test from nova.tests.fixtures import libvirt_data as fake_libvirt_data @@ -70,6 +71,23 @@ class LibvirtConfigTest(LibvirtConfigBaseTest): obj = config.LibvirtConfigObject(root_name="demo") obj.parse_str(inxml) + def test_parse_on_off_str(self): + obj = config.LibvirtConfigObject(root_name="demo") + self.assertTrue(obj.parse_on_off_str('on')) + self.assertFalse(obj.parse_on_off_str('off')) + self.assertFalse(obj.parse_on_off_str(None)) + self.assertRaises(exception.InvalidInput, obj.parse_on_off_str, 'foo') + + def test_get_yes_no_str(self): + obj = config.LibvirtConfigObject(root_name="demo") + self.assertEqual('yes', obj.get_yes_no_str(True)) + self.assertEqual('no', obj.get_yes_no_str(False)) + + def test_get_on_off_str(self): + obj = config.LibvirtConfigObject(root_name="demo") + self.assertEqual('on', obj.get_on_off_str(True)) + self.assertEqual('off', obj.get_on_off_str(False)) + class LibvirtConfigCapsTest(LibvirtConfigBaseTest): @@ -2365,6 +2383,13 @@ class LibvirtConfigGuestFeatureTest(LibvirtConfigBaseTest): xml = obj.to_xml() self.assertXmlEqual(xml, "") + def test_feature_ioapic(self): + obj = config.LibvirtConfigGuestFeatureIOAPIC() + obj.driver = "libvirt" + + xml = obj.to_xml() + self.assertXmlEqual(xml, "") + class LibvirtConfigGuestTest(LibvirtConfigBaseTest): @@ -3993,6 +4018,28 @@ class LibvirtConfigGuestVPMEMTest(LibvirtConfigBaseTest): """) +class LibvirtConfigGuestIOMMUTest(LibvirtConfigBaseTest): + + def test_config_iommu(self): + obj = config.LibvirtConfigGuestIOMMU() + obj.model = "intel" + obj.interrupt_remapping = True + obj.caching_mode = True + obj.aw_bits = 48 + obj.eim = True + obj.iotlb = True + + xml = obj.to_xml() + self.assertXmlEqual( + xml, + """ + + + + """, + ) + + class LibvirtConfigDomainCapsVideoModelsTests(LibvirtConfigBaseTest): def test_parse_video_model(self): diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py index b90d6a2ef6a9..1b349205ccab 100644 --- a/nova/tests/unit/virt/libvirt/test_driver.py +++ b/nova/tests/unit/virt/libvirt/test_driver.py @@ -2566,6 +2566,11 @@ class LibvirtConnTestCase(test.NoDBTestCase, @mock.patch.object(time, "time") def test_get_guest_config(self, time_mock): + """Generate a "standard" guest with minimal configuration. + + This uses i440fx by default since that's our default machine type and + x86 is our default architecture (in our test env, anyway). + """ time_mock.return_value = 1234567.89 drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) @@ -2574,172 +2579,103 @@ class LibvirtConnTestCase(test.NoDBTestCase, test_instance["display_name"] = "purple tomatoes" test_instance['system_metadata']['owner_project_name'] = 'sweetshop' test_instance['system_metadata']['owner_user_name'] = 'cupcake' - - ctxt = context.RequestContext(project_id=123, - project_name="aubergine", - user_id=456, - user_name="pie") - - flavor = objects.Flavor(name='m1.small', - memory_mb=6, - vcpus=28, - root_gb=496, - ephemeral_gb=8128, - swap=33550336, - extra_specs={}) + ctxt = context.RequestContext( + project_id=123, + project_name="aubergine", + user_id=456, + user_name="pie", + ) + flavor = objects.Flavor( + name='m1.small', + memory_mb=6, + vcpus=28, + root_gb=496, + ephemeral_gb=8128, + swap=33550336, + extra_specs={}, + ) instance_ref = objects.Instance(**test_instance) instance_ref.flavor = flavor image_meta = objects.ImageMeta.from_dict(self.test_image_meta) + disk_info = blockinfo.get_disk_info( + CONF.libvirt.virt_type, + instance_ref, + image_meta, + ) - disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, - instance_ref, - image_meta) - - cfg = drvr._get_guest_config(instance_ref, - _fake_network_info(self), - image_meta, disk_info, - context=ctxt) + cfg = drvr._get_guest_config( + instance_ref, + _fake_network_info(self), + image_meta, disk_info, + context=ctxt, + ) self.assertEqual(cfg.uuid, instance_ref["uuid"]) - self.assertEqual(3, len(cfg.features)) - self.assertIsInstance(cfg.features[0], - vconfig.LibvirtConfigGuestFeatureACPI) - self.assertIsInstance(cfg.features[1], - vconfig.LibvirtConfigGuestFeatureAPIC) - self.assertIsInstance( - cfg.features[2], vconfig.LibvirtConfigGuestFeatureVMCoreInfo) self.assertEqual(cfg.memory, 6 * units.Ki) self.assertEqual(cfg.vcpus, 28) self.assertEqual(cfg.os_type, fields.VMMode.HVM) self.assertEqual(cfg.os_boot_dev, ["hd"]) self.assertIsNone(cfg.os_root) + + self.assertEqual(3, len(cfg.features)) + for idx, device_type in enumerate([ + vconfig.LibvirtConfigGuestFeatureACPI, + vconfig.LibvirtConfigGuestFeatureAPIC, + vconfig.LibvirtConfigGuestFeatureVMCoreInfo, + ]): + self.assertIsInstance(cfg.features[idx], device_type) + self.assertEqual(len(cfg.devices), 11) - self.assertIsInstance(cfg.devices[0], - vconfig.LibvirtConfigGuestDisk) - self.assertIsInstance(cfg.devices[1], - vconfig.LibvirtConfigGuestDisk) - self.assertIsInstance(cfg.devices[2], - vconfig.LibvirtConfigGuestDisk) - self.assertIsInstance(cfg.devices[3], - vconfig.LibvirtConfigGuestInterface) - self.assertIsInstance(cfg.devices[4], - vconfig.LibvirtConfigGuestSerial) - self.assertIsInstance(cfg.devices[5], - vconfig.LibvirtConfigGuestGraphics) - self.assertIsInstance(cfg.devices[6], - vconfig.LibvirtConfigGuestVideo) - self.assertIsInstance(cfg.devices[7], - vconfig.LibvirtConfigGuestInput) - self.assertIsInstance(cfg.devices[8], - vconfig.LibvirtConfigGuestRng) - self.assertIsInstance(cfg.devices[9], - vconfig.LibvirtConfigGuestUSBHostController) - self.assertIsInstance(cfg.devices[10], - vconfig.LibvirtConfigMemoryBalloon) + for idx, device_type in enumerate([ + vconfig.LibvirtConfigGuestDisk, + vconfig.LibvirtConfigGuestDisk, + vconfig.LibvirtConfigGuestDisk, + vconfig.LibvirtConfigGuestInterface, + vconfig.LibvirtConfigGuestSerial, + vconfig.LibvirtConfigGuestGraphics, + vconfig.LibvirtConfigGuestVideo, + vconfig.LibvirtConfigGuestInput, + vconfig.LibvirtConfigGuestRng, + vconfig.LibvirtConfigGuestUSBHostController, + vconfig.LibvirtConfigMemoryBalloon, + ]): + self.assertIsInstance(cfg.devices[idx], device_type) + self.assertEqual(len(cfg.metadata), 1) - self.assertIsInstance(cfg.metadata[0], - vconfig.LibvirtConfigGuestMetaNovaInstance) - self.assertEqual(version.version_string_with_package(), - cfg.metadata[0].package) - self.assertEqual("purple tomatoes", - cfg.metadata[0].name) - self.assertEqual(1234567.89, - cfg.metadata[0].creationTime) - self.assertEqual("image", - cfg.metadata[0].roottype) - self.assertEqual(str(instance_ref["image_ref"]), - cfg.metadata[0].rootid) + self.assertIsInstance( + cfg.metadata[0], vconfig.LibvirtConfigGuestMetaNovaInstance) + self.assertEqual( + version.version_string_with_package(), cfg.metadata[0].package) + self.assertEqual("purple tomatoes", cfg.metadata[0].name) + self.assertEqual(1234567.89, cfg.metadata[0].creationTime) + self.assertEqual("image", cfg.metadata[0].roottype) + self.assertEqual( + str(instance_ref["image_ref"]), cfg.metadata[0].rootid) - self.assertIsInstance(cfg.metadata[0].owner, - vconfig.LibvirtConfigGuestMetaNovaOwner) - self.assertEqual("838a72b0-0d54-4827-8fd6-fb1227633ceb", - cfg.metadata[0].owner.userid) - self.assertEqual("cupcake", - cfg.metadata[0].owner.username) - self.assertEqual("fake", - cfg.metadata[0].owner.projectid) - self.assertEqual("sweetshop", - cfg.metadata[0].owner.projectname) - - self.assertIsInstance(cfg.metadata[0].flavor, - vconfig.LibvirtConfigGuestMetaNovaFlavor) - self.assertEqual("m1.small", - cfg.metadata[0].flavor.name) - self.assertEqual(6, - cfg.metadata[0].flavor.memory) - self.assertEqual(28, - cfg.metadata[0].flavor.vcpus) - self.assertEqual(496, - cfg.metadata[0].flavor.disk) - self.assertEqual(8128, - cfg.metadata[0].flavor.ephemeral) - self.assertEqual(33550336, - cfg.metadata[0].flavor.swap) - - @mock.patch.object(host.Host, "_check_machine_type", new=mock.Mock()) - def test_get_guest_config_q35(self): - self.flags(virt_type="kvm", - group='libvirt') - - TEST_AMOUNT_OF_PCIE_SLOTS = 8 - CONF.set_override("num_pcie_ports", TEST_AMOUNT_OF_PCIE_SLOTS, - group='libvirt') - - drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) - instance_ref = objects.Instance(**self.test_instance) - image_meta = objects.ImageMeta.from_dict({ - "disk_format": "raw", - "properties": {"hw_machine_type": - "pc-q35-test"}}) - - disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, - instance_ref, - image_meta) - - cfg = drvr._get_guest_config(instance_ref, - _fake_network_info(self), - image_meta, disk_info) + self.assertIsInstance( + cfg.metadata[0].owner, vconfig.LibvirtConfigGuestMetaNovaOwner) + self.assertEqual( + "838a72b0-0d54-4827-8fd6-fb1227633ceb", + cfg.metadata[0].owner.userid) + self.assertEqual("cupcake", cfg.metadata[0].owner.username) + self.assertEqual("fake", cfg.metadata[0].owner.projectid) + self.assertEqual("sweetshop", cfg.metadata[0].owner.projectname) + self.assertIsInstance( + cfg.metadata[0].flavor, vconfig.LibvirtConfigGuestMetaNovaFlavor) + self.assertEqual("m1.small", cfg.metadata[0].flavor.name) + self.assertEqual(6, cfg.metadata[0].flavor.memory) + self.assertEqual(28, cfg.metadata[0].flavor.vcpus) + self.assertEqual(496, cfg.metadata[0].flavor.disk) + self.assertEqual(8128, cfg.metadata[0].flavor.ephemeral) + self.assertEqual(33550336, cfg.metadata[0].flavor.swap) num_ports = 0 for device in cfg.devices: try: - if (device.root_name == 'controller' and - device.model == 'pcie-root-port'): - num_ports += 1 - except AttributeError: - pass - - self.assertEqual(TEST_AMOUNT_OF_PCIE_SLOTS, num_ports) - - @mock.patch.object(host.Host, "_check_machine_type", new=mock.Mock()) - def test_get_guest_config_pcie_i440fx(self): - self.flags(virt_type="kvm", - group='libvirt') - - TEST_AMOUNT_OF_PCIE_SLOTS = 8 - CONF.set_override("num_pcie_ports", TEST_AMOUNT_OF_PCIE_SLOTS, - group='libvirt') - - drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) - instance_ref = objects.Instance(**self.test_instance) - image_meta = objects.ImageMeta.from_dict({ - "disk_format": "raw", - "properties": {"hw_machine_type": - "pc-i440fx-test"}}) - - disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, - instance_ref, - image_meta) - - cfg = drvr._get_guest_config(instance_ref, - _fake_network_info(self), - image_meta, disk_info) - - num_ports = 0 - for device in cfg.devices: - try: - if (device.root_name == 'controller' and - device.model == 'pcie-root-port'): + if ( + device.root_name == 'controller' and + device.model == 'pcie-root-port' + ): num_ports += 1 except AttributeError: pass @@ -2747,6 +2683,146 @@ class LibvirtConnTestCase(test.NoDBTestCase, # i440fx is not pcie machine so there should be no pcie ports self.assertEqual(0, num_ports) + @mock.patch.object(time, "time") + def test_get_guest_config_no_pcie_ports(self, time_mock): + """Generate a "standard" guest with minimal configuration. + + This uses i440fx by default since that's our default machine type and + x86 is our default architecture (in our test env, anyway). + """ + time_mock.return_value = 1234567.89 + + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + + test_instance = copy.deepcopy(self.test_instance) + test_instance["display_name"] = "purple tomatoes" + test_instance['system_metadata']['owner_project_name'] = 'sweetshop' + test_instance['system_metadata']['owner_user_name'] = 'cupcake' + ctxt = context.RequestContext( + project_id=123, + project_name="aubergine", + user_id=456, + user_name="pie", + ) + flavor = objects.Flavor( + name='m1.small', + memory_mb=6, + vcpus=28, + root_gb=496, + ephemeral_gb=8128, + swap=33550336, + extra_specs={}, + ) + instance_ref = objects.Instance(**test_instance) + instance_ref.flavor = flavor + image_meta = objects.ImageMeta.from_dict(self.test_image_meta) + disk_info = blockinfo.get_disk_info( + CONF.libvirt.virt_type, + instance_ref, + image_meta, + ) + + cfg = drvr._get_guest_config( + instance_ref, + _fake_network_info(self), + image_meta, disk_info, + context=ctxt, + ) + + num_ports = 0 + for device in cfg.devices: + try: + if ( + device.root_name == 'controller' and + device.model == 'pcie-root-port' + ): + num_ports += 1 + except AttributeError: + pass + + # i440fx is not pcie machine so there should be no pcie ports + self.assertEqual(0, num_ports) + + @mock.patch.object(host.Host, "_check_machine_type", new=mock.Mock()) + def test_get_guest_config_q35(self): + """Generate a "q35" guest with minimal configuration. + + This configures an explicit machine type (q35) but defaults to x86 + since this is our default architecture (in our test env, anyway). + """ + self.flags(virt_type="kvm", group='libvirt') + + TEST_AMOUNT_OF_PCIE_SLOTS = 8 + CONF.set_override( + "num_pcie_ports", TEST_AMOUNT_OF_PCIE_SLOTS, + group='libvirt', + ) + + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + instance_ref = objects.Instance(**self.test_instance) + image_meta = objects.ImageMeta.from_dict({ + "disk_format": "raw", + "properties": {"hw_machine_type": "q35"}, + }) + + disk_info = blockinfo.get_disk_info( + CONF.libvirt.virt_type, + instance_ref, + image_meta, + ) + + cfg = drvr._get_guest_config( + instance_ref, + _fake_network_info(self), + image_meta, + disk_info, + ) + + self.assertEqual(3, len(cfg.features)) + for idx, device_type in enumerate([ + vconfig.LibvirtConfigGuestFeatureACPI, + vconfig.LibvirtConfigGuestFeatureAPIC, + vconfig.LibvirtConfigGuestFeatureVMCoreInfo, + ]): + self.assertIsInstance(cfg.features[idx], device_type) + + self.assertEqual(len(cfg.devices), 19) + for idx, device_type in enumerate([ + vconfig.LibvirtConfigGuestDisk, + vconfig.LibvirtConfigGuestDisk, + vconfig.LibvirtConfigGuestInterface, + vconfig.LibvirtConfigGuestSerial, + vconfig.LibvirtConfigGuestGraphics, + vconfig.LibvirtConfigGuestVideo, + vconfig.LibvirtConfigGuestInput, + vconfig.LibvirtConfigGuestRng, + vconfig.LibvirtConfigGuestPCIeRootController, + vconfig.LibvirtConfigGuestPCIeRootPortController, + vconfig.LibvirtConfigGuestPCIeRootPortController, + vconfig.LibvirtConfigGuestPCIeRootPortController, + vconfig.LibvirtConfigGuestPCIeRootPortController, + vconfig.LibvirtConfigGuestPCIeRootPortController, + vconfig.LibvirtConfigGuestPCIeRootPortController, + vconfig.LibvirtConfigGuestPCIeRootPortController, + vconfig.LibvirtConfigGuestPCIeRootPortController, + vconfig.LibvirtConfigGuestUSBHostController, + vconfig.LibvirtConfigMemoryBalloon, + ]): + self.assertIsInstance(cfg.devices[idx], device_type) + + num_ports = 0 + for device in cfg.devices: + try: + if ( + device.root_name == 'controller' and + device.model == 'pcie-root-port' + ): + num_ports += 1 + except AttributeError: + pass + + self.assertEqual(TEST_AMOUNT_OF_PCIE_SLOTS, num_ports) + @mock.patch.object(host.Host, "_check_machine_type", new=mock.Mock()) @mock.patch('nova.virt.libvirt.utils.get_default_machine_type', new=mock.Mock(return_value='config-machine_type')) @@ -8436,6 +8512,206 @@ class LibvirtConnTestCase(test.NoDBTestCase, self.assertEqual(conf.cpu.cores, 2) self.assertEqual(conf.cpu.threads, 1) + def test_get_guest_iommu_not_enabled(self): + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + + test_instance = _create_test_instance() + instance_ref = objects.Instance(**test_instance) + image_meta = objects.ImageMeta.from_dict(self.test_image_meta) + disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, + instance_ref, + image_meta) + + cfg = drvr._get_guest_config(instance_ref, [], + image_meta, disk_info) + for device in cfg.devices: + self.assertNotEqual('iommu', device.root_name) + + def test_get_guest_iommu_config_model(self): + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + image_meta = objects.ImageMeta.from_dict({ + "disk_format": "raw", + "properties": {"hw_machine_type": "q35"}, + }) + + extra_specs = { + "hw:viommu_model": 'intel', + } + test_instance = _create_test_instance() + test_instance["flavor"]["extra_specs"] = extra_specs + instance_ref = objects.Instance(**test_instance) + disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, + instance_ref, + image_meta) + + cfg = drvr._get_guest_config(instance_ref, [], + image_meta, disk_info) + count = 0 + for device in cfg.devices: + if device.root_name == 'iommu': + count += 1 + self.assertIsInstance(device, + vconfig.LibvirtConfigGuestIOMMU) + self.assertEqual('intel', device.model) + self.assertFalse(hasattr(device, "aw_bits")) + self.assertTrue(device.interrupt_remapping) + self.assertTrue(device.caching_mode) + self.assertTrue(device.eim) + self.assertTrue(device.iotlb) + + self.assertEqual(1, count) + self.assertEqual('q35', cfg.os_mach_type) + + @mock.patch.object(host.Host, 'has_min_version', return_value=True) + def test_get_guest_iommu_config_model_auto(self, has_min_version): + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + image_meta = objects.ImageMeta.from_dict({ + "disk_format": "raw", + "properties": {"hw_machine_type": "q35"}, + }) + + extra_specs = { + "hw:viommu_model": 'auto', + } + test_instance = _create_test_instance() + test_instance["flavor"]["extra_specs"] = extra_specs + instance_ref = objects.Instance(**test_instance) + disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, + instance_ref, + image_meta) + + cfg = drvr._get_guest_config(instance_ref, [], + image_meta, disk_info) + count = 0 + for device in cfg.devices: + if device.root_name == 'iommu': + count += 1 + self.assertIsInstance(device, + vconfig.LibvirtConfigGuestIOMMU) + self.assertEqual('virtio', device.model) + self.assertEqual(48, device.aw_bits) + self.assertTrue(device.interrupt_remapping) + self.assertTrue(device.caching_mode) + self.assertTrue(device.eim) + self.assertTrue(device.iotlb) + + self.assertEqual(1, count) + self.assertEqual('q35', cfg.os_mach_type) + + @mock.patch.object(host.Host, 'has_min_version', return_value=False) + def test_get_guest_iommu_config_model_auto_intel(self, has_min_version): + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + image_meta = objects.ImageMeta.from_dict({ + "disk_format": "raw", + "properties": {"hw_machine_type": "q35"}, + }) + + extra_specs = { + "hw:viommu_model": 'auto', + } + test_instance = _create_test_instance() + test_instance["flavor"]["extra_specs"] = extra_specs + instance_ref = objects.Instance(**test_instance) + disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, + instance_ref, + image_meta) + + cfg = drvr._get_guest_config(instance_ref, [], + image_meta, disk_info) + count = 0 + for device in cfg.devices: + if device.root_name == 'iommu': + count += 1 + self.assertIsInstance(device, + vconfig.LibvirtConfigGuestIOMMU) + self.assertEqual('intel', device.model) + self.assertTrue(device.interrupt_remapping) + self.assertTrue(device.caching_mode) + self.assertTrue(device.eim) + self.assertTrue(device.iotlb) + + self.assertEqual(1, count) + self.assertEqual('q35', cfg.os_mach_type) + + @mock.patch.object(host.Host, 'has_min_version', return_value=False) + def test_get_guest_iommu_config_model_auto_aarch64(self, has_min_version): + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + + image_meta = objects.ImageMeta.from_dict({ + "disk_format": "raw", + "properties": {"hw_viommu_model": 'auto', + "hw_architecture": fields.Architecture.AARCH64, + "hw_machine_type": "virt"}, + }) + extra_specs = { + "hw:viommu_model": 'auto', + } + test_instance = _create_test_instance() + test_instance["flavor"]["extra_specs"] = extra_specs + instance_ref = objects.Instance(**test_instance) + disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, + instance_ref, + image_meta) + + cfg = drvr._get_guest_config(instance_ref, [], + image_meta, disk_info) + count = 0 + for device in cfg.devices: + if device.root_name == 'iommu': + count += 1 + self.assertIsInstance(device, + vconfig.LibvirtConfigGuestIOMMU) + self.assertEqual('smmuv3', device.model) + self.assertFalse(hasattr(device, "aw_bits")) + self.assertTrue(device.interrupt_remapping) + self.assertTrue(device.caching_mode) + self.assertFalse(device.eim) + self.assertTrue(device.iotlb) + self.assertEqual(1, count) + + def test_get_guest_iommu_config_not_support_machine_type(self): + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + image_meta = objects.ImageMeta.from_dict({ + "disk_format": "raw", + "properties": {"hw_machine_type": "pc-i440fx-2.11"}, + }) + extra_specs = { + "hw:viommu_model": 'auto', + } + test_instance = _create_test_instance() + test_instance["flavor"]["extra_specs"] = extra_specs + instance_ref = objects.Instance(**test_instance) + disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, + instance_ref, + image_meta) + + self.assertRaises( + exception.InvalidVIOMMUMachineType, drvr._get_guest_config, + instance_ref, [], image_meta, disk_info + ) + + def test_get_guest_iommu_config_not_support_architecture(self): + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + image_meta = objects.ImageMeta.from_dict({ + "disk_format": "raw", + "properties": {"hw_architecture": fields.Architecture.PPC64LE, + "hw_machine_type": "pc-i440fx-2.11"}, + }) + extra_specs = { + "hw:viommu_model": 'auto', + } + test_instance = _create_test_instance() + test_instance["flavor"]["extra_specs"] = extra_specs + instance_ref = objects.Instance(**test_instance) + disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, + instance_ref, + image_meta) + + self.assertRaises( + exception.InvalidVIOMMUArchitecture, drvr._get_guest_config, + instance_ref, [], image_meta, disk_info + ) + def test_get_guest_memory_balloon_config_by_default(self): drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) instance_ref = objects.Instance(**self.test_instance) diff --git a/nova/virt/libvirt/config.py b/nova/virt/libvirt/config.py index 07f22c9b918a..3d91c325c397 100644 --- a/nova/virt/libvirt/config.py +++ b/nova/virt/libvirt/config.py @@ -24,6 +24,7 @@ helpers for populating up config object instances. """ import time +import typing as ty from collections import OrderedDict from lxml import etree @@ -32,6 +33,7 @@ from oslo_utils import units from nova import exception from nova.i18n import _ +from nova.objects import fields from nova.pci import utils as pci_utils from nova.virt import hardware @@ -66,9 +68,6 @@ class LibvirtConfigObject(object): child.text = str(value) return child - def get_yes_no_str(self, value): - return 'yes' if value else 'no' - def format_dom(self): return self._new_node(self.root_name) @@ -87,6 +86,25 @@ class LibvirtConfigObject(object): pretty_print=pretty_print) return xml_str + @classmethod + def parse_on_off_str(self, value: ty.Optional[str]) -> bool: + if value is not None and value not in ('on', 'off'): + msg = _( + "Element should contain either 'on' or 'off'; " + "found: '%(value)s'" + ) + raise exception.InvalidInput(msg % {'value': value}) + + return value == 'on' + + @classmethod + def get_yes_no_str(self, value: bool) -> str: + return 'yes' if value else 'no' + + @classmethod + def get_on_off_str(self, value: bool) -> str: + return 'on' if value else 'off' + def __repr__(self): return self.to_xml(pretty_print=False) @@ -2735,6 +2753,18 @@ class LibvirtConfigGuestFeaturePMU(LibvirtConfigGuestFeature): return root +class LibvirtConfigGuestFeatureIOAPIC(LibvirtConfigGuestFeature): + + def __init__(self, **kwargs): + super().__init__("ioapic", **kwargs) + self.driver = "qemu" + + def format_dom(self): + root = super().format_dom() + root.set('driver', self.driver) + return root + + class LibvirtConfigGuestFeatureHyperV(LibvirtConfigGuestFeature): # QEMU requires at least this value to be set @@ -3090,6 +3120,7 @@ class LibvirtConfigGuest(LibvirtConfigObject): # LibvirtConfigGuestGidMap # LibvirtConfigGuestCPU # LibvirtConfigGuestVPMEM + # LibvirtConfigGuestIOMMU for c in xmldoc: if c.tag == 'devices': for d in c: @@ -3117,6 +3148,10 @@ class LibvirtConfigGuest(LibvirtConfigObject): obj = LibvirtConfigGuestVPMEM() obj.parse_dom(d) self.devices.append(obj) + elif d.tag == 'iommu': + obj = LibvirtConfigGuestIOMMU() + obj.parse_dom(d) + self.devices.append(obj) if c.tag == 'idmap': for idmap in c: obj = None @@ -3141,7 +3176,10 @@ class LibvirtConfigGuest(LibvirtConfigObject): else: self._parse_basic_props(c) - def add_device(self, dev): + def add_feature(self, dev: LibvirtConfigGuestFeature) -> None: + self.features.append(dev) + + def add_device(self, dev: LibvirtConfigGuestDevice) -> None: self.devices.append(dev) def add_perf_event(self, event): @@ -3680,6 +3718,53 @@ class LibvirtConfigGuestVPMEM(LibvirtConfigGuestDevice): self.target_size = sub.text +class LibvirtConfigGuestIOMMU(LibvirtConfigGuestDevice): + """https://libvirt.org/formatdomain.html#iommu-devices""" + + def __init__(self, **kwargs): + super().__init__(root_name="iommu", **kwargs) + + self.model: str = fields.VIOMMUModel.AUTO + self.interrupt_remapping: bool = False + self.caching_mode: bool = False + self.eim: bool = False + self.iotlb: bool = False + + def format_dom(self): + iommu = super().format_dom() + iommu.set("model", self.model) + + driver = etree.Element("driver") + driver.set("intremap", self.get_on_off_str(self.interrupt_remapping)) + driver.set("caching_mode", self.get_on_off_str(self.caching_mode)) + + # Set aw_bits to None when the Libvirt version not satisfy + # MIN_LIBVIRT_VIOMMU_AW_BITS in driver. When it's None, means it's not + # supported to have aw_bits. + if hasattr(self, "aw_bits"): + driver.set("aw_bits", str(self.aw_bits)) + driver.set("eim", self.get_on_off_str(self.eim)) + driver.set("iotlb", self.get_on_off_str(self.iotlb)) + iommu.append(driver) + + return iommu + + def parse_dom(self, xmldoc): + super().parse_dom(xmldoc) + self.model = xmldoc.get("model") + + driver = xmldoc.find("./driver") + if driver: + self.interrupt_remapping = self.parse_on_off_str( + driver.get("intremap")) + self.caching_mode = self.parse_on_off_str( + driver.get("caching_mode")) + if driver.get("aw_bits") is not None: + self.aw_bits = int(driver.get("aw_bits")) + self.iotlb = self.parse_on_off_str(driver.get("iotlb")) + self.eim = self.parse_on_off_str(driver.get("eim")) + + class LibvirtConfigGuestMetaNovaPorts(LibvirtConfigObject): def __init__(self, ports=None): diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 925c98aa88ef..cf3c4464e8b3 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -221,6 +221,12 @@ MIN_QEMU_VERSION = (4, 2, 0) NEXT_MIN_LIBVIRT_VERSION = (7, 0, 0) NEXT_MIN_QEMU_VERSION = (5, 2, 0) +# vIOMMU driver attribute aw_bits minimal support version. +MIN_LIBVIRT_VIOMMU_AW_BITS = (6, 5, 0) + +# vIOMMU model value `virtio` minimal support version +MIN_LIBVIRT_VIOMMU_VIRTIO_MODEL = (8, 3, 0) + MIN_LIBVIRT_AARCH64_CPU_COMPARE = (6, 9, 0) # Virtuozzo driver support @@ -6134,9 +6140,9 @@ class LibvirtDriver(driver.ComputeDriver): image_meta.properties.get('img_hide_hypervisor_id')) if CONF.libvirt.virt_type in ('qemu', 'kvm'): - guest.features.append(vconfig.LibvirtConfigGuestFeatureACPI()) + guest.add_feature(vconfig.LibvirtConfigGuestFeatureACPI()) if not CONF.workarounds.libvirt_disable_apic: - guest.features.append(vconfig.LibvirtConfigGuestFeatureAPIC()) + guest.add_feature(vconfig.LibvirtConfigGuestFeatureAPIC()) if CONF.libvirt.virt_type in ('qemu', 'kvm') and os_type == 'windows': hv = vconfig.LibvirtConfigGuestFeatureHyperV() @@ -6180,16 +6186,16 @@ class LibvirtDriver(driver.ComputeDriver): fields.Architecture.I686, fields.Architecture.X86_64, fields.Architecture.AARCH64, ): - guest.features.append( + guest.add_feature( vconfig.LibvirtConfigGuestFeatureVMCoreInfo()) if hide_hypervisor_id: - guest.features.append( + guest.add_feature( vconfig.LibvirtConfigGuestFeatureKvmHidden()) pmu = hardware.get_pmu_constraint(flavor, image_meta) if pmu is not None: - guest.features.append( + guest.add_feature( vconfig.LibvirtConfigGuestFeaturePMU(pmu)) def _check_number_of_serial_console(self, num_ports): @@ -6671,18 +6677,26 @@ class LibvirtDriver(driver.ComputeDriver): self._create_consoles_qemu_kvm( guest_cfg, instance, flavor, image_meta) - def _is_mipsel_guest(self, image_meta): + def _is_mipsel_guest(self, image_meta: 'objects.ImageMeta') -> bool: archs = (fields.Architecture.MIPSEL, fields.Architecture.MIPS64EL) return self._check_emulation_arch(image_meta) in archs - def _is_s390x_guest(self, image_meta): + def _is_s390x_guest(self, image_meta: 'objects.ImageMeta') -> bool: archs = (fields.Architecture.S390, fields.Architecture.S390X) return self._check_emulation_arch(image_meta) in archs - def _is_ppc64_guest(self, image_meta): + def _is_ppc64_guest(self, image_meta: 'objects.ImageMeta') -> bool: archs = (fields.Architecture.PPC64, fields.Architecture.PPC64LE) return self._check_emulation_arch(image_meta) in archs + def _is_aarch64_guest(self, image_meta: 'objects.ImageMeta') -> bool: + arch = fields.Architecture.AARCH64 + return self._check_emulation_arch(image_meta) == arch + + def _is_x86_guest(self, image_meta: 'objects.ImageMeta') -> bool: + archs = (fields.Architecture.I686, fields.Architecture.X86_64) + return self._check_emulation_arch(image_meta) in archs + def _create_consoles_qemu_kvm(self, guest_cfg, instance, flavor, image_meta): char_dev_cls = vconfig.LibvirtConfigGuestSerial @@ -7060,6 +7074,8 @@ class LibvirtDriver(driver.ComputeDriver): if vpmems: self._guest_add_vpmems(guest, vpmems) + self._guest_add_iommu_device(guest, image_meta, flavor) + return guest def _get_ordered_vpmems(self, instance, flavor): @@ -7365,6 +7381,92 @@ class LibvirtDriver(driver.ComputeDriver): # returned for unit testing purposes return keyboard + def _get_iommu_model( + self, + guest: vconfig.LibvirtConfigGuest, + image_meta: 'objects.ImageMeta', + flavor: 'objects.Flavor', + ) -> ty.Optional[str]: + model = flavor.extra_specs.get( + 'hw:viommu_model') or image_meta.properties.get( + 'hw_viommu_model') + if not model: + return None + + is_x86 = self._is_x86_guest(image_meta) + is_aarch64 = self._is_aarch64_guest(image_meta) + + if is_x86: + if guest.os_mach_type is not None and not ( + 'q35' in guest.os_mach_type + ): + arch = self._check_emulation_arch(image_meta) + mtype = guest.os_mach_type if ( + guest.os_mach_type is not None + ) else "unknown" + raise exception.InvalidVIOMMUMachineType( + mtype=mtype, arch=arch) + elif is_aarch64: + if guest.os_mach_type is not None and not ( + 'virt' in guest.os_mach_type + ): + arch = self._check_emulation_arch(image_meta) + mtype = guest.os_mach_type if ( + guest.os_mach_type is not None + ) else "unknown" + raise exception.InvalidVIOMMUMachineType( + mtype=mtype, arch=arch) + else: + raise exception.InvalidVIOMMUArchitecture( + arch=self._check_emulation_arch(image_meta)) + + if model == fields.VIOMMUModel.AUTO: + if self._host.has_min_version(MIN_LIBVIRT_VIOMMU_VIRTIO_MODEL): + model = fields.VIOMMUModel.VIRTIO + elif self._is_x86_guest(image_meta) and ( + guest.os_mach_type is not None and 'q35' in guest.os_mach_type + ): + model = fields.VIOMMUModel.INTEL + else: + # AArch64 + model = fields.VIOMMUModel.SMMUV3 + return model + + def _guest_add_iommu_device( + self, + guest: vconfig.LibvirtConfigGuest, + image_meta: 'objects.ImageMeta', + flavor: 'objects.Flavor', + ) -> None: + """Add a virtual IOMMU device to allow e.g. vfio-pci usage.""" + if CONF.libvirt.virt_type not in ('qemu', 'kvm'): + # vIOMMU requires QEMU + return + + iommu = vconfig.LibvirtConfigGuestIOMMU() + + iommu.model = self._get_iommu_model(guest, image_meta, flavor) + if iommu.model is None: + return + + iommu.interrupt_remapping = True + iommu.caching_mode = True + iommu.iotlb = True + + # As Qemu supported values are 39 and 48, we set this to + # larger width (48) by default and will not exposed to end user. + if self._host.has_min_version(MIN_LIBVIRT_VIOMMU_AW_BITS): + iommu.aw_bits = 48 + + if guest.os_mach_type is not None and 'q35' in guest.os_mach_type: + iommu.eim = True + else: + iommu.eim = False + guest.add_device(iommu) + + ioapic = vconfig.LibvirtConfigGuestFeatureIOAPIC() + guest.add_feature(ioapic) + def _get_guest_xml(self, context, instance, network_info, disk_info, image_meta, rescue=None, block_device_info=None, diff --git a/releasenotes/notes/guest-iommu-device-4795c3a060aca424.yaml b/releasenotes/notes/guest-iommu-device-4795c3a060aca424.yaml new file mode 100644 index 000000000000..314c2c0ffe7c --- /dev/null +++ b/releasenotes/notes/guest-iommu-device-4795c3a060aca424.yaml @@ -0,0 +1,21 @@ +--- +features: + - | + The Libvirt driver can now add a virtual IOMMU device + to all created guests, when running on an x86 host and using the Q35 + machine type or on AArch64. + + To enable this, provide `hw:viommu_model` in flavor extra + spec or equivalent image metadata property `hw_viommu_model` and with the + guest CPU architecture and OS allows, we will enable viommu in Libvirt + driver. Support values intel|smmuv3|virtio|auto. Default to ``auto``. + Which ``auto`` will automatically select ``virtio`` if Libvirt supports it, + else ``intel`` on X86 (Q35) and ``smmuv3`` on AArch64. + vIOMMU config will raise invalid exception if the guest architecture is + neither X86 (Q35) or AArch64. + + Note that, enable vIOMMU might introduce significant performance overhead. + You can see performance comparision table from + `AMD vIOMMU session on KVM Forum 2021`_. + For above reason, vIOMMU should only be enable for workflow that require it. + .. _`AMD vIOMMU session on KVM Forum 2021`: https://static.sched.com/hosted_files/kvmforum2021/da/vIOMMU%20KVM%20Forum%202021%20-%20v4.pdf