From 572c2b18e27f6fcbbd4a1f416b0ec21098b3ba74 Mon Sep 17 00:00:00 2001 From: Sean Mooney Date: Wed, 3 Mar 2021 00:49:22 +0000 Subject: [PATCH] Add locked_memory extra spec and image property This change adds a new hw:locked_memory extra spec and hw_locked_memory image property to contol preventing guest memory from swapping. This change adds docs and extend the flavor validators for the new extra spec. Also add new image property. Blueprint: libvirt-viommu-device Change-Id: Id3779594f0078a5045031aded2ed68ee4301abbd --- .../ImageMetaPropsPayload.json | 2 +- doc/source/admin/libvirt-misc.rst | 30 ++++++++++++ nova/api/validation/extra_specs/hw.py | 12 +++++ nova/exception.py | 11 +++++ nova/notifications/objects/image.py | 3 +- nova/objects/image_meta.py | 9 +++- .../test_instance.py | 4 +- .../objects/test_notification.py | 2 +- nova/tests/unit/objects/test_image_meta.py | 24 ++++++++++ nova/tests/unit/objects/test_objects.py | 2 +- nova/tests/unit/virt/libvirt/test_driver.py | 35 ++++++++++++++ nova/tests/unit/virt/test_hardware.py | 48 +++++++++++++++++++ nova/virt/hardware.py | 44 +++++++++++++++++ nova/virt/libvirt/driver.py | 5 ++ ...locked_memory_option-b68a031779366828.yaml | 13 +++++ 15 files changed, 237 insertions(+), 7 deletions(-) create mode 100644 releasenotes/notes/new_locked_memory_option-b68a031779366828.yaml diff --git a/doc/notification_samples/common_payloads/ImageMetaPropsPayload.json b/doc/notification_samples/common_payloads/ImageMetaPropsPayload.json index c4af49022f2f..6aa4d9cbe55f 100644 --- a/doc/notification_samples/common_payloads/ImageMetaPropsPayload.json +++ b/doc/notification_samples/common_payloads/ImageMetaPropsPayload.json @@ -4,5 +4,5 @@ "hw_architecture": "x86_64" }, "nova_object.name": "ImageMetaPropsPayload", - "nova_object.version": "1.10" + "nova_object.version": "1.11" } diff --git a/doc/source/admin/libvirt-misc.rst b/doc/source/admin/libvirt-misc.rst index 87dbe18ea47f..cf5c10c64e08 100644 --- a/doc/source/admin/libvirt-misc.rst +++ b/doc/source/admin/libvirt-misc.rst @@ -138,3 +138,33 @@ For example, to hide your signature from the guest OS, run: .. code:: console $ openstack flavor set $FLAVOR --property hw:hide_hypervisor_id=true + + +.. _extra-spec-locked_memory: + +Locked memory allocation +------------------------ + +.. versionadded:: 26.0.0 (Zed) + +Locking memory marks the guest memory allocations as unmovable and +unswappable. It is implicitly enabled in a number of cases such as SEV or +realtime guests but can also be enabled explictly using the +``hw:locked_memory`` extra spec (or use ``hw_locked_memory`` image property). +``hw:locked_memory`` (also ``hw_locked_memory`` image property) accept +boolean values in string format like 'true' or 'false' value. +It will raise `FlavorImageLockedMemoryConflict` exception if both flavor and +image property are specified but with different boolean values. +This will only be allowed if you have also set ``hw:mem_page_size``, +so we can ensure that the scheduler can actually account for this correctly +and prevent out of memory events. Otherwise, will raise `LockMemoryForbidden` +exception. + +.. code:: console + + $ openstack flavor set FLAVOR-NAME \ + --property hw:locked_memory=BOOLEAN_VALUE + +.. note:: + + This is currently only supported by the libvirt driver. diff --git a/nova/api/validation/extra_specs/hw.py b/nova/api/validation/extra_specs/hw.py index bb23e7ce8e3d..02e8de9cf29f 100644 --- a/nova/api/validation/extra_specs/hw.py +++ b/nova/api/validation/extra_specs/hw.py @@ -163,6 +163,18 @@ hugepage_validators = [ 'pattern': r'(large|small|any|\d+([kKMGT]i?)?(b|bit|B)?)', }, ), + base.ExtraSpecValidator( + name='hw:locked_memory', + description=( + 'Determine if **guest** (instance) memory should be locked ' + 'preventing swaping. This is required in rare cases for device ' + 'DMA transfers. Only supported by the libvirt virt driver.' + ), + value={ + 'type': bool, + 'description': 'Whether to lock **guest** (instance) memory.', + }, + ), ] numa_validators = [ diff --git a/nova/exception.py b/nova/exception.py index eedf2a4e7d55..8a0fb5aff7fe 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -1861,6 +1861,17 @@ class MemoryPageSizeNotSupported(Invalid): msg_fmt = _("Page size %(pagesize)s is not supported by the host.") +class LockMemoryForbidden(Forbidden): + msg_fmt = _("locked_memory value in image or flavor is forbidden when " + "mem_page_size is not set.") + + +class FlavorImageLockedMemoryConflict(NovaException): + msg_fmt = _("locked_memory value in image (%(image)s) and flavor " + "(%(flavor)s) conflict. A consistent value is expected if " + "both specified.") + + class CPUPinningInvalid(Invalid): msg_fmt = _("CPU set to pin %(requested)s must be a subset of " "free CPU set %(available)s") diff --git a/nova/notifications/objects/image.py b/nova/notifications/objects/image.py index 6f6e3b7c0fbc..a408b27eab50 100644 --- a/nova/notifications/objects/image.py +++ b/nova/notifications/objects/image.py @@ -128,7 +128,8 @@ class ImageMetaPropsPayload(base.NotificationPayloadBase): # Version 1.9: Added 'hw_emulation_architecture' field # Version 1.10: Added 'hw_ephemeral_encryption' and # 'hw_ephemeral_encryption_format' fields - VERSION = '1.10' + # Version 1.11: Added 'hw_locked_memory' field + VERSION = '1.11' SCHEMA = { k: ('image_meta_props', k) for k in image_meta.ImageMetaProps.fields} diff --git a/nova/objects/image_meta.py b/nova/objects/image_meta.py index f17f145dafc4..0ca8ed571fd4 100644 --- a/nova/objects/image_meta.py +++ b/nova/objects/image_meta.py @@ -190,14 +190,17 @@ class ImageMetaProps(base.NovaObject): # Version 1.31: Added 'hw_emulation_architecture' field # Version 1.32: Added 'hw_ephemeral_encryption' and # 'hw_ephemeral_encryption_format' fields + # Version 1.33: Added 'hw_locked_memory' field # NOTE(efried): When bumping this version, the version of # ImageMetaPropsPayload must also be bumped. See its docstring for details. - VERSION = '1.32' + VERSION = '1.33' def obj_make_compatible(self, primitive, target_version): super(ImageMetaProps, self).obj_make_compatible(primitive, target_version) target_version = versionutils.convert_version_to_tuple(target_version) + if target_version < (1, 33): + primitive.pop('hw_locked_memory', None) if target_version < (1, 32): primitive.pop('hw_ephemeral_encryption', None) primitive.pop('hw_ephemeral_encryption_format', None) @@ -368,6 +371,10 @@ class ImageMetaProps(base.NovaObject): # image with a network boot image 'hw_ipxe_boot': fields.FlexibleBooleanField(), + # string - make sure ``locked`` element is present in the + # ``memoryBacking``. + 'hw_locked_memory': fields.FlexibleBooleanField(), + # There are sooooooooooo many possible machine types in # QEMU - several new ones with each new release - that it # is not practical to enumerate them all. So we use a free diff --git a/nova/tests/functional/notification_sample_tests/test_instance.py b/nova/tests/functional/notification_sample_tests/test_instance.py index 84c7246f6708..e6c88be239d2 100644 --- a/nova/tests/functional/notification_sample_tests/test_instance.py +++ b/nova/tests/functional/notification_sample_tests/test_instance.py @@ -1231,7 +1231,7 @@ class TestInstanceNotificationSample( 'nova_object.data': {}, 'nova_object.name': 'ImageMetaPropsPayload', 'nova_object.namespace': 'nova', - 'nova_object.version': '1.10', + 'nova_object.version': '1.11', }, 'image.size': 58145823, 'image.tags': [], @@ -1327,7 +1327,7 @@ class TestInstanceNotificationSample( 'nova_object.data': {}, 'nova_object.name': 'ImageMetaPropsPayload', 'nova_object.namespace': 'nova', - 'nova_object.version': '1.10', + 'nova_object.version': '1.11', }, 'image.size': 58145823, 'image.tags': [], diff --git a/nova/tests/unit/notifications/objects/test_notification.py b/nova/tests/unit/notifications/objects/test_notification.py index 1fddd2604525..41352f2e48fc 100644 --- a/nova/tests/unit/notifications/objects/test_notification.py +++ b/nova/tests/unit/notifications/objects/test_notification.py @@ -386,7 +386,7 @@ notification_object_data = { # ImageMetaProps, so when you see a fail here for that reason, you must # *also* bump the version of ImageMetaPropsPayload. See its docstring for # more information. - 'ImageMetaPropsPayload': '1.10-44cf0030dc94a1a60ba7a0e222e854d6', + 'ImageMetaPropsPayload': '1.11-938809cd33367c52cbc814fb9b6783dc', 'InstanceActionNotification': '1.0-a73147b93b520ff0061865849d3dfa56', 'InstanceActionPayload': '1.8-4fa3da9cbf0761f1f700ae578f36dc2f', 'InstanceActionRebuildNotification': diff --git a/nova/tests/unit/objects/test_image_meta.py b/nova/tests/unit/objects/test_image_meta.py index e47f653ba2d1..27d91290ad6f 100644 --- a/nova/tests/unit/objects/test_image_meta.py +++ b/nova/tests/unit/objects/test_image_meta.py @@ -108,6 +108,7 @@ class TestImageMetaProps(test.NoDBTestCase): 'hw_video_model': 'vga', 'hw_video_ram': '512', 'hw_qemu_guest_agent': 'yes', + 'hw_locked_memory': 'true', 'trait:CUSTOM_TRUSTED': 'required', # Fill sane values for the rest here } @@ -116,6 +117,7 @@ class TestImageMetaProps(test.NoDBTestCase): self.assertEqual('vga', virtprops.hw_video_model) self.assertEqual(512, virtprops.hw_video_ram) self.assertTrue(virtprops.hw_qemu_guest_agent) + self.assertTrue(virtprops.hw_locked_memory) self.assertIsNotNone(virtprops.traits_required) self.assertIn('CUSTOM_TRUSTED', virtprops.traits_required) @@ -285,6 +287,28 @@ class TestImageMetaProps(test.NoDBTestCase): self.assertEqual([set([0, 1, 2, 3])], virtprops.hw_numa_cpus) + def test_locked_memory_prop(self): + props = {'hw_locked_memory': 'true'} + virtprops = objects.ImageMetaProps.from_dict(props) + self.assertTrue(virtprops.hw_locked_memory) + + def test_obj_make_compatible_hw_locked_memory(self): + """Check 'hw_locked_memory' compatibility.""" + # assert that 'hw_locked_memory' is supported + # on a suitably new version + obj = objects.ImageMetaProps( + hw_locked_memory='true', + ) + primitive = obj.obj_to_primitive('1.33') + self.assertIn('hw_locked_memory', + primitive['nova_object.data']) + self.assertTrue(primitive['nova_object.data']['hw_locked_memory']) + + # and is absent on older versions + primitive = obj.obj_to_primitive('1.32') + self.assertNotIn('hw_locked_memory', + primitive['nova_object.data']) + def test_get_unnumbered_trait_fields(self): """Tests that only valid un-numbered required traits are parsed from the properties. diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py index 3f5ce8700abb..821ab25be638 100644 --- a/nova/tests/unit/objects/test_objects.py +++ b/nova/tests/unit/objects/test_objects.py @@ -1072,7 +1072,7 @@ object_data = { 'HyperVLiveMigrateData': '1.4-e265780e6acfa631476c8170e8d6fce0', 'IDEDeviceBus': '1.0-29d4c9f27ac44197f01b6ac1b7e16502', 'ImageMeta': '1.8-642d1b2eb3e880a367f37d72dd76162d', - 'ImageMetaProps': '1.32-4967d35948af08b710b8b861f3fff0f9', + 'ImageMetaProps': '1.33-6b7a29f769e6b8eee3f05832d78c85a2', 'Instance': '2.7-d187aec68cad2e4d8b8a03a68e4739ce', 'InstanceAction': '1.2-9a5abc87fdd3af46f45731960651efb5', 'InstanceActionEvent': '1.4-5b1f361bd81989f8bb2c20bb7e8a4cb4', diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py index 386e8c0cbeb1..0a8d0314449e 100644 --- a/nova/tests/unit/virt/libvirt/test_driver.py +++ b/nova/tests/unit/virt/libvirt/test_driver.py @@ -3131,6 +3131,41 @@ class LibvirtConnTestCase(test.NoDBTestCase, self.assertTrue(membacking.locked) self.assertFalse(membacking.sharedpages) + def test_get_guest_memory_backing_config_locked_flavor(self): + extra_specs = { + "hw:locked_memory": "True", + "hw:mem_page_size": 1000, + } + flavor = objects.Flavor( + name='m1.small', memory_mb=6, vcpus=28, root_gb=496, + ephemeral_gb=8128, swap=33550336, extra_specs=extra_specs) + image_meta = objects.ImageMeta.from_dict(self.test_image_meta) + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + membacking = drvr._get_guest_memory_backing_config( + None, None, flavor, image_meta) + self.assertTrue(membacking.locked) + + def test_get_guest_memory_backing_config_locked_image_meta(self): + extra_specs = {} + flavor = objects.Flavor( + name='m1.small', + memory_mb=6, + vcpus=28, + root_gb=496, + ephemeral_gb=8128, + swap=33550336, + extra_specs=extra_specs) + image_meta = objects.ImageMeta.from_dict({ + "disk_format": "raw", + "properties": { + "hw_locked_memory": "True", + "hw_mem_page_size": 1000, + }}) + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + membacking = drvr._get_guest_memory_backing_config( + None, None, flavor, image_meta) + self.assertTrue(membacking.locked) + def test_get_guest_memory_backing_config_realtime_invalid_share(self): """Test behavior when there is no pool of shared CPUS on which to place the emulator threads, isolating them from the instance CPU processes. diff --git a/nova/tests/unit/virt/test_hardware.py b/nova/tests/unit/virt/test_hardware.py index 0afedb9d7477..26ec198f08c6 100644 --- a/nova/tests/unit/virt/test_hardware.py +++ b/nova/tests/unit/virt/test_hardware.py @@ -2814,6 +2814,54 @@ class NumberOfSerialPortsTest(test.NoDBTestCase): flavor, image_meta) +class VirtLockMemoryTestCase(test.NoDBTestCase): + def _test_get_locked_memory_constraint(self, spec=None, props=None): + flavor = objects.Flavor(vcpus=16, memory_mb=2048, + extra_specs=spec or {}) + image_meta = objects.ImageMeta.from_dict({"properties": props or {}}) + return hw.get_locked_memory_constraint(flavor, image_meta) + + def test_get_locked_memory_constraint_image(self): + self.assertTrue( + self._test_get_locked_memory_constraint( + spec={"hw:mem_page_size": "small"}, + props={"hw_locked_memory": "True"})) + + def test_get_locked_memory_conflict(self): + ex = self.assertRaises( + exception.FlavorImageLockedMemoryConflict, + self._test_get_locked_memory_constraint, + spec={ + "hw:locked_memory": "False", + "hw:mem_page_size": "small" + }, + props={"hw_locked_memory": "True"} + ) + ex_msg = ("locked_memory value in image (True) and flavor (False) " + "conflict. A consistent value is expected if both " + "specified.") + self.assertEqual(ex_msg, str(ex)) + + def test_get_locked_memory_constraint_forbidden(self): + self.assertRaises( + exception.LockMemoryForbidden, + self._test_get_locked_memory_constraint, + {"hw:locked_memory": "True"}) + + self.assertRaises( + exception.LockMemoryForbidden, + self._test_get_locked_memory_constraint, + {}, + {"hw_locked_memory": "True"}) + + def test_get_locked_memory_constraint_image_false(self): + # False value of locked_memory will not raise LockMemoryForbidden + self.assertFalse( + self._test_get_locked_memory_constraint( + spec=None, + props={"hw_locked_memory": "False"})) + + class VirtMemoryPagesTestCase(test.NoDBTestCase): def test_cell_instance_pagesize(self): cell = objects.InstanceNUMACell( diff --git a/nova/virt/hardware.py b/nova/virt/hardware.py index ce3d23271039..271a719aa298 100644 --- a/nova/virt/hardware.py +++ b/nova/virt/hardware.py @@ -1337,6 +1337,48 @@ def _get_constraint_mappings_from_flavor(flavor, key, func): return hw_numa_map or None +def get_locked_memory_constraint( + flavor: 'objects.Flavor', + image_meta: 'objects.ImageMeta', +) -> ty.Optional[bool]: + """Validate and return the requested locked memory. + + :param flavor: ``nova.objects.Flavor`` instance + :param image_meta: ``nova.objects.ImageMeta`` instance + :raises: exception.LockMemoryForbidden if mem_page_size is not set + while provide locked_memory value in image or flavor. + :returns: The locked memory flag requested. + """ + mem_page_size_flavor, mem_page_size_image = _get_flavor_image_meta( + 'mem_page_size', flavor, image_meta) + + locked_memory_flavor, locked_memory_image = _get_flavor_image_meta( + 'locked_memory', flavor, image_meta) + + if locked_memory_flavor is not None: + # locked_memory_image is boolean type already + locked_memory_flavor = strutils.bool_from_string(locked_memory_flavor) + + if locked_memory_image is not None and ( + locked_memory_flavor != locked_memory_image + ): + # We don't allow provide different value to flavor and image + raise exception.FlavorImageLockedMemoryConflict( + image=locked_memory_image, flavor=locked_memory_flavor) + + locked_memory = locked_memory_flavor + + else: + locked_memory = locked_memory_image + + if locked_memory and not ( + mem_page_size_flavor or mem_page_size_image + ): + raise exception.LockMemoryForbidden() + + return locked_memory + + def _get_numa_cpu_constraint( flavor: 'objects.Flavor', image_meta: 'objects.ImageMeta', @@ -2107,6 +2149,8 @@ def numa_get_constraints(flavor, image_meta): pagesize = _get_numa_pagesize_constraint(flavor, image_meta) vpmems = get_vpmems(flavor) + get_locked_memory_constraint(flavor, image_meta) + # If 'hw:cpu_dedicated_mask' is not found in flavor extra specs, the # 'dedicated_cpus' variable is None, while we hope it being an empty set. dedicated_cpus = dedicated_cpus or set() diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index d9cf1f203d4a..b1df49c6da4b 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -6342,6 +6342,11 @@ class LibvirtDriver(driver.ComputeDriver): membacking = vconfig.LibvirtConfigGuestMemoryBacking() membacking.locked = True + if hardware.get_locked_memory_constraint(flavor, image_meta): + if not membacking: + membacking = vconfig.LibvirtConfigGuestMemoryBacking() + membacking.locked = True + return membacking def _get_memory_backing_hugepages_support(self, inst_topology, numatune): diff --git a/releasenotes/notes/new_locked_memory_option-b68a031779366828.yaml b/releasenotes/notes/new_locked_memory_option-b68a031779366828.yaml new file mode 100644 index 000000000000..72d6e763aaa2 --- /dev/null +++ b/releasenotes/notes/new_locked_memory_option-b68a031779366828.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + Add new ``hw:locked_memory`` extra spec and ``hw_locked_memory`` image + property to lock memory on libvirt guest. Locking memory marks the guest + memory allocations as unmovable and unswappable. + ``hw:locked_memory`` extra spec and ``hw_locked_memory`` image property + accept boolean values in string format like 'Yes' or 'false' value. + Exception `LockMemoryForbidden` will raise, if you set lock memory value + but not set either flavor extra spec + ``hw:mem_page_size`` or image property ``hw_mem_page_size``, + so we can ensure that the scheduler can actually account for this correctly + and prevent out of memory events.