diff --git a/doc/source/admin/cpu-topologies.rst b/doc/source/admin/cpu-topologies.rst index e99f5c3d712d..fdb2d7b59ef6 100644 --- a/doc/source/admin/cpu-topologies.rst +++ b/doc/source/admin/cpu-topologies.rst @@ -189,10 +189,10 @@ CPUs can use the CPUs of another pinned instance, thus preventing resource contention between instances. CPU pinning policies can be used to determine whether an instance should be -pinned or not. There are two policies: ``dedicated`` and ``shared`` (the -default). The ``dedicated`` CPU policy is used to specify that an instance -should use pinned CPUs. To configure a flavor to use the ``dedicated`` CPU -policy, run: +pinned or not. There are three policies: ``dedicated``, ``mixed`` and +``shared`` (the default). The ``dedicated`` CPU policy is used to specify +that all CPUs of an instance should use pinned CPUs. To configure a flavor to +use the ``dedicated`` CPU policy, run: .. code-block:: console @@ -220,6 +220,22 @@ use pinned CPUs. To configure a flavor to use the ``shared`` CPU policy, run: $ openstack flavor set [FLAVOR_ID] --property hw:cpu_policy=shared +The ``mixed`` CPU policy is used to specify that an instance use pinned CPUs +along with unpinned CPUs. The instance pinned CPU is specified in the +``hw:cpu_dedicated_mask`` extra spec. For example, to configure a flavor to +use the ``mixed`` CPU policy with 4 vCPUs in total and the first 2 vCPUs as +pinned CPUs: + +.. code-block:: console + + $ openstack flavor set [FLAVOR_ID] \ + --vcpus=4 \ + --property hw:cpu_policy=mixed \ + --property hw:cpu_dedicated_mask=0-1 + +For more information about the syntax for ``hw:cpu_dedicated_mask``, refer +to the :doc:`/user/flavors` guide. + .. note:: For more information about the syntax for ``hw:cpu_policy``, refer to the diff --git a/doc/source/user/flavors.rst b/doc/source/user/flavors.rst index 30704be4a024..55b976b815e6 100644 --- a/doc/source/user/flavors.rst +++ b/doc/source/user/flavors.rst @@ -458,6 +458,16 @@ CPU pinning policy an overcommit ratio of 1.0. For example, if a two vCPU guest is pinned to a single host core with two threads, then the guest will get a topology of one socket, one core, two threads. + - ``mixed``: This policy will create an instance combined with the ``shared`` + policy vCPUs and ``dedicated`` policy vCPUs, as a result, some guest vCPUs + will be freely float across host pCPUs and the rest of guest vCPUs will be + pinned to host pCPUs. The pinned guest vCPUs are configured using the + ``hw:cpu_dedicated_mask`` extra spec. + + .. note:: + + The ``hw:cpu_dedicated_mask`` option is only valid if ``hw:cpu_policy`` + is set to ``mixed``. Valid CPU-THREAD-POLICY values are: @@ -477,8 +487,8 @@ CPU pinning policy .. note:: - The ``hw:cpu_thread_policy`` option is only valid if ``hw:cpu_policy`` is - set to ``dedicated``. + The ``hw:cpu_thread_policy`` option is valid if ``hw:cpu_policy`` is set + to ``dedicated`` or ``mixed``. .. _pci_numa_affinity_policy: diff --git a/nova/api/openstack/compute/servers.py b/nova/api/openstack/compute/servers.py index f9a8966b201d..9517021f925d 100644 --- a/nova/api/openstack/compute/servers.py +++ b/nova/api/openstack/compute/servers.py @@ -89,6 +89,7 @@ INVALID_FLAVOR_IMAGE_EXCEPTIONS = ( exception.RealtimeMaskNotFoundOrInvalid, exception.RequiredMixedInstancePolicy, exception.RequiredMixedOrRealtimeCPUMask, + exception.InvalidMixedInstanceDedicatedMask, ) MIN_COMPUTE_MOVE_BANDWIDTH = 39 diff --git a/nova/api/validation/extra_specs/hw.py b/nova/api/validation/extra_specs/hw.py index fde3675db412..235890ae72c0 100644 --- a/nova/api/validation/extra_specs/hw.py +++ b/nova/api/validation/extra_specs/hw.py @@ -114,6 +114,22 @@ cpu_policy_validators = [ ], }, ), + base.ExtraSpecValidator( + name='hw:cpu_dedicated_mask', + description=( + 'A mapping of **guest** CPUs to be pinned to **host** CPUs for an ' + 'instance with a ``mixed`` CPU policy. For **guest** CPUs which ' + 'are not in this mapping it will float across host cores.' + ), + value={ + 'type': str, + 'description': ( + 'The **guest** CPU mapping to be pinned to **host** CPUs for ' + 'an instance with a ``mixed`` CPU policy.'), + # This pattern is identical to 'hw:cpu_realtime_mask' pattern. + 'pattern': r'\^?\d+((-\d+)?(,\^?\d+(-\d+)?)?)*', + }, + ), ] hugepage_validators = [ diff --git a/nova/exception.py b/nova/exception.py index 8f6ec42a63af..5d60ce009be6 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -2330,3 +2330,8 @@ class RequiredMixedOrRealtimeCPUMask(Invalid): class MixedInstanceNotSupportByComputeService(NovaException): msg_fmt = _("To support 'mixed' policy instance 'nova-compute' service " "must be upgraded to 'Victoria' or later.") + + +class InvalidMixedInstanceDedicatedMask(Invalid): + msg_fmt = _("Mixed instance must have at least 1 pinned vCPU and 1 " + "unpinned vCPU. See 'hw:cpu_dedicated_mask'.") diff --git a/nova/tests/unit/api/validation/extra_specs/test_validators.py b/nova/tests/unit/api/validation/extra_specs/test_validators.py index a7ab210f39e6..1aa30d765b99 100644 --- a/nova/tests/unit/api/validation/extra_specs/test_validators.py +++ b/nova/tests/unit/api/validation/extra_specs/test_validators.py @@ -62,6 +62,7 @@ class TestValidators(test.NoDBTestCase): ('hw:cpu_realtime_mask', '0'), ('hw:cpu_realtime_mask', '^0'), ('hw:cpu_realtime_mask', '^0,2-3,1'), + ('hw:cpu_dedicated_mask', '0-4,^2,6'), ('hw:mem_page_size', 'large'), ('hw:mem_page_size', '2kbit'), ('hw:mem_page_size', '1GB'), diff --git a/nova/tests/unit/compute/test_compute_api.py b/nova/tests/unit/compute/test_compute_api.py index 5936a55f36c2..6fd2bbf8092a 100644 --- a/nova/tests/unit/compute/test_compute_api.py +++ b/nova/tests/unit/compute/test_compute_api.py @@ -361,10 +361,6 @@ class _ComputeAPIUnitTestMixIn(object): requested_networks) # TODO(huaqiang): Remove in Wallaby - # TODO(huaqiang): To be removed when 'hw:cpu_dedicated_mask' could be - # parsed from flavor extra spec. - @mock.patch('nova.virt.hardware.get_dedicated_cpu_constraint', - mock.Mock(return_value=set([0, 1, 2]))) @mock.patch('nova.compute.api.API._check_requested_networks', new=mock.Mock(return_value=1)) @mock.patch('nova.virt.hardware.get_pci_numa_policy_constraint', @@ -2365,10 +2361,6 @@ class _ComputeAPIUnitTestMixIn(object): self.fail("Exception not raised") # TODO(huaqiang): Remove in Wallaby - # TODO(huaqiang): To be removed when 'hw:cpu_dedicated_mask' could be - # parsed from flavor extra spec. - @mock.patch('nova.virt.hardware.get_dedicated_cpu_constraint', - mock.Mock(return_value=set([3]))) @mock.patch('nova.compute.api.API.get_instance_host_status', new=mock.Mock(return_value=fields_obj.HostStatus.UP)) @mock.patch.object(compute_utils, 'is_volume_backed_instance', diff --git a/nova/tests/unit/scheduler/test_utils.py b/nova/tests/unit/scheduler/test_utils.py index a9c5ba406ac9..c9cc77980a4f 100644 --- a/nova/tests/unit/scheduler/test_utils.py +++ b/nova/tests/unit/scheduler/test_utils.py @@ -1080,10 +1080,6 @@ class TestUtils(TestUtilsBase): self.assertResourceRequestsEqual(expected, rr) self.assertFalse(rr.cpu_pinning_requested) - # TODO(huaqiang): Remove the mocked 'get_dedicated_cpu_constraint' once - # get_dedicated_cpu_constraint function is ready. - @mock.patch('nova.virt.hardware.get_dedicated_cpu_constraint', - new=mock.Mock(return_value={2, 3})) def test_resource_request_init_with_mixed_cpus(self): """Ensure the mixed instance properly requests the PCPU, VCPU, MEMORY_MB, DISK_GB resources. @@ -1110,10 +1106,6 @@ class TestUtils(TestUtilsBase): rr = utils.ResourceRequest(rs) self.assertResourceRequestsEqual(expected, rr) - # TODO(huaqiang): Remove the mocked 'get_dedicated_cpu_constraint' once - # get_dedicated_cpu_constraint function is ready. - @mock.patch('nova.virt.hardware.get_dedicated_cpu_constraint', - new=mock.Mock(return_value={2, 3})) def test_resource_request_init_with_mixed_cpus_isolate_emulator(self): """Ensure the mixed instance properly requests the PCPU, VCPU, MEMORY_MB, DISK_GB resources, ensure an extra PCPU resource is diff --git a/nova/tests/unit/virt/test_hardware.py b/nova/tests/unit/virt/test_hardware.py index 62214f8d2101..6554e6abacfc 100644 --- a/nova/tests/unit/virt/test_hardware.py +++ b/nova/tests/unit/virt/test_hardware.py @@ -1507,6 +1507,228 @@ class NUMATopologyTest(test.NoDBTestCase): }, "expect": exception.RealtimeConfigurationInvalid, }, + { + # NUMA + mixed policy instance and vCPU is evenly distributed + "flavor": objects.Flavor( + vcpus=8, memory_mb=2048, + extra_specs={ + "hw:cpu_policy": fields.CPUAllocationPolicy.MIXED, + "hw:cpu_dedicated_mask": "3,7", + "hw:numa_nodes": "2", + } + ), + "image": { + "properties": {} + }, + "expect": objects.InstanceNUMATopology(cells=[ + objects.InstanceNUMACell( + id=0, cpuset=set([0, 1, 2]), pcpuset=set([3]), + memory=1024, + cpu_policy=fields.CPUAllocationPolicy.MIXED), + objects.InstanceNUMACell( + id=1, cpuset=set([4, 5, 6]), pcpuset=set([7]), + memory=1024, + cpu_policy=fields.CPUAllocationPolicy.MIXED), + ]) + }, + { + # mixed policy instance + "flavor": objects.Flavor( + vcpus=4, memory_mb=2048, + extra_specs={ + "hw:cpu_policy": fields.CPUAllocationPolicy.MIXED, + "hw:cpu_dedicated_mask": "1,3" + } + ), + "image": { + "properties": {} + }, + "expect": objects.InstanceNUMATopology(cells=[ + objects.InstanceNUMACell( + id=0, cpuset=set([0, 2]), pcpuset=set([1, 3]), + memory=2048, + cpu_policy=fields.CPUAllocationPolicy.MIXED), + ]) + }, + { + # mixed policy instance, 'hw:cpu_dedicated_mask' specifies the + # exclusive CPU set. + "flavor": objects.Flavor( + vcpus=8, memory_mb=4096, + extra_specs={ + "hw:cpu_policy": fields.CPUAllocationPolicy.MIXED, + "hw:cpu_dedicated_mask": "^3-5", + "hw:numa_nodes": "2", + } + ), + "image": { + "properties": {} + }, + "expect": objects.InstanceNUMATopology(cells=[ + objects.InstanceNUMACell( + id=0, cpuset=set([3]), pcpuset=set([0, 1, 2]), + memory=2048, + cpu_policy=fields.CPUAllocationPolicy.MIXED), + objects.InstanceNUMACell( + id=1, cpuset=set([4, 5]), pcpuset=set([6, 7]), + memory=2048, + cpu_policy=fields.CPUAllocationPolicy.MIXED), + ]) + }, + { + # NUMA + mixed policy instance + "flavor": objects.Flavor( + vcpus=8, memory_mb=2048, + extra_specs={ + "hw:cpu_policy": fields.CPUAllocationPolicy.MIXED, + "hw:cpu_dedicated_mask": "1,3", + "hw:numa_nodes": "2", + "hw:numa_cpus.0": "0-1", + "hw:numa_mem.0": "1024", + "hw:numa_cpus.1": "2-7", + "hw:numa_mem.1": "1024", + } + ), + "image": { + "properties": {} + }, + "expect": objects.InstanceNUMATopology(cells=[ + objects.InstanceNUMACell( + id=0, cpuset=set([0]), pcpuset=set([1]), + memory=1024, + cpu_policy=fields.CPUAllocationPolicy.MIXED), + objects.InstanceNUMACell( + id=1, cpuset=set([2, 4, 5, 6, 7]), + pcpuset=set([3]), memory=1024, + cpu_policy=fields.CPUAllocationPolicy.MIXED), + ]) + }, + { + # dedicated CPU distributes in one NUMA cell + "flavor": objects.Flavor( + vcpus=8, memory_mb=2048, + extra_specs={ + "hw:cpu_policy": fields.CPUAllocationPolicy.MIXED, + "hw:cpu_dedicated_mask": "7", + "hw:numa_nodes": "2", + "hw:numa_cpus.0": "0-1", + "hw:numa_mem.0": "1024", + "hw:numa_cpus.1": "2-7", + "hw:numa_mem.1": "1024", + } + ), + "image": { + "properties": {} + }, + "expect": objects.InstanceNUMATopology(cells=[ + objects.InstanceNUMACell( + id=0, cpuset=set([0, 1]), pcpuset=set(), + memory=1024, + cpu_policy=fields.CPUAllocationPolicy.MIXED), + objects.InstanceNUMACell( + id=1, cpuset=set([2, 3, 4, 5, 6]), + pcpuset=set([7]), memory=1024, + cpu_policy=fields.CPUAllocationPolicy.MIXED), + ]) + }, + { + # CPU number in 'hw:cpu_dedicated_mask' should not be equal to + # flavor.vcpus + "flavor": objects.Flavor( + vcpus=4, memory_mb=2048, + extra_specs={ + "hw:cpu_policy": fields.CPUAllocationPolicy.MIXED, + "hw:cpu_dedicated_mask": "0-3", + } + ), + "image": { + "properties": {} + }, + "expect": exception.InvalidMixedInstanceDedicatedMask, + }, + { + # CPU ID in 'hw:cpu_dedicated_mask' should not exceed + # flavor.vcpus + "flavor": objects.Flavor( + vcpus=4, memory_mb=2048, + extra_specs={ + "hw:cpu_policy": fields.CPUAllocationPolicy.MIXED, + "hw:cpu_dedicated_mask": "0-3,4", + } + ), + "image": { + "properties": {} + }, + "expect": exception.InvalidMixedInstanceDedicatedMask, + }, + { + # 'hw:cpu_dedicated_mask' should not be defined along with + # 'hw:cpu_policy=shared' + "flavor": objects.Flavor( + vcpus=4, memory_mb=2048, + extra_specs={ + "hw:cpu_policy": fields.CPUAllocationPolicy.SHARED, + "hw:cpu_dedicated_mask": "0" + } + ), + "image": { + "properties": {} + }, + "expect": exception.RequiredMixedInstancePolicy, + }, + { + # 'hw:cpu_dedicated_mask' should not be defined along with + # 'hw:cpu_policy=dedicated' + "flavor": objects.Flavor( + vcpus=4, memory_mb=2048, + extra_specs={ + "hw:cpu_policy": fields.CPUAllocationPolicy.DEDICATED, + "hw:cpu_dedicated_mask": "0" + } + ), + "image": { + "properties": {} + }, + "expect": exception.RequiredMixedInstancePolicy, + }, + { + # 'hw:cpu_dedicated_mask' should be defined along with + # 'hw:cpu_policy=mixed' + "flavor": objects.Flavor( + vcpus=4, memory_mb=2048, + extra_specs={ + "hw:cpu_policy": fields.CPUAllocationPolicy.MIXED, + } + ), + "image": { + "properties": {} + }, + "expect": exception.RequiredMixedOrRealtimeCPUMask, + }, + { + # Create 'mixed' instance with the 'ISOLATE' emulator + # thread policy + "flavor": objects.Flavor( + vcpus=4, memory_mb=2048, + extra_specs={ + "hw:emulator_threads_policy": "isolate", + "hw:cpu_policy": "mixed", + "hw:cpu_dedicated_mask": "3" + } + ), + "image": { + "properties": {} + }, + "expect": objects.InstanceNUMATopology( + emulator_threads_policy= + fields.CPUEmulatorThreadsPolicy.ISOLATE, + cells=[objects.InstanceNUMACell( + id=0, cpuset=set([0, 1, 2]), + pcpuset=set([3]), memory=2048, + cpu_policy=fields.CPUAllocationPolicy.MIXED) + ] + ), + }, { # Invalid CPU thread pinning override "flavor": objects.Flavor( diff --git a/nova/virt/hardware.py b/nova/virt/hardware.py index 5b5d09e9e58b..4b30d6beab7f 100644 --- a/nova/virt/hardware.py +++ b/nova/virt/hardware.py @@ -1707,9 +1707,6 @@ def _get_hyperthreading_trait( # NOTE(stephenfin): This must be public as it's used elsewhere -# TODO(Huaqiang): To be filled with the logic of parsing -# 'hw:cpu_dedicated_mask' and relevant test cases in later patches once the -# code is ready to build up an instance in 'mixed' CPU allocation policy. def get_dedicated_cpu_constraint( flavor: 'objects.Flavor', ) -> ty.Optional[ty.Set[int]]: @@ -1718,7 +1715,26 @@ def get_dedicated_cpu_constraint( :param flavor: ``nova.objects.Flavor`` instance :returns: The dedicated CPUs requested, else None. """ - return None + mask = flavor.get('extra_specs', {}).get('hw:cpu_dedicated_mask') + if not mask: + return None + + if mask.strip().startswith('^'): + pcpus = parse_cpu_spec("0-%d,%s" % (flavor.vcpus - 1, mask)) + else: + pcpus = parse_cpu_spec("%s" % (mask)) + + cpus = set(range(flavor.vcpus)) + vcpus = cpus - pcpus + if not pcpus or not vcpus: + raise exception.InvalidMixedInstanceDedicatedMask() + + if not pcpus.issubset(cpus): + msg = _('Mixed instance dedicated vCPU(s) mask is not a subset of ' + 'vCPUs in the flavor. See "hw:cpu_dedicated_mask"') + raise exception.InvalidMixedInstanceDedicatedMask(msg) + + return pcpus # NOTE(stephenfin): This must be public as it's used elsewhere @@ -1866,6 +1882,8 @@ def numa_get_constraints(flavor, image_meta): :raises: exception.RequiredMixedOrRealtimeCPUMask the mixed policy instance dedicated CPU mask can only be specified through either 'hw:cpu_realtime_mask' or 'hw:cpu_dedicated_mask', not both. + :raises: exception.InvalidMixedInstanceDedicatedMask if specify an invalid + CPU mask for 'hw:cpu_dedicated_mask'. :returns: objects.InstanceNUMATopology, or None """ @@ -1951,11 +1969,6 @@ def numa_get_constraints(flavor, image_meta): if dedicated_cpus: raise exception.RequiredMixedInstancePolicy() else: # MIXED - # FIXME(huaqiang): So far, 'mixed' instance is not supported - # and the 'dedicated_cpus' variable is set to 'None' due to being not - # ready to parse 'hw:cpu_dedicated_mask'. - # The logic of parsing 'hw:cpu_dedicated_mask' should be added once - # the code is ready for setting up an 'mixed' instance. if dedicated_cpus is None: raise exception.RequiredMixedOrRealtimeCPUMask()