openstack
/
nova
Code Issues Proposed changes

Merge "hardware: Enable 'hw:cpu_dedicated_mask' for creating a mixed instance"

This commit is contained in:
Zuul 2020-07-22 09:37:32 +00:00 committed by Gerrit Code Review
parent 6072f408a4 5c71ac5e02
commit b7a9ff513b
10 changed files with 299 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
doc/source/admin/cpu-topologies.rst
View File

@ -189,10 +189,10 @@ CPUs can use the CPUs of another pinned instance, thus preventing resource
contention between instances.
CPU pinning policies can be used to determine whether an instance should be
pinned or not. There are two policies: ``dedicated`` and ``shared`` (the
default). The ``dedicated`` CPU policy is used to specify that an instance
should use pinned CPUs. To configure a flavor to use the ``dedicated`` CPU
policy, run:
pinned or not. There are three policies: ``dedicated``, ``mixed`` and
``shared`` (the default). The ``dedicated`` CPU policy is used to specify
that all CPUs of an instance should use pinned CPUs. To configure a flavor to
use the ``dedicated`` CPU policy, run:
.. code-block:: console
@ -220,6 +220,22 @@ use pinned CPUs. To configure a flavor to use the ``shared`` CPU policy, run:
$ openstack flavor set [FLAVOR_ID] --property hw:cpu_policy=shared
The ``mixed`` CPU policy is used to specify that an instance use pinned CPUs
along with unpinned CPUs. The instance pinned CPU is specified in the
``hw:cpu_dedicated_mask`` extra spec. For example, to configure a flavor to
use the ``mixed`` CPU policy with 4 vCPUs in total and the first 2 vCPUs as
pinned CPUs:
.. code-block:: console
$ openstack flavor set [FLAVOR_ID] \
--vcpus=4 \
--property hw:cpu_policy=mixed \
--property hw:cpu_dedicated_mask=0-1
For more information about the syntax for ``hw:cpu_dedicated_mask``, refer
to the :doc:`/user/flavors` guide.
.. note::
For more information about the syntax for ``hw:cpu_policy``, refer to the

14
doc/source/user/flavors.rst
View File

@ -458,6 +458,16 @@ CPU pinning policy
an overcommit ratio of 1.0. For example, if a two vCPU guest is pinned to a
single host core with two threads, then the guest will get a topology of
one socket, one core, two threads.
- ``mixed``: This policy will create an instance combined with the ``shared``
policy vCPUs and ``dedicated`` policy vCPUs, as a result, some guest vCPUs
will be freely float across host pCPUs and the rest of guest vCPUs will be
pinned to host pCPUs. The pinned guest vCPUs are configured using the
``hw:cpu_dedicated_mask`` extra spec.
.. note::
The ``hw:cpu_dedicated_mask`` option is only valid if ``hw:cpu_policy``
is set to ``mixed``.
Valid CPU-THREAD-POLICY values are:
@ -477,8 +487,8 @@ CPU pinning policy
.. note::
The ``hw:cpu_thread_policy`` option is only valid if ``hw:cpu_policy`` is
set to ``dedicated``.
The ``hw:cpu_thread_policy`` option is valid if ``hw:cpu_policy`` is set
to ``dedicated`` or ``mixed``.
.. _pci_numa_affinity_policy:

1
nova/api/openstack/compute/servers.py
View File

@ -89,6 +89,7 @@ INVALID_FLAVOR_IMAGE_EXCEPTIONS = (
exception.RealtimeMaskNotFoundOrInvalid,
exception.RequiredMixedInstancePolicy,
exception.RequiredMixedOrRealtimeCPUMask,
exception.InvalidMixedInstanceDedicatedMask,
)
MIN_COMPUTE_MOVE_BANDWIDTH = 39

16
nova/api/validation/extra_specs/hw.py
View File

@ -114,6 +114,22 @@ cpu_policy_validators = [
],
},
),
base.ExtraSpecValidator(
name='hw:cpu_dedicated_mask',
description=(
'A mapping of **guest** CPUs to be pinned to **host** CPUs for an '
'instance with a ``mixed`` CPU policy. For **guest** CPUs which '
'are not in this mapping it will float across host cores.'
),
value={
'type': str,
'description': (
'The **guest** CPU mapping to be pinned to **host** CPUs for '
'an instance with a ``mixed`` CPU policy.'),
# This pattern is identical to 'hw:cpu_realtime_mask' pattern.
'pattern': r'\^?\d+((-\d+)?(,\^?\d+(-\d+)?)?)*',
},
),
]
hugepage_validators = [

5
nova/exception.py
View File

@ -2334,3 +2334,8 @@ class RequiredMixedOrRealtimeCPUMask(Invalid):
class MixedInstanceNotSupportByComputeService(NovaException):
msg_fmt = _("To support 'mixed' policy instance 'nova-compute' service "
"must be upgraded to 'Victoria' or later.")
class InvalidMixedInstanceDedicatedMask(Invalid):
msg_fmt = _("Mixed instance must have at least 1 pinned vCPU and 1 "
"unpinned vCPU. See 'hw:cpu_dedicated_mask'.")

1
nova/tests/unit/api/validation/extra_specs/test_validators.py
View File

@ -62,6 +62,7 @@ class TestValidators(test.NoDBTestCase):
('hw:cpu_realtime_mask', '0'),
('hw:cpu_realtime_mask', '^0'),
('hw:cpu_realtime_mask', '^0,2-3,1'),
('hw:cpu_dedicated_mask', '0-4,^2,6'),
('hw:mem_page_size', 'large'),
('hw:mem_page_size', '2kbit'),
('hw:mem_page_size', '1GB'),

8
nova/tests/unit/compute/test_compute_api.py
View File

@ -361,10 +361,6 @@ class _ComputeAPIUnitTestMixIn(object):
requested_networks)
# TODO(huaqiang): Remove in Wallaby
# TODO(huaqiang): To be removed when 'hw:cpu_dedicated_mask' could be
# parsed from flavor extra spec.
@mock.patch('nova.virt.hardware.get_dedicated_cpu_constraint',
mock.Mock(return_value=set([0, 1, 2])))
@mock.patch('nova.compute.api.API._check_requested_networks',
new=mock.Mock(return_value=1))
@mock.patch('nova.virt.hardware.get_pci_numa_policy_constraint',
@ -2365,10 +2361,6 @@ class _ComputeAPIUnitTestMixIn(object):
self.fail("Exception not raised")
# TODO(huaqiang): Remove in Wallaby
# TODO(huaqiang): To be removed when 'hw:cpu_dedicated_mask' could be
# parsed from flavor extra spec.
@mock.patch('nova.virt.hardware.get_dedicated_cpu_constraint',
mock.Mock(return_value=set([3])))
@mock.patch('nova.compute.api.API.get_instance_host_status',
new=mock.Mock(return_value=fields_obj.HostStatus.UP))
@mock.patch.object(compute_utils, 'is_volume_backed_instance',

8
nova/tests/unit/scheduler/test_utils.py
View File

@ -1080,10 +1080,6 @@ class TestUtils(TestUtilsBase):
self.assertResourceRequestsEqual(expected, rr)
self.assertFalse(rr.cpu_pinning_requested)
# TODO(huaqiang): Remove the mocked 'get_dedicated_cpu_constraint' once
# get_dedicated_cpu_constraint function is ready.
@mock.patch('nova.virt.hardware.get_dedicated_cpu_constraint',
new=mock.Mock(return_value={2, 3}))
def test_resource_request_init_with_mixed_cpus(self):
"""Ensure the mixed instance properly requests the PCPU, VCPU,
MEMORY_MB, DISK_GB resources.
@ -1110,10 +1106,6 @@ class TestUtils(TestUtilsBase):
rr = utils.ResourceRequest(rs)
self.assertResourceRequestsEqual(expected, rr)
# TODO(huaqiang): Remove the mocked 'get_dedicated_cpu_constraint' once
# get_dedicated_cpu_constraint function is ready.
@mock.patch('nova.virt.hardware.get_dedicated_cpu_constraint',
new=mock.Mock(return_value={2, 3}))
def test_resource_request_init_with_mixed_cpus_isolate_emulator(self):
"""Ensure the mixed instance properly requests the PCPU, VCPU,
MEMORY_MB, DISK_GB resources, ensure an extra PCPU resource is

222
nova/tests/unit/virt/test_hardware.py
View File

@ -1507,6 +1507,228 @@ class NUMATopologyTest(test.NoDBTestCase):
},
"expect": exception.RealtimeConfigurationInvalid,
},
{
# NUMA + mixed policy instance and vCPU is evenly distributed
"flavor": objects.Flavor(
vcpus=8, memory_mb=2048,
extra_specs={
"hw:cpu_policy": fields.CPUAllocationPolicy.MIXED,
"hw:cpu_dedicated_mask": "3,7",
"hw:numa_nodes": "2",
}
),
"image": {
"properties": {}
},
"expect": objects.InstanceNUMATopology(cells=[
objects.InstanceNUMACell(
id=0, cpuset=set([0, 1, 2]), pcpuset=set([3]),
memory=1024,
cpu_policy=fields.CPUAllocationPolicy.MIXED),
objects.InstanceNUMACell(
id=1, cpuset=set([4, 5, 6]), pcpuset=set([7]),
memory=1024,
cpu_policy=fields.CPUAllocationPolicy.MIXED),
])
},
{
# mixed policy instance
"flavor": objects.Flavor(
vcpus=4, memory_mb=2048,
extra_specs={
"hw:cpu_policy": fields.CPUAllocationPolicy.MIXED,
"hw:cpu_dedicated_mask": "1,3"
}
),
"image": {
"properties": {}
},
"expect": objects.InstanceNUMATopology(cells=[
objects.InstanceNUMACell(
id=0, cpuset=set([0, 2]), pcpuset=set([1, 3]),
memory=2048,
cpu_policy=fields.CPUAllocationPolicy.MIXED),
])
},
{
# mixed policy instance, 'hw:cpu_dedicated_mask' specifies the
# exclusive CPU set.
"flavor": objects.Flavor(
vcpus=8, memory_mb=4096,
extra_specs={
"hw:cpu_policy": fields.CPUAllocationPolicy.MIXED,
"hw:cpu_dedicated_mask": "^3-5",
"hw:numa_nodes": "2",
}
),
"image": {
"properties": {}
},
"expect": objects.InstanceNUMATopology(cells=[
objects.InstanceNUMACell(
id=0, cpuset=set([3]), pcpuset=set([0, 1, 2]),
memory=2048,
cpu_policy=fields.CPUAllocationPolicy.MIXED),
objects.InstanceNUMACell(
id=1, cpuset=set([4, 5]), pcpuset=set([6, 7]),
memory=2048,
cpu_policy=fields.CPUAllocationPolicy.MIXED),
])
},
{
# NUMA + mixed policy instance
"flavor": objects.Flavor(
vcpus=8, memory_mb=2048,
extra_specs={
"hw:cpu_policy": fields.CPUAllocationPolicy.MIXED,
"hw:cpu_dedicated_mask": "1,3",
"hw:numa_nodes": "2",
"hw:numa_cpus.0": "0-1",
"hw:numa_mem.0": "1024",
"hw:numa_cpus.1": "2-7",
"hw:numa_mem.1": "1024",
}
),
"image": {
"properties": {}
},
"expect": objects.InstanceNUMATopology(cells=[
objects.InstanceNUMACell(
id=0, cpuset=set([0]), pcpuset=set([1]),
memory=1024,
cpu_policy=fields.CPUAllocationPolicy.MIXED),
objects.InstanceNUMACell(
id=1, cpuset=set([2, 4, 5, 6, 7]),
pcpuset=set([3]), memory=1024,
cpu_policy=fields.CPUAllocationPolicy.MIXED),
])
},
{
# dedicated CPU distributes in one NUMA cell
"flavor": objects.Flavor(
vcpus=8, memory_mb=2048,
extra_specs={
"hw:cpu_policy": fields.CPUAllocationPolicy.MIXED,
"hw:cpu_dedicated_mask": "7",
"hw:numa_nodes": "2",
"hw:numa_cpus.0": "0-1",
"hw:numa_mem.0": "1024",
"hw:numa_cpus.1": "2-7",
"hw:numa_mem.1": "1024",
}
),
"image": {
"properties": {}
},
"expect": objects.InstanceNUMATopology(cells=[
objects.InstanceNUMACell(
id=0, cpuset=set([0, 1]), pcpuset=set(),
memory=1024,
cpu_policy=fields.CPUAllocationPolicy.MIXED),
objects.InstanceNUMACell(
id=1, cpuset=set([2, 3, 4, 5, 6]),
pcpuset=set([7]), memory=1024,
cpu_policy=fields.CPUAllocationPolicy.MIXED),
])
},
{
# CPU number in 'hw:cpu_dedicated_mask' should not be equal to
# flavor.vcpus
"flavor": objects.Flavor(
vcpus=4, memory_mb=2048,
extra_specs={
"hw:cpu_policy": fields.CPUAllocationPolicy.MIXED,
"hw:cpu_dedicated_mask": "0-3",
}
),
"image": {
"properties": {}
},
"expect": exception.InvalidMixedInstanceDedicatedMask,
},
{
# CPU ID in 'hw:cpu_dedicated_mask' should not exceed
# flavor.vcpus
"flavor": objects.Flavor(
vcpus=4, memory_mb=2048,
extra_specs={
"hw:cpu_policy": fields.CPUAllocationPolicy.MIXED,
"hw:cpu_dedicated_mask": "0-3,4",
}
),
"image": {
"properties": {}
},
"expect": exception.InvalidMixedInstanceDedicatedMask,
},
{
# 'hw:cpu_dedicated_mask' should not be defined along with
# 'hw:cpu_policy=shared'
"flavor": objects.Flavor(
vcpus=4, memory_mb=2048,
extra_specs={
"hw:cpu_policy": fields.CPUAllocationPolicy.SHARED,
"hw:cpu_dedicated_mask": "0"
}
),
"image": {
"properties": {}
},
"expect": exception.RequiredMixedInstancePolicy,
},
{
# 'hw:cpu_dedicated_mask' should not be defined along with
# 'hw:cpu_policy=dedicated'
"flavor": objects.Flavor(
vcpus=4, memory_mb=2048,
extra_specs={
"hw:cpu_policy": fields.CPUAllocationPolicy.DEDICATED,
"hw:cpu_dedicated_mask": "0"
}
),
"image": {
"properties": {}
},
"expect": exception.RequiredMixedInstancePolicy,
},
{
# 'hw:cpu_dedicated_mask' should be defined along with
# 'hw:cpu_policy=mixed'
"flavor": objects.Flavor(
vcpus=4, memory_mb=2048,
extra_specs={
"hw:cpu_policy": fields.CPUAllocationPolicy.MIXED,
}
),
"image": {
"properties": {}
},
"expect": exception.RequiredMixedOrRealtimeCPUMask,
},
{
# Create 'mixed' instance with the 'ISOLATE' emulator
# thread policy
"flavor": objects.Flavor(
vcpus=4, memory_mb=2048,
extra_specs={
"hw:emulator_threads_policy": "isolate",
"hw:cpu_policy": "mixed",
"hw:cpu_dedicated_mask": "3"
}
),
"image": {
"properties": {}
},
"expect": objects.InstanceNUMATopology(
emulator_threads_policy=
fields.CPUEmulatorThreadsPolicy.ISOLATE,
cells=[objects.InstanceNUMACell(
id=0, cpuset=set([0, 1, 2]),
pcpuset=set([3]), memory=2048,
cpu_policy=fields.CPUAllocationPolicy.MIXED)
]
),
},
{
# Invalid CPU thread pinning override
"flavor": objects.Flavor(

31
nova/virt/hardware.py
View File

@ -1712,9 +1712,6 @@ def _get_hyperthreading_trait(
# NOTE(stephenfin): This must be public as it's used elsewhere
# TODO(Huaqiang): To be filled with the logic of parsing
# 'hw:cpu_dedicated_mask' and relevant test cases in later patches once the
# code is ready to build up an instance in 'mixed' CPU allocation policy.
def get_dedicated_cpu_constraint(
flavor: 'objects.Flavor',
) -> ty.Optional[ty.Set[int]]:
@ -1723,7 +1720,26 @@ def get_dedicated_cpu_constraint(
:param flavor: ``nova.objects.Flavor`` instance
:returns: The dedicated CPUs requested, else None.
"""
return None
mask = flavor.get('extra_specs', {}).get('hw:cpu_dedicated_mask')
if not mask:
return None
if mask.strip().startswith('^'):
pcpus = parse_cpu_spec("0-%d,%s" % (flavor.vcpus - 1, mask))
else:
pcpus = parse_cpu_spec("%s" % (mask))
cpus = set(range(flavor.vcpus))
vcpus = cpus - pcpus
if not pcpus or not vcpus:
raise exception.InvalidMixedInstanceDedicatedMask()
if not pcpus.issubset(cpus):
msg = _('Mixed instance dedicated vCPU(s) mask is not a subset of '
'vCPUs in the flavor. See "hw:cpu_dedicated_mask"')
raise exception.InvalidMixedInstanceDedicatedMask(msg)
return pcpus
# NOTE(stephenfin): This must be public as it's used elsewhere
@ -1871,6 +1887,8 @@ def numa_get_constraints(flavor, image_meta):
:raises: exception.RequiredMixedOrRealtimeCPUMask the mixed policy instance
dedicated CPU mask can only be specified through either
'hw:cpu_realtime_mask' or 'hw:cpu_dedicated_mask', not both.
:raises: exception.InvalidMixedInstanceDedicatedMask if specify an invalid
CPU mask for 'hw:cpu_dedicated_mask'.
:returns: objects.InstanceNUMATopology, or None
"""
@ -1956,11 +1974,6 @@ def numa_get_constraints(flavor, image_meta):
if dedicated_cpus:
raise exception.RequiredMixedInstancePolicy()
else: # MIXED
# FIXME(huaqiang): So far, 'mixed' instance is not supported
# and the 'dedicated_cpus' variable is set to 'None' due to being not
# ready to parse 'hw:cpu_dedicated_mask'.
# The logic of parsing 'hw:cpu_dedicated_mask' should be added once
# the code is ready for setting up an 'mixed' instance.
if dedicated_cpus is None:
raise exception.RequiredMixedOrRealtimeCPUMask()