Enable support for CPU pinning
- enable ACPI for nodes with environment variable DRIVER_ENABLE_ACPI - use environment variable NUMA_NODES to calculate number of CPUs per numa cell: cpus = *_NODE_CPU / NUMA_CELLS , and amount of memory for each numa cell: memory = *_NODE_MEMORY / NUMA_CELLS - added unit test for node with numa To enable NUMA for fuel-qa system tests, the following environment variables should be set: export DRIVER_ENABLE_ACPI=true export NUMA_NODES=2 # or any other suitable number greater than 0 export IFACE_0=ens3 export IFACE_1=ens4 export IFACE_2=ens5 export IFACE_3=ens6 export IFACE_4=ens7 export IFACE_5=ens8 Change-Id: Ica0822a75ff3a2c7d6f2579019b7bf32732c8dda blueprint: fuel-devops-numa-support
This commit is contained in:
Dennis Dmitriev
parent
b54fa52948
commit
6e00417ace
|
|
@ -12,6 +12,7 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from __future__ import division
|
||||
import datetime
|
||||
from time import sleep
|
||||
import xml.etree.ElementTree as ET
|
||||
|
|
@ -21,6 +22,7 @@ import libvirt
|
|||
import os
|
||||
|
||||
from devops.driver.libvirt.libvirt_xml_builder import LibvirtXMLBuilder
|
||||
from devops.error import DevopsError
|
||||
from devops.helpers.helpers import _get_file_size
|
||||
from devops.helpers.retry import retry
|
||||
from devops.helpers import scancodes
|
||||
|
|
@ -121,7 +123,7 @@ class DevopsDriver(object):
|
|||
def __init__(self,
|
||||
connection_string="qemu:///system",
|
||||
storage_pool_name="default",
|
||||
stp=True, hpet=True, use_host_cpu=True):
|
||||
stp=True, hpet=True, use_host_cpu=True, enable_acpi=False):
|
||||
"""libvirt driver
|
||||
|
||||
:param use_host_cpu: When creating nodes, should libvirt's
|
||||
|
|
@ -137,6 +139,7 @@ class DevopsDriver(object):
|
|||
self.storage_pool_name = storage_pool_name
|
||||
self.reboot_timeout = None
|
||||
self.use_host_cpu = use_host_cpu
|
||||
self.enable_acpi = enable_acpi
|
||||
self.use_hugepages = settings.USE_HUGEPAGES
|
||||
|
||||
if settings.VNC_PASSWORD:
|
||||
|
|
@ -424,8 +427,37 @@ class DevopsDriver(object):
|
|||
'guest/arch[@name="{0:>s}"]/'
|
||||
'domain[@type="{1:>s}"]/emulator'.format(
|
||||
node.architecture, node.hypervisor)).text
|
||||
|
||||
# NUMA nodes
|
||||
# TODO(ddmitriev): pass 'numa' structure from the YAML template
|
||||
# for fuel-devops-3.0 instead of calculating parameters here.
|
||||
numa_nodes = settings.HARDWARE["numa_nodes"]
|
||||
numa = []
|
||||
if numa_nodes:
|
||||
cpus_per_numa = node.vcpu // numa_nodes
|
||||
if cpus_per_numa * numa_nodes != node.vcpu:
|
||||
raise DevopsError(
|
||||
"NUMA_NODES={0} is not a multiple of the number of CPU={1}"
|
||||
" for node '{2}'".format(numa_nodes, node.vcpu, node.name))
|
||||
|
||||
memory_per_numa = (node.memory * 1024) // numa_nodes
|
||||
if memory_per_numa * numa_nodes != (node.memory * 1024):
|
||||
raise DevopsError(
|
||||
"NUMA_NODES={0} is not a multiple of the amount of "
|
||||
"MEMORY={1} for node '{2}'".format(numa_nodes,
|
||||
node.memory,
|
||||
node.name))
|
||||
for x in range(numa_nodes):
|
||||
# List of cpu IDs for the numa node
|
||||
cpus = range(x * cpus_per_numa, (x + 1) * cpus_per_numa)
|
||||
cell = {
|
||||
'cpus': ','.join(map(str, cpus)),
|
||||
'memory': memory_per_numa,
|
||||
}
|
||||
numa.append(cell)
|
||||
|
||||
node_xml = self.xml_builder.build_node_xml(
|
||||
node, emulator, if_prefix=settings.LIBVIRT_IF_PREFIX)
|
||||
node, emulator, numa, if_prefix=settings.LIBVIRT_IF_PREFIX)
|
||||
logger.info(node_xml)
|
||||
node.uuid = self.conn.defineXML(node_xml).UUIDString()
|
||||
|
||||
|
|
|
|||
|
|
@ -247,7 +247,7 @@ class LibvirtXMLBuilder(object):
|
|||
|
||||
return str(filter_xml)
|
||||
|
||||
def build_node_xml(self, node, emulator, if_prefix="fuelnet"):
|
||||
def build_node_xml(self, node, emulator, numa, if_prefix="fuelnet"):
|
||||
"""Generate node XML
|
||||
|
||||
:type node: Node
|
||||
|
|
@ -258,8 +258,26 @@ class LibvirtXMLBuilder(object):
|
|||
node_xml.name(
|
||||
self._get_name(node.environment and node.environment.name or '',
|
||||
node.name))
|
||||
|
||||
# TODO(ddmitriev): add a libvirt node attribute 'acpi'
|
||||
# for fuel-devops3.0.0
|
||||
if self.driver.enable_acpi:
|
||||
with node_xml.features:
|
||||
node_xml.acpi
|
||||
|
||||
if self.driver.use_host_cpu:
|
||||
node_xml.cpu(mode='host-passthrough')
|
||||
cpu_args = {'mode': 'host-passthrough'}
|
||||
else:
|
||||
cpu_args = {}
|
||||
with node_xml.cpu(**cpu_args):
|
||||
if numa:
|
||||
with node_xml.numa:
|
||||
for cell in numa:
|
||||
node_xml.cell(
|
||||
cpus=str(cell['cpus']),
|
||||
memory=str(cell['memory'])
|
||||
)
|
||||
|
||||
node_xml.vcpu(str(node.vcpu))
|
||||
node_xml.memory(str(node.memory * 1024), unit='KiB')
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ DRIVER_PARAMETERS = {
|
|||
'stp': True,
|
||||
'hpet': False,
|
||||
'use_host_cpu': get_var_as_bool('DRIVER_USE_HOST_CPU', True),
|
||||
'enable_acpi': get_var_as_bool('DRIVER_ENABLE_ACPI', False),
|
||||
}
|
||||
|
||||
INSTALLED_APPS = ['south', 'devops']
|
||||
|
|
@ -204,7 +205,11 @@ HARDWARE = {
|
|||
"admin_node_memory": int(os.environ.get("ADMIN_NODE_MEMORY", 3072)),
|
||||
"admin_node_cpu": int(os.environ.get("ADMIN_NODE_CPU", 2)),
|
||||
"slave_node_cpu": int(os.environ.get("SLAVE_NODE_CPU", 2)),
|
||||
"slave_node_memory": int(os.environ.get("SLAVE_NODE_MEMORY", 3027)),
|
||||
"slave_node_memory": int(os.environ.get("SLAVE_NODE_MEMORY", 3072)),
|
||||
# Number of NUMA nodes on each VM node.
|
||||
# Each NUMA node will have (<admin|slave>_node_cpu/numa_nodes) CPUs
|
||||
# and (<admin|slave>_node_memory/numa_nodes) memory.
|
||||
"numa_nodes": int(os.environ.get("NUMA_NODES", 0)),
|
||||
}
|
||||
|
||||
USE_ALL_DISKS = get_var_as_bool('USE_ALL_DISKS', True)
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ class BaseTestXMLBuilder(TestCase):
|
|||
self.net = mock.Mock()
|
||||
self.node = mock.Mock()
|
||||
self.xml_builder.driver.use_hugepages = None
|
||||
self.xml_builder.driver.enable_acpi = None
|
||||
|
||||
def _reformat_xml(self, xml):
|
||||
"""Takes XML in string, parses it and returns pretty printed XML."""
|
||||
|
|
@ -355,7 +356,7 @@ class TestNodeXml(BaseTestXMLBuilder):
|
|||
self.node.interfaces = []
|
||||
|
||||
def test_node(self):
|
||||
xml = self.xml_builder.build_node_xml(self.node, 'test_emulator')
|
||||
xml = self.xml_builder.build_node_xml(self.node, 'test_emulator', [])
|
||||
boot = json.loads(self.node.boot)
|
||||
expected = '''
|
||||
<domain type="test_hypervisor">
|
||||
|
|
@ -398,6 +399,67 @@ class TestNodeXml(BaseTestXMLBuilder):
|
|||
boot[0], boot[1])
|
||||
self.assertXMLIn(expected, xml)
|
||||
|
||||
def test_node_with_numa(self):
|
||||
self.node.vcpu = 4
|
||||
self.node.memory = 1024
|
||||
numa = [
|
||||
{
|
||||
'cpus': '0,1',
|
||||
'memory': 512 * 1024
|
||||
},
|
||||
{
|
||||
'cpus': '2,3',
|
||||
'memory': 512 * 1024
|
||||
}
|
||||
]
|
||||
xml = self.xml_builder.build_node_xml(self.node, 'test_emulator', numa)
|
||||
boot = json.loads(self.node.boot)
|
||||
expected = '''
|
||||
<domain type="test_hypervisor">
|
||||
<name>test_env_name_test_name</name>
|
||||
<cpu mode="host-passthrough">
|
||||
<numa>
|
||||
<cell cpus="0,1" memory="524288"/>
|
||||
<cell cpus="2,3" memory="524288"/>
|
||||
</numa>
|
||||
</cpu>
|
||||
<vcpu>{0}</vcpu>
|
||||
<memory unit="KiB">{1}</memory>
|
||||
<clock offset="utc" />
|
||||
<clock>
|
||||
<timer name="rtc" tickpolicy="catchup" track="wall">
|
||||
<catchup limit="10000" slew="120" threshold="123" />
|
||||
</timer>
|
||||
</clock>
|
||||
<clock>
|
||||
<timer name="pit" tickpolicy="delay" />
|
||||
</clock>
|
||||
<clock>
|
||||
<timer name="hpet" present="yes" />
|
||||
</clock>
|
||||
<os>
|
||||
<type arch="{2}">{3}</type>
|
||||
<boot dev="{4}" />
|
||||
<boot dev="{5}" />
|
||||
</os>
|
||||
<devices>
|
||||
<controller model="nec-xhci" type="usb" />
|
||||
<emulator>test_emulator</emulator>
|
||||
<video>
|
||||
<model heads="1" type="vga" vram="9216" />
|
||||
</video>
|
||||
<serial type="pty">
|
||||
<target port="0" />
|
||||
</serial>
|
||||
<console type="pty">
|
||||
<target port="0" type="serial" />
|
||||
</console>
|
||||
</devices>
|
||||
</domain>'''.format(self.node.vcpu, str(self.node.memory * 1024),
|
||||
self.node.architecture, self.node.os_type,
|
||||
boot[0], boot[1])
|
||||
self.assertXMLIn(expected, xml)
|
||||
|
||||
@mock.patch('devops.driver.libvirt.libvirt_xml_builder.uuid')
|
||||
def test_node_devices(self, mock_uuid):
|
||||
mock_uuid.uuid4.return_value.hex = 'disk-serial'
|
||||
|
|
@ -413,9 +475,8 @@ class TestNodeXml(BaseTestXMLBuilder):
|
|||
bus='bus{0}'.format(i)
|
||||
) for i in range(3)
|
||||
]
|
||||
|
||||
self.node.disk_devices = disk_devices
|
||||
xml = self.xml_builder.build_node_xml(self.node, 'test_emulator')
|
||||
xml = self.xml_builder.build_node_xml(self.node, 'test_emulator', [])
|
||||
expected = '''
|
||||
<devices>
|
||||
<controller model="nec-xhci" type="usb" />
|
||||
|
|
@ -467,7 +528,7 @@ class TestNodeXml(BaseTestXMLBuilder):
|
|||
]
|
||||
|
||||
self.node.disk_devices = disk_devices + disk_devices
|
||||
xml = self.xml_builder.build_node_xml(self.node, 'test_emulator')
|
||||
xml = self.xml_builder.build_node_xml(self.node, 'test_emulator', [])
|
||||
expected = '''
|
||||
<devices>
|
||||
<controller model="nec-xhci" type="usb" />
|
||||
|
|
@ -535,7 +596,7 @@ class TestNodeXml(BaseTestXMLBuilder):
|
|||
mock.Mock(type='network', mac_address='mac{0}'.format(i),
|
||||
network=networks[i], id='id{0}'.format(i),
|
||||
model='model{0}'.format(i)) for i in range(3)]
|
||||
xml = self.xml_builder.build_node_xml(self.node, 'test_emulator')
|
||||
xml = self.xml_builder.build_node_xml(self.node, 'test_emulator', [])
|
||||
self.assertXMLIn('''
|
||||
<devices>
|
||||
<controller model="nec-xhci" type="usb" />
|
||||
|
|
|
|||
Loading…
Reference in New Issue