Allow to configure amount of PCIe ports

On x86-64/q35 and aarch64/virt instances libvirt adds as many
pcie-root-port entries (aka virtual pcie slots) as it needs and adds one
free. If we want to hotplug network interfaces or storage devices then
we quickly run out of available pcie slots.

This patch allows to configure amount of PCIe slots in instance. Method
was discussed with upstream libvirt developers.

To have requested amount of pcie-root-port entries we have to create
whole PCIe structure starting with pcie-root/0 and then add as many
pcie-root-port/0 entries as we want slots. Too low value may get bumped
by libvirt to same as amount of inserted cards.

Systems not using new option will work same way as they did.

Implements: bp configure-amount-of-pcie-ports

Change-Id: Ic3c8761bcde3e842d1b8e1feff1d158630de59ae

nova/conf/libvirt.py

@@ -636,6 +636,25 @@ Possible values:
   The supported events list can be found in
   https://libvirt.org/html/libvirt-libvirt-domain.html ,
   which you may need to search key words ``VIR_PERF_PARAM_*``
+"""),
+    cfg.IntOpt('num_pcie_ports',
+               default=0,
+               min=0,
+               max=28,
+               help= """
+The number of PCIe ports an instance will get.
+
+Libvirt allows a custom number of PCIe ports (pcie-root-port controllers) a
+target instance will get. Some will be used by default, rest will be available
+for hotplug use.
+
+By default we have just 1-2 free ports which limits hotplug.
+
+More info: https://github.com/qemu/qemu/blob/master/docs/pcie.txt
+
+Due to QEMU limitations for aarch64/virt maximum value is set to '28'.
+
+Default value '0' moves calculating amount of ports to libvirt.
 """),
 ]
 

nova/tests/unit/virt/libvirt/test_driver.py

@@ -2137,6 +2137,75 @@ class LibvirtConnTestCase(test.NoDBTestCase,
         self.assertEqual(33550336,
                          cfg.metadata[0].flavor.swap)
 
+    def test_get_guest_config_q35(self):
+        self.flags(virt_type="kvm",
+                   group='libvirt')
+
+        TEST_AMOUNT_OF_PCIE_SLOTS = 8
+        CONF.set_override("num_pcie_ports", TEST_AMOUNT_OF_PCIE_SLOTS,
+                group='libvirt')
+
+        drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
+        instance_ref = objects.Instance(**self.test_instance)
+        image_meta = objects.ImageMeta.from_dict({
+            "disk_format": "raw",
+            "properties": {"hw_machine_type":
+                           "pc-q35-test"}})
+
+        disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type,
+                                            instance_ref,
+                                            image_meta)
+
+        cfg = drvr._get_guest_config(instance_ref,
+                                     _fake_network_info(self, 1),
+                                     image_meta, disk_info)
+
+        num_ports = 0
+        for device in cfg.devices:
+            try:
+                if (device.root_name == 'controller' and
+                        device.model == 'pcie-root-port'):
+                    num_ports += 1
+            except AttributeError:
+                pass
+
+        self.assertEqual(TEST_AMOUNT_OF_PCIE_SLOTS, num_ports)
+
+    def test_get_guest_config_pcie_i440fx(self):
+        self.flags(virt_type="kvm",
+                   group='libvirt')
+
+        TEST_AMOUNT_OF_PCIE_SLOTS = 8
+        CONF.set_override("num_pcie_ports", TEST_AMOUNT_OF_PCIE_SLOTS,
+                group='libvirt')
+
+        drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
+        instance_ref = objects.Instance(**self.test_instance)
+        image_meta = objects.ImageMeta.from_dict({
+            "disk_format": "raw",
+            "properties": {"hw_machine_type":
+                           "pc-i440fx-test"}})
+
+        disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type,
+                                            instance_ref,
+                                            image_meta)
+
+        cfg = drvr._get_guest_config(instance_ref,
+                                     _fake_network_info(self, 1),
+                                     image_meta, disk_info)
+
+        num_ports = 0
+        for device in cfg.devices:
+            try:
+                if (device.root_name == 'controller' and
+                        device.model == 'pcie-root-port'):
+                    num_ports += 1
+            except AttributeError:
+                pass
+
+        # i440fx is not pcie machine so there should be no pcie ports
+        self.assertEqual(0, num_ports)
+
     def test_get_guest_config_missing_ownership_info(self):
         drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
 
@@ -5647,6 +5716,10 @@ class LibvirtConnTestCase(test.NoDBTestCase,
             caps.host.cpu = cpu
             return caps
 
+        TEST_AMOUNT_OF_PCIE_SLOTS = 8
+        CONF.set_override("num_pcie_ports", TEST_AMOUNT_OF_PCIE_SLOTS,
+                group='libvirt')
+
         self.flags(virt_type="kvm",
                    group="libvirt")
 
@@ -5669,6 +5742,17 @@ class LibvirtConnTestCase(test.NoDBTestCase,
             libvirt_driver.DEFAULT_UEFI_LOADER_PATH['aarch64'])
         self.assertEqual(cfg.os_mach_type, "virt")
 
+        num_ports = 0
+        for device in cfg.devices:
+            try:
+                if (device.root_name == 'controller' and
+                        device.model == 'pcie-root-port'):
+                    num_ports += 1
+            except AttributeError:
+                pass
+
+        self.assertEqual(TEST_AMOUNT_OF_PCIE_SLOTS, num_ports)
+
     @mock.patch.object(libvirt_driver.LibvirtDriver,
                        "_get_guest_storage_config")
     @mock.patch.object(libvirt_driver.LibvirtDriver, "_has_numa_support")

nova/virt/libvirt/config.py

@@ -1645,6 +1645,24 @@ class LibvirtConfigGuestUSBHostController(LibvirtConfigGuestController):
         self.type = 'usb'
 
 
+class LibvirtConfigGuestPCIeRootController(LibvirtConfigGuestController):
+
+    def __init__(self, **kwargs):
+        super(LibvirtConfigGuestPCIeRootController, self).\
+                __init__(**kwargs)
+        self.type = 'pci'
+        self.model = 'pcie-root'
+
+
+class LibvirtConfigGuestPCIeRootPortController(LibvirtConfigGuestController):
+
+    def __init__(self, **kwargs):
+        super(LibvirtConfigGuestPCIeRootPortController, self).\
+                __init__(**kwargs)
+        self.type = 'pci'
+        self.model = 'pcie-root-port'
+
+
 class LibvirtConfigGuestHostdev(LibvirtConfigGuestDevice):
     def __init__(self, **kwargs):
         super(LibvirtConfigGuestHostdev, self).\

nova/virt/libvirt/driver.py

@@ -5028,6 +5028,23 @@ class LibvirtDriver(driver.ComputeDriver):
                 cpu_config.features.add(xf)
         return cpu_config
 
+    def _guest_add_pcie_root_ports(self, guest):
+        """Add PCI Express root ports.
+
+        PCI Express machine can have as many PCIe devices as it has
+        pcie-root-port controllers (slots in virtual motherboard).
+
+        If we want to have more PCIe slots for hotplug then we need to create
+        whole PCIe structure (libvirt limitation).
+        """
+
+        pcieroot = vconfig.LibvirtConfigGuestPCIeRootController()
+        guest.add_device(pcieroot)
+
+        for x in range(0, CONF.libvirt.num_pcie_ports):
+            pcierootport = vconfig.LibvirtConfigGuestPCIeRootPortController()
+            guest.add_device(pcierootport)
+
     def _guest_add_usb_host_keyboard(self, guest):
         """Add USB Host controller and keyboard for graphical console use.
 
@@ -5158,6 +5175,16 @@ class LibvirtDriver(driver.ComputeDriver):
         if virt_type in ('qemu', 'kvm'):
             self._set_qemu_guest_agent(guest, flavor, instance, image_meta)
 
+        # Add PCIe root port controllers for PCI Express machines
+        # but only if their amount is configured
+        if (CONF.libvirt.num_pcie_ports and
+                ((caps.host.cpu.arch == fields.Architecture.AARCH64 and
+                guest.os_mach_type.startswith('virt')) or
+                (caps.host.cpu.arch == fields.Architecture.X86_64 and
+                guest.os_mach_type is not None and
+                'q35' in guest.os_mach_type))):
+            self._guest_add_pcie_root_ports(guest)
+
         self._guest_add_pci_devices(guest, instance)
 
         self._guest_add_watchdog_action(guest, flavor, image_meta)

releasenotes/notes/configure-amount-of-pcie-ports-486bfa44e9fbdd84.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    The amount of PCI Express ports (slots in virtual motherboard) can now be
+    configured using ``num_pcie_ports`` option in ``libvirt`` section of
+    ``nova.conf`` file.  This affects x86-64 with ``hw_machine_type`` set to
+    'pc-q35' value and AArch64 instances of 'virt' ``hw_machine_type`` (which
+    is default for that architecture). Due to QEMU's memory map limits on
+    aarch64/virt maximum value is limited to 28.