diff --git a/config.yaml b/config.yaml index cac7e953..048f3297 100644 --- a/config.yaml +++ b/config.yaml @@ -37,6 +37,15 @@ options: Port can also be a linuxbridge bridge. In this case a veth pair will be created, the ovs bridge and the linuxbridge bridge will be connected. It can be useful to connect the ovs bridge to juju bridge. + dpdk-bond-mappings: + type: string + default: + description: | + Space-delimited list of bond:port mappings. The DPDK assigned ports will + be added to their corresponding bond, which in turn will be put into the + bridge as specified in data-port. + . + This option is supported only when enable-dpdk is true. disable-security-groups: type: boolean default: false diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index 104cb96d..498b8e1f 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -240,7 +240,7 @@ class L3AgentContext(OSContextGenerator): return ctxt -def resolve_dpdk_ports(): +def resolve_dpdk_bridges(): ''' Resolve local PCI devices from configured mac addresses using the data-port configuration option @@ -269,6 +269,35 @@ def resolve_dpdk_ports(): return resolved_devices +def resolve_dpdk_bonds(): + ''' + Resolve local PCI devices from configured mac addresses + using the dpdk-bond-mappings configuration option + + @return: OrderDict indexed by PCI device address. + ''' + bonds = config('dpdk-bond-mappings') + devices = PCINetDevices() + resolved_devices = collections.OrderedDict() + db = kv() + if bonds: + # NOTE: ordered dict of format {[mac]: bond} + bondmap = parse_data_port_mappings(bonds) + for mac, bond in bondmap.items(): + pcidev = devices.get_device_from_mac(mac) + if pcidev: + # NOTE: store mac->pci allocation as post binding + # to dpdk, it disappears from PCIDevices. + db.set(mac, pcidev.pci_address) + db.flush() + + pci_address = db.get(mac) + if pci_address: + resolved_devices[pci_address] = bond + + return resolved_devices + + def parse_cpu_list(cpulist): ''' Parses a linux cpulist for a numa node @@ -304,7 +333,7 @@ class DPDKDeviceContext(OSContextGenerator): driver = config('dpdk-driver') if driver is None: return {} - return {'devices': resolve_dpdk_ports(), + return {'devices': resolve_dpdk_bridges(), 'driver': driver} @@ -340,7 +369,7 @@ class OVSDPDKDeviceContext(OSContextGenerator): '''Formatted list of devices to whitelist for dpdk''' _flag = '-w {device}' whitelist = [] - for device in resolve_dpdk_ports(): + for device in resolve_dpdk_bridges(): whitelist.append(_flag.format(device=device)) return ' '.join(whitelist) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index ca618c6e..130d0a29 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -426,14 +426,25 @@ def configure_ovs(): else: # NOTE: when in dpdk mode, add based on pci bus order # with type 'dpdk' - bridgemaps = neutron_ovs_context.resolve_dpdk_ports() + bridgemaps = neutron_ovs_context.resolve_dpdk_bridges() + bondmaps = neutron_ovs_context.resolve_dpdk_bonds() device_index = 0 + bridge_bond_map = DPDKBridgeBondMap() for pci_address, br in bridgemaps.items(): add_bridge(br, datapath_type) - dpdk_add_bridge_port(br, 'dpdk{}'.format(device_index), - pci_address) + portname = 'dpdk{}'.format(device_index) + if pci_address in bondmaps: + bond = bondmaps[pci_address] + bridge_bond_map.add_port(br, bond, portname, pci_address) + else: + dpdk_add_bridge_port(br, portname, + pci_address) device_index += 1 + for br, bonds in bridge_bond_map.items(): + for bond, t in bonds.items(): + dpdk_add_bridge_bond(br, bond, *t) + target = config('ipfix-target') bridges = [INT_BRIDGE, EXT_BRIDGE] if bridgemaps: @@ -606,6 +617,24 @@ def dpdk_add_bridge_port(name, port, pci_address=None): subprocess.check_call(cmd) +def dpdk_add_bridge_bond(bridge_name, bond_name, port_list, pci_address_list): + ''' Add ports to a bond attached to the named openvswitch bridge ''' + if ovs_has_late_dpdk_init(): + cmd = ["ovs-vsctl", "--may-exist", + "add-bond", bridge_name, bond_name] + for port in port_list: + cmd.append(port) + id = 0 + for pci_address in pci_address_list: + cmd.extend(["--", "set", "Interface", port_list[id], + "type=dpdk", + "options:dpdk-devargs={}".format(pci_address)]) + id += 1 + else: + raise Exception("Bond's not supported for OVS pre-2.6.0") + subprocess.check_call(cmd) + + def enable_nova_metadata(): return use_dvr() or enable_local_dhcp() @@ -685,3 +714,20 @@ def _pause_resume_helper(f, configs): f(assess_status_func(configs), services=services(), ports=None) + + +class DPDKBridgeBondMap(): + + def __init__(self): + self.map = {} + + def add_port(self, bridge, bond, portname, pci_address): + if bridge not in self.map: + self.map[bridge] = {} + if bond not in self.map[bridge]: + self.map[bridge][bond] = ([], []) + self.map[bridge][bond][0].append(portname) + self.map[bridge][bond][1].append(pci_address) + + def items(self): + return list(self.map.items()) diff --git a/unit_tests/test_neutron_ovs_context.py b/unit_tests/test_neutron_ovs_context.py index 912b0bb5..caa09fb0 100644 --- a/unit_tests/test_neutron_ovs_context.py +++ b/unit_tests/test_neutron_ovs_context.py @@ -501,6 +501,11 @@ DPDK_DATA_PORTS = ( "br-phynet1:fe:16:41:df:23:fd " "br-phynet2:fe:f2:d0:45:dc:66" ) +BOND_MAPPINGS = ( + "bond0:fe:16:41:df:23:fe " + "bond0:fe:16:41:df:23:fd " + "bond1:fe:f2:d0:45:dc:66" +) PCI_DEVICE_MAP = { 'fe:16:41:df:23:fd': MockPCIDevice('0000:00:1c.0'), 'fe:16:41:df:23:fe': MockPCIDevice('0000:00:1d.0'), @@ -534,20 +539,29 @@ class TestDPDKUtils(CharmTestCase): self.glob.glob.assert_called_with('/sys/devices/system/node/node*') _parse_cpu_list.assert_called_with(TEST_CPULIST_1) - def test_resolve_dpdk_ports(self): + def test_resolve_dpdk_bridges(self): self.test_config.set('data-port', DPDK_DATA_PORTS) _pci_devices = Mock() _pci_devices.get_device_from_mac.side_effect = PCI_DEVICE_MAP.get self.PCINetDevices.return_value = _pci_devices - self.assertEqual(context.resolve_dpdk_ports(), + self.assertEqual(context.resolve_dpdk_bridges(), {'0000:00:1c.0': 'br-phynet1', '0000:00:1d.0': 'br-phynet3'}) + def test_resolve_dpdk_bonds(self): + self.test_config.set('dpdk-bond-mappings', BOND_MAPPINGS) + _pci_devices = Mock() + _pci_devices.get_device_from_mac.side_effect = PCI_DEVICE_MAP.get + self.PCINetDevices.return_value = _pci_devices + self.assertEqual(context.resolve_dpdk_bonds(), + {'0000:00:1c.0': 'bond0', + '0000:00:1d.0': 'bond0'}) + DPDK_PATCH = [ 'parse_cpu_list', 'numa_node_cores', - 'resolve_dpdk_ports', + 'resolve_dpdk_bridges', 'glob', ] @@ -572,7 +586,7 @@ class TestOVSDPDKDeviceContext(CharmTestCase): def test_device_whitelist(self): '''Test device whitelist generation''' - self.resolve_dpdk_ports.return_value = [ + self.resolve_dpdk_bridges.return_value = [ '0000:00:1c.0', '0000:00:1d.0' ] @@ -606,12 +620,12 @@ class TestOVSDPDKDeviceContext(CharmTestCase): def test_context_no_devices(self): '''Ensure that DPDK is disable when no devices detected''' - self.resolve_dpdk_ports.return_value = [] + self.resolve_dpdk_bridges.return_value = [] self.assertEqual(self.test_context(), {}) def test_context_devices(self): '''Ensure DPDK is enabled when devices are detected''' - self.resolve_dpdk_ports.return_value = [ + self.resolve_dpdk_bridges.return_value = [ '0000:00:1c.0', '0000:00:1d.0' ] @@ -635,7 +649,7 @@ class TestDPDKDeviceContext(CharmTestCase): def test_context(self): self.test_config.set('dpdk-driver', 'uio_pci_generic') - self.resolve_dpdk_ports.return_value = [ + self.resolve_dpdk_bridges.return_value = [ '0000:00:1c.0', '0000:00:1d.0' ] @@ -646,7 +660,7 @@ class TestDPDKDeviceContext(CharmTestCase): self.config.assert_called_with('dpdk-driver') def test_context_none_driver(self): - self.resolve_dpdk_ports.return_value = [ + self.resolve_dpdk_bridges.return_value = [ '0000:00:1c.0', '0000:00:1d.0' ] diff --git a/unit_tests/test_neutron_ovs_utils.py b/unit_tests/test_neutron_ovs_utils.py index 3f4e16d8..95107bfe 100644 --- a/unit_tests/test_neutron_ovs_utils.py +++ b/unit_tests/test_neutron_ovs_utils.py @@ -34,6 +34,7 @@ TO_PATCH = [ 'add_ovsbridge_linuxbridge', 'is_linuxbridge_interface', 'dpdk_add_bridge_port', + 'dpdk_add_bridge_bond', 'apt_install', 'apt_update', 'config', @@ -473,12 +474,21 @@ class TestNeutronOVSUtils(CharmTestCase): self.add_bridge_port.assert_called_with('br-ex', 'eth0') def _run_configure_ovs_dpdk(self, mock_config, _use_dvr, - _resolve_dpdk_ports, _late_init): - _resolve_dpdk_ports.return_value = OrderedDict([ + _resolve_dpdk_bridges, _resolve_dpdk_bonds, + _late_init, _test_bonds): + _resolve_dpdk_bridges.return_value = OrderedDict([ ('0000:001c.01', 'br-phynet1'), ('0000:001c.02', 'br-phynet2'), ('0000:001c.03', 'br-phynet3'), ]) + if _test_bonds: + _resolve_dpdk_bonds.return_value = OrderedDict([ + ('0000:001c.01', 'bond0'), + ('0000:001c.02', 'bond1'), + ('0000:001c.03', 'bond2'), + ]) + else: + _resolve_dpdk_bonds.return_value = OrderedDict() _use_dvr.return_value = True self.use_dpdk.return_value = True self.ovs_has_late_dpdk_init.return_value = _late_init @@ -494,28 +504,59 @@ class TestNeutronOVSUtils(CharmTestCase): call('br-phynet3', 'netdev')], any_order=True ) - self.dpdk_add_bridge_port.assert_has_calls([ - call('br-phynet1', 'dpdk0', '0000:001c.01'), - call('br-phynet2', 'dpdk1', '0000:001c.02'), - call('br-phynet3', 'dpdk2', '0000:001c.03')], - any_order=True - ) + if _test_bonds: + self.dpdk_add_bridge_bond.assert_has_calls([ + call('br-phynet1', 'bond0', ['dpdk0'], ['0000:001c.01']), + call('br-phynet2', 'bond1', ['dpdk1'], ['0000:001c.02']), + call('br-phynet3', 'bond2', ['dpdk2'], ['0000:001c.03'])], + any_order=True + ) + else: + self.dpdk_add_bridge_port.assert_has_calls([ + call('br-phynet1', 'dpdk0', '0000:001c.01'), + call('br-phynet2', 'dpdk1', '0000:001c.02'), + call('br-phynet3', 'dpdk2', '0000:001c.03')], + any_order=True + ) - @patch.object(neutron_ovs_context, 'resolve_dpdk_ports') + @patch.object(neutron_ovs_context, 'resolve_dpdk_bonds') + @patch.object(neutron_ovs_context, 'resolve_dpdk_bridges') @patch.object(nutils, 'use_dvr') @patch('charmhelpers.contrib.openstack.context.config') def test_configure_ovs_dpdk(self, mock_config, _use_dvr, - _resolve_dpdk_ports): + _resolve_dpdk_bridges, + _resolve_dpdk_bonds): return self._run_configure_ovs_dpdk(mock_config, _use_dvr, - _resolve_dpdk_ports, False) + _resolve_dpdk_bridges, + _resolve_dpdk_bonds, + _late_init=False, + _test_bonds=False) - @patch.object(neutron_ovs_context, 'resolve_dpdk_ports') + @patch.object(neutron_ovs_context, 'resolve_dpdk_bonds') + @patch.object(neutron_ovs_context, 'resolve_dpdk_bridges') @patch.object(nutils, 'use_dvr') @patch('charmhelpers.contrib.openstack.context.config') def test_configure_ovs_dpdk_late_init(self, mock_config, _use_dvr, - _resolve_dpdk_ports): + _resolve_dpdk_bridges, + _resolve_dpdk_bonds): return self._run_configure_ovs_dpdk(mock_config, _use_dvr, - _resolve_dpdk_ports, True) + _resolve_dpdk_bridges, + _resolve_dpdk_bonds, + _late_init=True, + _test_bonds=False) + + @patch.object(neutron_ovs_context, 'resolve_dpdk_bonds') + @patch.object(neutron_ovs_context, 'resolve_dpdk_bridges') + @patch.object(nutils, 'use_dvr') + @patch('charmhelpers.contrib.openstack.context.config') + def test_configure_ovs_dpdk_late_init_bonds(self, mock_config, _use_dvr, + _resolve_dpdk_bridges, + _resolve_dpdk_bonds): + return self._run_configure_ovs_dpdk(mock_config, _use_dvr, + _resolve_dpdk_bridges, + _resolve_dpdk_bonds, + _late_init=True, + _test_bonds=True) @patch.object(nutils, 'use_dvr') @patch('charmhelpers.contrib.openstack.context.config') @@ -694,3 +735,30 @@ class TestNeutronOVSUtils(CharmTestCase): self.mock_sriov_device2.set_sriov_numvfs.assert_not_called() self.assertTrue(self.remote_restart.called) + + +class TestDPDKBridgeBondMap(CharmTestCase): + + def setUp(self): + super(TestDPDKBridgeBondMap, self).setUp(nutils, + TO_PATCH) + self.config.side_effect = self.test_config.get + + def test_add_port(self): + ctx = nutils.DPDKBridgeBondMap() + ctx.add_port("br1", "bond1", "port1", "00:00:00:00:00:01") + ctx.add_port("br1", "bond1", "port2", "00:00:00:00:00:02") + ctx.add_port("br1", "bond2", "port3", "00:00:00:00:00:03") + ctx.add_port("br1", "bond2", "port4", "00:00:00:00:00:04") + + expected = [('br1', + {'bond1': + (['port1', 'port2'], + ['00:00:00:00:00:01', '00:00:00:00:00:02']), + 'bond2': + (['port3', 'port4'], + ['00:00:00:00:00:03', '00:00:00:00:00:04']) + }) + ] + + self.assertEqual(ctx.items(), expected)