Configure switchdev mode in SR-IOV in os-net-config
While sriov vfs are created now using os-net-config, so doing some modification to configure switchdev mode there Change-Id: I7c85ab380dc4f0377a65fc954fcce3e3d39f5029 Closes-Bug: #1818034
This commit is contained in:
parent
3e3f928d19
commit
3dcad07311
|
@ -0,0 +1,84 @@
|
|||
{
|
||||
"network_config": [
|
||||
{
|
||||
"type": "sriov_pf",
|
||||
"name": "p2p1",
|
||||
"numvfs": 10,
|
||||
"use_dhcp": false,
|
||||
"promisc": true,
|
||||
"link_mode": "switchdev"
|
||||
},
|
||||
{
|
||||
"type": "ovs_bridge",
|
||||
"name": "br-pfs",
|
||||
"members": [
|
||||
{
|
||||
"type": "sriov_pf",
|
||||
"name": "p2p2",
|
||||
"numvfs": 10,
|
||||
"promisc": true,
|
||||
"use_dhcp": false,
|
||||
"link_mode": "switchdev"
|
||||
}
|
||||
],
|
||||
"use_dhcp": true
|
||||
},
|
||||
{
|
||||
"type": "ovs_bridge",
|
||||
"name": "br-bond",
|
||||
"use_dhcp": true,
|
||||
"members": [
|
||||
{
|
||||
"type": "ovs_bond",
|
||||
"name": "bond_pf",
|
||||
"ovs_options": "bond_mode=active-backup",
|
||||
"members": [
|
||||
{
|
||||
"type": "sriov_pf",
|
||||
"name": "p5p1",
|
||||
"numvfs": 10,
|
||||
"primary": true,
|
||||
"promisc": true,
|
||||
"use_dhcp": false,
|
||||
"link_mode": "switchdev"
|
||||
},
|
||||
{
|
||||
"type": "sriov_pf",
|
||||
"name": "p5p2",
|
||||
"numvfs": 10,
|
||||
"promisc": true,
|
||||
"use_dhcp": false,
|
||||
"link_mode": "switchdev"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "linux_bond",
|
||||
"name": "bond_lnx",
|
||||
"use_dhcp": true,
|
||||
"bonding_options": "mode=active-backup",
|
||||
"members": [
|
||||
{
|
||||
"type": "sriov_pf",
|
||||
"name": "p6p1",
|
||||
"numvfs": 10,
|
||||
"primary": true,
|
||||
"promisc": true,
|
||||
"use_dhcp": false,
|
||||
"link_mode": "switchdev"
|
||||
},
|
||||
{
|
||||
"type": "sriov_pf",
|
||||
"name": "p6p2",
|
||||
"numvfs": 10,
|
||||
"promisc": true,
|
||||
"use_dhcp": false,
|
||||
"link_mode": "switchdev"
|
||||
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
|
@ -0,0 +1,87 @@
|
|||
network_config:
|
||||
# sriov_pf type shall be used to configure the PF's of NICs.
|
||||
# The numvfs configured for the PF's shall be set on the sriov_numvfs of the
|
||||
# sysfs for the corresponding NIC and the persistence of the same across reboots
|
||||
# shall be handled.
|
||||
# The link_mode configured the mode of the sriov_pf which can be "switchdev"
|
||||
# or legacy.
|
||||
# In case fo switchdev, you will enable the hardware offloading and VF-LAG
|
||||
# capabilities
|
||||
# For now, if you configured link_mode with "switchdev" you will not be able
|
||||
# to use any of its sriov_vfs, the vfs will be only used for vms.
|
||||
-
|
||||
type: sriov_pf
|
||||
# nic name or nic number of the NIC that needs to be configured for SRIOV
|
||||
name: p2p1
|
||||
# number of VFs required on the particular NIC
|
||||
numvfs: 10
|
||||
addresses:
|
||||
- ip_netmask: 192.0.2.1/24
|
||||
# Allow all the traffic received. Default: true
|
||||
promisc: true
|
||||
# The mode of sriov_pf which:
|
||||
# - switchdev
|
||||
# - legacy (default)
|
||||
link_mode: switchdev
|
||||
|
||||
-
|
||||
type: ovs_bridge
|
||||
name: br-pfs
|
||||
use_dhcp: true
|
||||
members:
|
||||
-
|
||||
type: sriov_pf
|
||||
name: p2p2
|
||||
numvfs: 10
|
||||
promisc: true
|
||||
use_dhcp: false
|
||||
link_mode: switchdev
|
||||
|
||||
-
|
||||
type: ovs_bridge
|
||||
name: br-bond
|
||||
use_dhcp: true
|
||||
members:
|
||||
-
|
||||
type: ovs_bond
|
||||
name: bond_pf
|
||||
ovs_options: "bond_mode=active-backup"
|
||||
members:
|
||||
-
|
||||
type: sriov_pf
|
||||
name: p5p1
|
||||
numvfs: 10
|
||||
primary: true
|
||||
promisc: true
|
||||
use_dhcp: false
|
||||
link_mode: switchdev
|
||||
-
|
||||
type: sriov_pf
|
||||
name: p5p2
|
||||
numvfs: 10
|
||||
promisc: true
|
||||
use_dhcp: false
|
||||
link_mode: switchdev
|
||||
|
||||
-
|
||||
# VF-LAG capability, which means that the vfs will be bonded in hardware
|
||||
type: linux_bond
|
||||
name: bond_lnx
|
||||
use_dhcp: true
|
||||
bonding_options: "mode=active-backup"
|
||||
members:
|
||||
-
|
||||
type: sriov_pf
|
||||
name: p6p1
|
||||
numvfs: 10
|
||||
primary: true
|
||||
promisc: true
|
||||
use_dhcp: false
|
||||
link_mode: switchdev
|
||||
-
|
||||
type: sriov_pf
|
||||
name: p6p2
|
||||
numvfs: 10
|
||||
promisc: true
|
||||
use_dhcp: false
|
||||
link_mode: switchdev
|
|
@ -933,7 +933,8 @@ class IfcfgNetConfig(os_net_config.NetConfig):
|
|||
data = self._add_common(sriov_pf)
|
||||
logger.debug('sriov pf data: %s' % data)
|
||||
utils.update_sriov_pf_map(sriov_pf.name, sriov_pf.numvfs,
|
||||
self.noop, promisc=sriov_pf.promisc)
|
||||
self.noop, promisc=sriov_pf.promisc,
|
||||
link_mode=sriov_pf.link_mode)
|
||||
self.interface_data[sriov_pf.name] = data
|
||||
|
||||
def add_sriov_vf(self, sriov_vf):
|
||||
|
|
|
@ -1415,7 +1415,7 @@ class SriovPF(_BaseOpts):
|
|||
primary=False, nic_mapping=None, persist_mapping=False,
|
||||
defroute=True, dhclient_args=None, dns_servers=None,
|
||||
nm_controlled=False, onboot=True, domain=None, members=None,
|
||||
promisc=None):
|
||||
promisc=None, link_mode='legacy'):
|
||||
addresses = addresses or []
|
||||
routes = routes or []
|
||||
rules = rules or []
|
||||
|
@ -1432,6 +1432,7 @@ class SriovPF(_BaseOpts):
|
|||
else:
|
||||
self.name = name
|
||||
self.promisc = promisc
|
||||
self.link_mode = link_mode
|
||||
|
||||
@staticmethod
|
||||
def get_on_off(config):
|
||||
|
@ -1449,8 +1450,13 @@ class SriovPF(_BaseOpts):
|
|||
# SR-IOV PF - promisc: on (default)
|
||||
promisc = json.get('promisc', True)
|
||||
promisc = SriovPF.get_on_off(promisc)
|
||||
link_mode = json.get('link_mode', 'legacy')
|
||||
if link_mode not in ['legacy', 'switchdev']:
|
||||
msg = 'Expecting link_mode to match legacy/switchdev'
|
||||
raise InvalidConfigException(msg)
|
||||
opts = _BaseOpts.base_opts_from_json(json)
|
||||
return SriovPF(name, numvfs, *opts, promisc=promisc)
|
||||
return SriovPF(name, numvfs, *opts, promisc=promisc,
|
||||
link_mode=link_mode)
|
||||
|
||||
|
||||
class OvsDpdkBond(_BaseOpts):
|
||||
|
|
|
@ -33,6 +33,11 @@ definitions:
|
|||
sriov_vf_state_string:
|
||||
type: string
|
||||
pattern: "^(auto|enable|disable)$"
|
||||
sriov_link_mode_or_param:
|
||||
oneOf:
|
||||
- type: string
|
||||
pattern: "^(legacy|switchdev)$"
|
||||
- $ref: "#/definitions/param"
|
||||
# MAC address type
|
||||
mac_address_string:
|
||||
type: string
|
||||
|
@ -344,6 +349,8 @@ definitions:
|
|||
$ref: "#/definitions/bool_or_param"
|
||||
domain:
|
||||
$ref: "#/definitions/domain_name_string"
|
||||
link_mode:
|
||||
$ref: "#/definitions/sriov_link_mode_or_param"
|
||||
required:
|
||||
- type
|
||||
- name
|
||||
|
|
|
@ -25,6 +25,7 @@ import argparse
|
|||
import logging
|
||||
import os
|
||||
import pyudev
|
||||
import re
|
||||
from six.moves import queue as Queue
|
||||
import sys
|
||||
import yaml
|
||||
|
@ -32,6 +33,7 @@ import yaml
|
|||
from oslo_concurrency import processutils
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_SYS_CLASS_NET = '/sys/class/net'
|
||||
# Create a queue for passing the udev network events
|
||||
vf_queue = Queue.Queue()
|
||||
|
||||
|
@ -94,6 +96,8 @@ def configure_sriov_pf():
|
|||
observer.start()
|
||||
|
||||
sriov_map = _get_sriov_map()
|
||||
MLNX_UNBIND_FILE_PATH = "/sys/bus/pci/drivers/mlx5_core/unbind"
|
||||
MLNX_VENDOR_ID = "0x15b3"
|
||||
for item in sriov_map:
|
||||
if item['device_type'] == 'pf':
|
||||
_pf_interface_up(item)
|
||||
|
@ -108,6 +112,19 @@ def configure_sriov_pf():
|
|||
raise SRIOVNumvfsException(msg)
|
||||
# Wait for the creation of VFs for each PF
|
||||
_wait_for_vf_creation(item['name'], item['numvfs'])
|
||||
# Configure switchdev mode
|
||||
vendor_id = get_vendor_id(item['name'])
|
||||
if (item.get('link_mode') == "switchdev" and
|
||||
vendor_id == MLNX_VENDOR_ID):
|
||||
vf_pcis_list = get_vf_pcis_list(item['name'])
|
||||
for vf_pci in vf_pcis_list:
|
||||
vf_pci_path = "/sys/bus/pci/devices/%s/driver" % vf_pci
|
||||
if os.path.exists(vf_pci_path):
|
||||
with open(MLNX_UNBIND_FILE_PATH, 'w') as f:
|
||||
f.write("%s" % vf_pci)
|
||||
configure_switchdev(item['name'])
|
||||
if_up_interface(item['name'])
|
||||
|
||||
observer.stop()
|
||||
|
||||
|
||||
|
@ -139,6 +156,30 @@ def _wait_for_vf_creation(pf_name, numvfs):
|
|||
logger.info("Required VFs are created for PF %s" % pf_name)
|
||||
|
||||
|
||||
def configure_switchdev(pf_name):
|
||||
pf_pci = get_pf_pci(pf_name)
|
||||
pf_device_id = get_pf_device_id(pf_name)
|
||||
if pf_device_id == "0x1013" or pf_device_id == "0x1015":
|
||||
try:
|
||||
processutils.execute('/usr/sbin/devlink', 'dev', 'eswitch', 'set',
|
||||
'pci/%s' % pf_pci, 'inline-mode', 'transport')
|
||||
except processutils.ProcessExecutionError:
|
||||
logger.error("Failed to set inline-mode to transport")
|
||||
raise
|
||||
try:
|
||||
processutils.execute('/usr/sbin/devlink', 'dev', 'eswitch', 'set',
|
||||
'pci/%s' % pf_pci, 'mode', 'switchdev')
|
||||
except processutils.ProcessExecutionError:
|
||||
logger.error("Failed to set mode to switchdev")
|
||||
raise
|
||||
try:
|
||||
processutils.execute('/usr/sbin/ethtool', '-K', pf_name,
|
||||
'hw-tc-offload', 'on')
|
||||
except processutils.ProcessExecutionError:
|
||||
logger.error("Failed to enable hw-tc-offload")
|
||||
raise
|
||||
|
||||
|
||||
def run_ip_config_cmd(*cmd, **kwargs):
|
||||
logger.info("Running %s" % ' '.join(cmd))
|
||||
try:
|
||||
|
@ -156,6 +197,51 @@ def _pf_interface_up(pf_device):
|
|||
run_ip_config_cmd('ip', 'link', 'set', 'dev', pf_device['name'], 'up')
|
||||
|
||||
|
||||
def get_vendor_id(ifname):
|
||||
try:
|
||||
with open(os.path.join(_SYS_CLASS_NET, ifname, "device/vendor"),
|
||||
'r') as f:
|
||||
out = f.read().strip()
|
||||
return out
|
||||
except IOError:
|
||||
return
|
||||
|
||||
|
||||
def get_pf_pci(pf_name):
|
||||
pf_pci_path = os.path.join(_SYS_CLASS_NET, pf_name, "device/uevent")
|
||||
pf_info = get_file_data(pf_pci_path)
|
||||
pf_pci = re.search(r'PCI_SLOT_NAME=(.*)', pf_info, re.MULTILINE).group(1)
|
||||
return pf_pci
|
||||
|
||||
|
||||
def get_pf_device_id(pf_name):
|
||||
pf_device_path = os.path.join(_SYS_CLASS_NET, pf_name, "device/device")
|
||||
pf_device_id = get_file_data(pf_device_path).strip()
|
||||
return pf_device_id
|
||||
|
||||
|
||||
def get_vf_pcis_list(pf_name):
|
||||
vf_pcis_list = []
|
||||
listOfPfFiles = os.listdir(os.path.join(_SYS_CLASS_NET, pf_name,
|
||||
"device"))
|
||||
for pf_file in listOfPfFiles:
|
||||
if pf_file.startswith("virtfn"):
|
||||
vf_info = get_file_data(os.path.join(_SYS_CLASS_NET, pf_name,
|
||||
"device", pf_file, "uevent"))
|
||||
vf_pcis_list.append(re.search(r'PCI_SLOT_NAME=(.*)',
|
||||
vf_info, re.MULTILINE).group(1))
|
||||
return vf_pcis_list
|
||||
|
||||
|
||||
def if_up_interface(device):
|
||||
logger.info("Running /sbin/ifup %s" % device)
|
||||
try:
|
||||
processutils.execute('/sbin/ifup', device)
|
||||
except processutils.ProcessExecutionError:
|
||||
logger.error("Failed to ifup %s" % device)
|
||||
raise
|
||||
|
||||
|
||||
def configure_sriov_vf():
|
||||
sriov_map = _get_sriov_map()
|
||||
for item in sriov_map:
|
||||
|
|
|
@ -1386,10 +1386,12 @@ NETMASK=255.255.255.0
|
|||
|
||||
pf = objects.SriovPF(name='nic3', numvfs=10)
|
||||
|
||||
def test_update_sriov_pf_map(name, numvfs, noop, promisc=None):
|
||||
def test_update_sriov_pf_map(name, numvfs, noop, promisc=None,
|
||||
link_mode='legacy'):
|
||||
self.assertEqual(name, 'eth2')
|
||||
self.assertEqual(numvfs, 10)
|
||||
self.assertEqual(promisc, None)
|
||||
self.assertEqual(link_mode, 'legacy')
|
||||
self.stub_out('os_net_config.utils.update_sriov_pf_map',
|
||||
test_update_sriov_pf_map)
|
||||
self.provider.add_sriov_pf(pf)
|
||||
|
@ -1409,10 +1411,12 @@ BOOTPROTO=none
|
|||
|
||||
pf = objects.SriovPF(name='nic3', numvfs=10, promisc=True)
|
||||
|
||||
def test_update_sriov_pf_map(name, numvfs, noop, promisc=None):
|
||||
def test_update_sriov_pf_map(name, numvfs, noop, promisc=None,
|
||||
link_mode='legacy'):
|
||||
self.assertEqual(name, 'eth2')
|
||||
self.assertEqual(numvfs, 10)
|
||||
self.assertTrue(promisc)
|
||||
self.assertEqual(link_mode, 'legacy')
|
||||
self.stub_out('os_net_config.utils.update_sriov_pf_map',
|
||||
test_update_sriov_pf_map)
|
||||
self.provider.add_sriov_pf(pf)
|
||||
|
@ -1432,10 +1436,12 @@ BOOTPROTO=none
|
|||
|
||||
pf = objects.SriovPF(name='nic3', numvfs=10, promisc=False)
|
||||
|
||||
def test_update_sriov_pf_map(name, numvfs, noop, promisc=None):
|
||||
def test_update_sriov_pf_map(name, numvfs, noop, promisc=None,
|
||||
link_mode='legacy'):
|
||||
self.assertEqual(name, 'eth2')
|
||||
self.assertEqual(numvfs, 10)
|
||||
self.assertFalse(promisc)
|
||||
self.assertEqual(link_mode, 'legacy')
|
||||
self.stub_out('os_net_config.utils.update_sriov_pf_map',
|
||||
test_update_sriov_pf_map)
|
||||
self.provider.add_sriov_pf(pf)
|
||||
|
|
|
@ -1726,6 +1726,7 @@ class TestSriovPF(base.TestCase):
|
|||
self.assertEqual(16, pf.numvfs)
|
||||
self.assertEqual("off", pf.promisc)
|
||||
self.assertFalse(pf.use_dhcp)
|
||||
self.assertEqual("legacy", pf.link_mode)
|
||||
|
||||
def test_from_json_numvfs_nic1(self):
|
||||
def dummy_mapped_nics(nic_mapping=None):
|
||||
|
@ -1753,6 +1754,27 @@ class TestSriovPF(base.TestCase):
|
|||
self.assertFalse(pf.use_dhcp)
|
||||
self.assertEqual('on', pf.promisc)
|
||||
|
||||
def test_from_json_link_mode(self):
|
||||
data = '{"type": "sriov_pf", "name": "p6p1", "numvfs": 8,' \
|
||||
'"use_dhcp": false, "promisc": false, "link_mode":' \
|
||||
'"switchdev"}'
|
||||
pf = objects.object_from_json(json.loads(data))
|
||||
self.assertEqual("p6p1", pf.name)
|
||||
self.assertEqual(8, pf.numvfs)
|
||||
self.assertEqual("off", pf.promisc)
|
||||
self.assertFalse(pf.use_dhcp)
|
||||
self.assertEqual("switchdev", pf.link_mode)
|
||||
|
||||
def test_from_json_link_mode_invalid(self):
|
||||
data = '{"type": "sriov_pf", "name": "p6p1", "numvfs": 8,' \
|
||||
'"use_dhcp": false, "promisc": false, "link_mode":' \
|
||||
'"none_switchdev"}'
|
||||
err = self.assertRaises(objects.InvalidConfigException,
|
||||
objects.object_from_json,
|
||||
json.loads(data))
|
||||
expected = 'Expecting link_mode to match legacy/switchdev'
|
||||
self.assertIn(expected, six.text_type(err))
|
||||
|
||||
|
||||
class TestSriovVF(base.TestCase):
|
||||
|
||||
|
|
|
@ -123,8 +123,8 @@ class TestUtils(base.TestCase):
|
|||
contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE)
|
||||
sriov_pf_map = yaml.load(contents) if contents else []
|
||||
self.assertEqual(1, len(sriov_pf_map))
|
||||
test_sriov_pf_map = [{'device_type': 'pf', 'name': 'eth1',
|
||||
'numvfs': 10}]
|
||||
test_sriov_pf_map = [{'device_type': 'pf', 'link_mode': 'legacy',
|
||||
'name': 'eth1', 'numvfs': 10}]
|
||||
self.assertListEqual(test_sriov_pf_map, sriov_pf_map)
|
||||
|
||||
def test_update_sriov_pf_map_new_with_promisc(self):
|
||||
|
@ -132,16 +132,18 @@ class TestUtils(base.TestCase):
|
|||
contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE)
|
||||
sriov_pf_map = yaml.load(contents) if contents else []
|
||||
self.assertEqual(1, len(sriov_pf_map))
|
||||
test_sriov_pf_map = [{'device_type': 'pf', 'name': 'eth1',
|
||||
'numvfs': 10, 'promisc': 'off'}]
|
||||
test_sriov_pf_map = [{'device_type': 'pf', 'link_mode': 'legacy',
|
||||
'name': 'eth1', 'numvfs': 10, 'promisc': 'off'}]
|
||||
self.assertListEqual(test_sriov_pf_map, sriov_pf_map)
|
||||
|
||||
def test_update_sriov_pf_map_exist(self):
|
||||
pf_initial = [{'device_type': 'pf', 'name': 'eth1', 'numvfs': 10}]
|
||||
pf_initial = [{'device_type': 'pf', 'link_mode': 'legacy',
|
||||
'name': 'eth1', 'numvfs': 10}]
|
||||
utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_initial)
|
||||
|
||||
utils.update_sriov_pf_map('eth1', 20, False)
|
||||
pf_final = [{'device_type': 'pf', 'name': 'eth1', 'numvfs': 20}]
|
||||
pf_final = [{'device_type': 'pf', 'link_mode': 'legacy',
|
||||
'name': 'eth1', 'numvfs': 20}]
|
||||
contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE)
|
||||
|
||||
pf_map = yaml.load(contents) if contents else []
|
||||
|
@ -149,13 +151,13 @@ class TestUtils(base.TestCase):
|
|||
self.assertListEqual(pf_final, pf_map)
|
||||
|
||||
def test_update_sriov_pf_map_exist_with_promisc(self):
|
||||
pf_initial = [{'device_type': 'pf', 'name': 'eth1', 'numvfs': 10,
|
||||
'promisc': 'on'}]
|
||||
pf_initial = [{'device_type': 'pf', 'link_mode': 'legacy',
|
||||
'name': 'eth1', 'numvfs': 10, 'promisc': 'on'}]
|
||||
utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_initial)
|
||||
|
||||
utils.update_sriov_pf_map('eth1', 20, False)
|
||||
pf_final = [{'device_type': 'pf', 'name': 'eth1', 'numvfs': 20,
|
||||
'promisc': 'on'}]
|
||||
pf_final = [{'device_type': 'pf', 'link_mode': 'legacy',
|
||||
'name': 'eth1', 'numvfs': 20, 'promisc': 'on'}]
|
||||
contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE)
|
||||
|
||||
pf_map = yaml.load(contents) if contents else []
|
||||
|
|
|
@ -414,7 +414,8 @@ def _get_dpdk_mac_address(name):
|
|||
return item['mac_address']
|
||||
|
||||
|
||||
def update_sriov_pf_map(ifname, numvfs, noop, promisc=None):
|
||||
def update_sriov_pf_map(ifname, numvfs, noop, promisc=None,
|
||||
link_mode='legacy'):
|
||||
if not noop:
|
||||
sriov_map = _get_sriov_map()
|
||||
for item in sriov_map:
|
||||
|
@ -422,6 +423,7 @@ def update_sriov_pf_map(ifname, numvfs, noop, promisc=None):
|
|||
item['numvfs'] = numvfs
|
||||
if promisc is not None:
|
||||
item['promisc'] = promisc
|
||||
item['link_mode'] = link_mode
|
||||
break
|
||||
else:
|
||||
new_item = {}
|
||||
|
@ -430,6 +432,7 @@ def update_sriov_pf_map(ifname, numvfs, noop, promisc=None):
|
|||
new_item['numvfs'] = numvfs
|
||||
if promisc is not None:
|
||||
new_item['promisc'] = promisc
|
||||
new_item['link_mode'] = link_mode
|
||||
sriov_map.append(new_item)
|
||||
|
||||
write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, sriov_map)
|
||||
|
|
Loading…
Reference in New Issue