Configure switchdev mode in SR-IOV in os-net-config

While sriov vfs are created now using os-net-config,
so doing some modification to configure switchdev mode there

Change-Id: I7c85ab380dc4f0377a65fc954fcce3e3d39f5029
Closes-Bug: #1818034
This commit is contained in:
waleed mousa 2019-01-30 02:39:06 -05:00
parent 3e3f928d19
commit 3dcad07311
10 changed files with 321 additions and 17 deletions

View File

@ -0,0 +1,84 @@
{
"network_config": [
{
"type": "sriov_pf",
"name": "p2p1",
"numvfs": 10,
"use_dhcp": false,
"promisc": true,
"link_mode": "switchdev"
},
{
"type": "ovs_bridge",
"name": "br-pfs",
"members": [
{
"type": "sriov_pf",
"name": "p2p2",
"numvfs": 10,
"promisc": true,
"use_dhcp": false,
"link_mode": "switchdev"
}
],
"use_dhcp": true
},
{
"type": "ovs_bridge",
"name": "br-bond",
"use_dhcp": true,
"members": [
{
"type": "ovs_bond",
"name": "bond_pf",
"ovs_options": "bond_mode=active-backup",
"members": [
{
"type": "sriov_pf",
"name": "p5p1",
"numvfs": 10,
"primary": true,
"promisc": true,
"use_dhcp": false,
"link_mode": "switchdev"
},
{
"type": "sriov_pf",
"name": "p5p2",
"numvfs": 10,
"promisc": true,
"use_dhcp": false,
"link_mode": "switchdev"
}
]
}
]
},
{
"type": "linux_bond",
"name": "bond_lnx",
"use_dhcp": true,
"bonding_options": "mode=active-backup",
"members": [
{
"type": "sriov_pf",
"name": "p6p1",
"numvfs": 10,
"primary": true,
"promisc": true,
"use_dhcp": false,
"link_mode": "switchdev"
},
{
"type": "sriov_pf",
"name": "p6p2",
"numvfs": 10,
"promisc": true,
"use_dhcp": false,
"link_mode": "switchdev"
}
]
}
]
}

View File

@ -0,0 +1,87 @@
network_config:
# sriov_pf type shall be used to configure the PF's of NICs.
# The numvfs configured for the PF's shall be set on the sriov_numvfs of the
# sysfs for the corresponding NIC and the persistence of the same across reboots
# shall be handled.
# The link_mode configured the mode of the sriov_pf which can be "switchdev"
# or legacy.
# In case fo switchdev, you will enable the hardware offloading and VF-LAG
# capabilities
# For now, if you configured link_mode with "switchdev" you will not be able
# to use any of its sriov_vfs, the vfs will be only used for vms.
-
type: sriov_pf
# nic name or nic number of the NIC that needs to be configured for SRIOV
name: p2p1
# number of VFs required on the particular NIC
numvfs: 10
addresses:
- ip_netmask: 192.0.2.1/24
# Allow all the traffic received. Default: true
promisc: true
# The mode of sriov_pf which:
# - switchdev
# - legacy (default)
link_mode: switchdev
-
type: ovs_bridge
name: br-pfs
use_dhcp: true
members:
-
type: sriov_pf
name: p2p2
numvfs: 10
promisc: true
use_dhcp: false
link_mode: switchdev
-
type: ovs_bridge
name: br-bond
use_dhcp: true
members:
-
type: ovs_bond
name: bond_pf
ovs_options: "bond_mode=active-backup"
members:
-
type: sriov_pf
name: p5p1
numvfs: 10
primary: true
promisc: true
use_dhcp: false
link_mode: switchdev
-
type: sriov_pf
name: p5p2
numvfs: 10
promisc: true
use_dhcp: false
link_mode: switchdev
-
# VF-LAG capability, which means that the vfs will be bonded in hardware
type: linux_bond
name: bond_lnx
use_dhcp: true
bonding_options: "mode=active-backup"
members:
-
type: sriov_pf
name: p6p1
numvfs: 10
primary: true
promisc: true
use_dhcp: false
link_mode: switchdev
-
type: sriov_pf
name: p6p2
numvfs: 10
promisc: true
use_dhcp: false
link_mode: switchdev

View File

@ -933,7 +933,8 @@ class IfcfgNetConfig(os_net_config.NetConfig):
data = self._add_common(sriov_pf)
logger.debug('sriov pf data: %s' % data)
utils.update_sriov_pf_map(sriov_pf.name, sriov_pf.numvfs,
self.noop, promisc=sriov_pf.promisc)
self.noop, promisc=sriov_pf.promisc,
link_mode=sriov_pf.link_mode)
self.interface_data[sriov_pf.name] = data
def add_sriov_vf(self, sriov_vf):

View File

@ -1415,7 +1415,7 @@ class SriovPF(_BaseOpts):
primary=False, nic_mapping=None, persist_mapping=False,
defroute=True, dhclient_args=None, dns_servers=None,
nm_controlled=False, onboot=True, domain=None, members=None,
promisc=None):
promisc=None, link_mode='legacy'):
addresses = addresses or []
routes = routes or []
rules = rules or []
@ -1432,6 +1432,7 @@ class SriovPF(_BaseOpts):
else:
self.name = name
self.promisc = promisc
self.link_mode = link_mode
@staticmethod
def get_on_off(config):
@ -1449,8 +1450,13 @@ class SriovPF(_BaseOpts):
# SR-IOV PF - promisc: on (default)
promisc = json.get('promisc', True)
promisc = SriovPF.get_on_off(promisc)
link_mode = json.get('link_mode', 'legacy')
if link_mode not in ['legacy', 'switchdev']:
msg = 'Expecting link_mode to match legacy/switchdev'
raise InvalidConfigException(msg)
opts = _BaseOpts.base_opts_from_json(json)
return SriovPF(name, numvfs, *opts, promisc=promisc)
return SriovPF(name, numvfs, *opts, promisc=promisc,
link_mode=link_mode)
class OvsDpdkBond(_BaseOpts):

View File

@ -33,6 +33,11 @@ definitions:
sriov_vf_state_string:
type: string
pattern: "^(auto|enable|disable)$"
sriov_link_mode_or_param:
oneOf:
- type: string
pattern: "^(legacy|switchdev)$"
- $ref: "#/definitions/param"
# MAC address type
mac_address_string:
type: string
@ -344,6 +349,8 @@ definitions:
$ref: "#/definitions/bool_or_param"
domain:
$ref: "#/definitions/domain_name_string"
link_mode:
$ref: "#/definitions/sriov_link_mode_or_param"
required:
- type
- name

View File

@ -25,6 +25,7 @@ import argparse
import logging
import os
import pyudev
import re
from six.moves import queue as Queue
import sys
import yaml
@ -32,6 +33,7 @@ import yaml
from oslo_concurrency import processutils
logger = logging.getLogger(__name__)
_SYS_CLASS_NET = '/sys/class/net'
# Create a queue for passing the udev network events
vf_queue = Queue.Queue()
@ -94,6 +96,8 @@ def configure_sriov_pf():
observer.start()
sriov_map = _get_sriov_map()
MLNX_UNBIND_FILE_PATH = "/sys/bus/pci/drivers/mlx5_core/unbind"
MLNX_VENDOR_ID = "0x15b3"
for item in sriov_map:
if item['device_type'] == 'pf':
_pf_interface_up(item)
@ -108,6 +112,19 @@ def configure_sriov_pf():
raise SRIOVNumvfsException(msg)
# Wait for the creation of VFs for each PF
_wait_for_vf_creation(item['name'], item['numvfs'])
# Configure switchdev mode
vendor_id = get_vendor_id(item['name'])
if (item.get('link_mode') == "switchdev" and
vendor_id == MLNX_VENDOR_ID):
vf_pcis_list = get_vf_pcis_list(item['name'])
for vf_pci in vf_pcis_list:
vf_pci_path = "/sys/bus/pci/devices/%s/driver" % vf_pci
if os.path.exists(vf_pci_path):
with open(MLNX_UNBIND_FILE_PATH, 'w') as f:
f.write("%s" % vf_pci)
configure_switchdev(item['name'])
if_up_interface(item['name'])
observer.stop()
@ -139,6 +156,30 @@ def _wait_for_vf_creation(pf_name, numvfs):
logger.info("Required VFs are created for PF %s" % pf_name)
def configure_switchdev(pf_name):
pf_pci = get_pf_pci(pf_name)
pf_device_id = get_pf_device_id(pf_name)
if pf_device_id == "0x1013" or pf_device_id == "0x1015":
try:
processutils.execute('/usr/sbin/devlink', 'dev', 'eswitch', 'set',
'pci/%s' % pf_pci, 'inline-mode', 'transport')
except processutils.ProcessExecutionError:
logger.error("Failed to set inline-mode to transport")
raise
try:
processutils.execute('/usr/sbin/devlink', 'dev', 'eswitch', 'set',
'pci/%s' % pf_pci, 'mode', 'switchdev')
except processutils.ProcessExecutionError:
logger.error("Failed to set mode to switchdev")
raise
try:
processutils.execute('/usr/sbin/ethtool', '-K', pf_name,
'hw-tc-offload', 'on')
except processutils.ProcessExecutionError:
logger.error("Failed to enable hw-tc-offload")
raise
def run_ip_config_cmd(*cmd, **kwargs):
logger.info("Running %s" % ' '.join(cmd))
try:
@ -156,6 +197,51 @@ def _pf_interface_up(pf_device):
run_ip_config_cmd('ip', 'link', 'set', 'dev', pf_device['name'], 'up')
def get_vendor_id(ifname):
try:
with open(os.path.join(_SYS_CLASS_NET, ifname, "device/vendor"),
'r') as f:
out = f.read().strip()
return out
except IOError:
return
def get_pf_pci(pf_name):
pf_pci_path = os.path.join(_SYS_CLASS_NET, pf_name, "device/uevent")
pf_info = get_file_data(pf_pci_path)
pf_pci = re.search(r'PCI_SLOT_NAME=(.*)', pf_info, re.MULTILINE).group(1)
return pf_pci
def get_pf_device_id(pf_name):
pf_device_path = os.path.join(_SYS_CLASS_NET, pf_name, "device/device")
pf_device_id = get_file_data(pf_device_path).strip()
return pf_device_id
def get_vf_pcis_list(pf_name):
vf_pcis_list = []
listOfPfFiles = os.listdir(os.path.join(_SYS_CLASS_NET, pf_name,
"device"))
for pf_file in listOfPfFiles:
if pf_file.startswith("virtfn"):
vf_info = get_file_data(os.path.join(_SYS_CLASS_NET, pf_name,
"device", pf_file, "uevent"))
vf_pcis_list.append(re.search(r'PCI_SLOT_NAME=(.*)',
vf_info, re.MULTILINE).group(1))
return vf_pcis_list
def if_up_interface(device):
logger.info("Running /sbin/ifup %s" % device)
try:
processutils.execute('/sbin/ifup', device)
except processutils.ProcessExecutionError:
logger.error("Failed to ifup %s" % device)
raise
def configure_sriov_vf():
sriov_map = _get_sriov_map()
for item in sriov_map:

View File

@ -1386,10 +1386,12 @@ NETMASK=255.255.255.0
pf = objects.SriovPF(name='nic3', numvfs=10)
def test_update_sriov_pf_map(name, numvfs, noop, promisc=None):
def test_update_sriov_pf_map(name, numvfs, noop, promisc=None,
link_mode='legacy'):
self.assertEqual(name, 'eth2')
self.assertEqual(numvfs, 10)
self.assertEqual(promisc, None)
self.assertEqual(link_mode, 'legacy')
self.stub_out('os_net_config.utils.update_sriov_pf_map',
test_update_sriov_pf_map)
self.provider.add_sriov_pf(pf)
@ -1409,10 +1411,12 @@ BOOTPROTO=none
pf = objects.SriovPF(name='nic3', numvfs=10, promisc=True)
def test_update_sriov_pf_map(name, numvfs, noop, promisc=None):
def test_update_sriov_pf_map(name, numvfs, noop, promisc=None,
link_mode='legacy'):
self.assertEqual(name, 'eth2')
self.assertEqual(numvfs, 10)
self.assertTrue(promisc)
self.assertEqual(link_mode, 'legacy')
self.stub_out('os_net_config.utils.update_sriov_pf_map',
test_update_sriov_pf_map)
self.provider.add_sriov_pf(pf)
@ -1432,10 +1436,12 @@ BOOTPROTO=none
pf = objects.SriovPF(name='nic3', numvfs=10, promisc=False)
def test_update_sriov_pf_map(name, numvfs, noop, promisc=None):
def test_update_sriov_pf_map(name, numvfs, noop, promisc=None,
link_mode='legacy'):
self.assertEqual(name, 'eth2')
self.assertEqual(numvfs, 10)
self.assertFalse(promisc)
self.assertEqual(link_mode, 'legacy')
self.stub_out('os_net_config.utils.update_sriov_pf_map',
test_update_sriov_pf_map)
self.provider.add_sriov_pf(pf)

View File

@ -1726,6 +1726,7 @@ class TestSriovPF(base.TestCase):
self.assertEqual(16, pf.numvfs)
self.assertEqual("off", pf.promisc)
self.assertFalse(pf.use_dhcp)
self.assertEqual("legacy", pf.link_mode)
def test_from_json_numvfs_nic1(self):
def dummy_mapped_nics(nic_mapping=None):
@ -1753,6 +1754,27 @@ class TestSriovPF(base.TestCase):
self.assertFalse(pf.use_dhcp)
self.assertEqual('on', pf.promisc)
def test_from_json_link_mode(self):
data = '{"type": "sriov_pf", "name": "p6p1", "numvfs": 8,' \
'"use_dhcp": false, "promisc": false, "link_mode":' \
'"switchdev"}'
pf = objects.object_from_json(json.loads(data))
self.assertEqual("p6p1", pf.name)
self.assertEqual(8, pf.numvfs)
self.assertEqual("off", pf.promisc)
self.assertFalse(pf.use_dhcp)
self.assertEqual("switchdev", pf.link_mode)
def test_from_json_link_mode_invalid(self):
data = '{"type": "sriov_pf", "name": "p6p1", "numvfs": 8,' \
'"use_dhcp": false, "promisc": false, "link_mode":' \
'"none_switchdev"}'
err = self.assertRaises(objects.InvalidConfigException,
objects.object_from_json,
json.loads(data))
expected = 'Expecting link_mode to match legacy/switchdev'
self.assertIn(expected, six.text_type(err))
class TestSriovVF(base.TestCase):

View File

@ -123,8 +123,8 @@ class TestUtils(base.TestCase):
contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE)
sriov_pf_map = yaml.load(contents) if contents else []
self.assertEqual(1, len(sriov_pf_map))
test_sriov_pf_map = [{'device_type': 'pf', 'name': 'eth1',
'numvfs': 10}]
test_sriov_pf_map = [{'device_type': 'pf', 'link_mode': 'legacy',
'name': 'eth1', 'numvfs': 10}]
self.assertListEqual(test_sriov_pf_map, sriov_pf_map)
def test_update_sriov_pf_map_new_with_promisc(self):
@ -132,16 +132,18 @@ class TestUtils(base.TestCase):
contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE)
sriov_pf_map = yaml.load(contents) if contents else []
self.assertEqual(1, len(sriov_pf_map))
test_sriov_pf_map = [{'device_type': 'pf', 'name': 'eth1',
'numvfs': 10, 'promisc': 'off'}]
test_sriov_pf_map = [{'device_type': 'pf', 'link_mode': 'legacy',
'name': 'eth1', 'numvfs': 10, 'promisc': 'off'}]
self.assertListEqual(test_sriov_pf_map, sriov_pf_map)
def test_update_sriov_pf_map_exist(self):
pf_initial = [{'device_type': 'pf', 'name': 'eth1', 'numvfs': 10}]
pf_initial = [{'device_type': 'pf', 'link_mode': 'legacy',
'name': 'eth1', 'numvfs': 10}]
utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_initial)
utils.update_sriov_pf_map('eth1', 20, False)
pf_final = [{'device_type': 'pf', 'name': 'eth1', 'numvfs': 20}]
pf_final = [{'device_type': 'pf', 'link_mode': 'legacy',
'name': 'eth1', 'numvfs': 20}]
contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE)
pf_map = yaml.load(contents) if contents else []
@ -149,13 +151,13 @@ class TestUtils(base.TestCase):
self.assertListEqual(pf_final, pf_map)
def test_update_sriov_pf_map_exist_with_promisc(self):
pf_initial = [{'device_type': 'pf', 'name': 'eth1', 'numvfs': 10,
'promisc': 'on'}]
pf_initial = [{'device_type': 'pf', 'link_mode': 'legacy',
'name': 'eth1', 'numvfs': 10, 'promisc': 'on'}]
utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_initial)
utils.update_sriov_pf_map('eth1', 20, False)
pf_final = [{'device_type': 'pf', 'name': 'eth1', 'numvfs': 20,
'promisc': 'on'}]
pf_final = [{'device_type': 'pf', 'link_mode': 'legacy',
'name': 'eth1', 'numvfs': 20, 'promisc': 'on'}]
contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE)
pf_map = yaml.load(contents) if contents else []

View File

@ -414,7 +414,8 @@ def _get_dpdk_mac_address(name):
return item['mac_address']
def update_sriov_pf_map(ifname, numvfs, noop, promisc=None):
def update_sriov_pf_map(ifname, numvfs, noop, promisc=None,
link_mode='legacy'):
if not noop:
sriov_map = _get_sriov_map()
for item in sriov_map:
@ -422,6 +423,7 @@ def update_sriov_pf_map(ifname, numvfs, noop, promisc=None):
item['numvfs'] = numvfs
if promisc is not None:
item['promisc'] = promisc
item['link_mode'] = link_mode
break
else:
new_item = {}
@ -430,6 +432,7 @@ def update_sriov_pf_map(ifname, numvfs, noop, promisc=None):
new_item['numvfs'] = numvfs
if promisc is not None:
new_item['promisc'] = promisc
new_item['link_mode'] = link_mode
sriov_map.append(new_item)
write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, sriov_map)