openstack
/
os-vif
Code Issues Proposed changes

hardware offload support for openvswitch 

In Kernel 4.8 we introduced Traffic Control (TC see [1]) hardware offloads
framework for SR-IOV VFs which allows us to configure the NIC [2].
Subsequent OVS patches [3] allow us to use the TC framework
to offload OVS datapath rules.

This patch adds plug/unplug when using ovs vif 'OVS_ACCELERATION'. The
plug method will lookup the VF representor and connect it to 'br-int',
while the unplug method will remove the VF representor from 'br-int'.

[1] https://linux.die.net/man/8/tc
[2] http://netdevconf.org/1.2/papers/efraim-gerlitz-sriov-ovs-final.pdf
[3] https://mail.openvswitch.org/pipermail/ovs-dev/2017-April/330606.html

Change-Id: I90120119800cc2d3083b832700cc3d2ca655e638
This commit is contained in:
Moshe Levi 2017-03-19 11:03:27 +02:00
parent e6d25e7c64
commit 157bf4c5cc
7 changed files with 362 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
doc/source/plugins/ovs.rst
View File

@ -32,6 +32,24 @@ The Open vSwitch plugin provides support for the following VIF types:
Refer to :ref:`vif-vhostuser` for more information.
`VIFHostDevice`
Configuration where an SR-IOV PCI device virtual function (VF) is passed
through to a guest. The ``hw-tc-offload`` feature should be enabled on the
SR-IOV Physical Function (PF) using ``ethtool``:
.. code-block:: shell
ethtool -K <PF> hw-tc-offload
This will create *VF representor* per VF. The VF representor plays the same
role as TAP devices in Para-Virtual (PV) setup. In this case the ``plug()``
method connects the VF representor to the OpenVSwitch bridge.
Refer to :ref:`vif-hostdevice` for more information.
.. versionadded:: 1.5.0
For information on the VIF type objects, refer to :doc:`../vif_types`. Note
that only the above VIF types are supported by this plugin.

3
os_vif/tests/test_os_vif.py
View File

@ -146,10 +146,11 @@ class TestOSVIF(base.TestCase):
self.assertEqual(info.plugin_info[1].plugin_name, "ovs")
vif_info = info.plugin_info[1].vif_info
self.assertEqual(len(vif_info), 3)
self.assertEqual(len(vif_info), 4)
self.assertEqual(vif_info[0].vif_object_name, "VIFBridge")
self.assertEqual(vif_info[1].vif_object_name, "VIFOpenVSwitch")
self.assertEqual(vif_info[2].vif_object_name, "VIFVHostUser")
self.assertEqual(vif_info[3].vif_object_name, "VIFHostDevice")
def test_host_info_filtered(self):
os_vif.initialize()

9
vif_plug_ovs/exception.py
View File

@ -26,3 +26,12 @@ class MissingPortProfile(osv_exception.ExceptionBase):
class WrongPortProfile(osv_exception.ExceptionBase):
msg_fmt = _('Port profile %(profile)s is not a subclass '
'of VIFPortProfileOpenVSwitch')
class RepresentorNotFound(osv_exception.ExceptionBase):
msg_fmt = _('Failed getting representor port for PF %(ifname)s with '
'%(vf_num)s')
class PciDeviceNotFoundById(osv_exception.ExceptionBase):
msg_fmt = _("PCI device %(id)s not found")

94
vif_plug_ovs/linux_net.py
View File

@ -19,7 +19,10 @@
"""Implements vlans, bridges using linux utilities."""
import errno
import glob
import os
import re
import sys
from oslo_concurrency import processutils
@ -32,6 +35,8 @@ from vif_plug_ovs import privsep
LOG = logging.getLogger(__name__)
VIRTFN_RE = re.compile("virtfn(\d+)")
def _ovs_vsctl(args, timeout=None):
full_args = ['ovs-vsctl']
@ -153,7 +158,7 @@ def ensure_bridge(bridge):
process_input='1',
check_exit_code=[0, 1])
# we bring up the bridge to allow it to switch packets
processutils.execute('ip', 'link', 'set', bridge, 'up')
set_interface_state(bridge, 'up')
@privsep.vif_plug.entrypoint
@ -197,6 +202,12 @@ def _set_device_mtu(dev, mtu):
check_exit_code=[0, 2, 254])
@privsep.vif_plug.entrypoint
def set_interface_state(interface_name, port_state):
processutils.execute('ip', 'link', 'set', interface_name, port_state,
check_exit_code=[0, 2, 254])
@privsep.vif_plug.entrypoint
def _set_mtu_request(dev, mtu, timeout=None):
args = ['--', 'set', 'interface', dev,
@ -212,3 +223,84 @@ def _ovs_supports_mtu_requests(timeout=None):
' a column whose name matches "mtu_request"'):
return False
return True
def get_representor_port(pf_ifname, vf_num):
"""Get the representor netdevice which is corresponding to the VF.
This method gets PF interface name and number of VF. It iterates over all
the interfaces under the PF location and looks for interface that has the
VF number in the phys_port_name. That interface is the representor for
the requested VF.
"""
path = "/sys/class/net/%s/subsystem/" % pf_ifname
try:
for device in os.listdir(path):
if device == pf_ifname:
continue
file_name = os.path.join(path, device, 'phys_port_name')
if not os.path.isfile(file_name):
continue
try:
representor_num = open("%s/%s/phys_port_name" %
(path, device)).readline().rstrip()
if int(representor_num) == int(vf_num):
return device
except IOError as e:
# We want to ignore interfaces which we can't read their
# phys_port_name file.
with excutils.save_and_reraise_exception() as ctxt:
if e.errno == errno.EOPNOTSUPP:
ctxt.reraise = False
except ValueError:
# skip representor_num which we can't convert to integer
continue
except IOError:
pass
raise exception.RepresentorNotFound(ifname=pf_ifname, vf_num=vf_num)
def _get_sysfs_netdev_path(pci_addr, pf_interface):
"""Get the sysfs path based on the PCI address of the device.
Assumes a networking device - will not check for the existence of the path.
"""
if pf_interface:
return "/sys/bus/pci/devices/%s/physfn/net" % (pci_addr)
return "/sys/bus/pci/devices/%s/net" % (pci_addr)
def get_ifname_by_pci_address(pci_addr, pf_interface=False):
"""Get the interface name based on a VF's pci address
The returned interface name is either the parent PF's or that of the VF
itself based on the argument of pf_interface.
"""
dev_path = _get_sysfs_netdev_path(pci_addr, pf_interface)
try:
dev_info = os.listdir(dev_path)
return dev_info.pop()
except Exception:
raise exception.PciDeviceNotFoundById(id=pci_addr)
def get_vf_num_by_pci_address(pci_addr):
"""Get the VF number based on a VF's pci address
A VF is associated with an VF number, which ip link command uses to
configure it. This number can be obtained from the PCI device filesystem.
"""
virtfns_path = "/sys/bus/pci/devices/%s/physfn/virtfn*" % (pci_addr)
vf_num = None
try:
for vf_path in glob.iglob(virtfns_path):
if re.search(pci_addr, os.readlink(vf_path)):
t = VIRTFN_RE.search(vf_path)
vf_num = t.group(1)
break
except Exception:
pass
if vf_num is None:
raise exception.PciDeviceNotFoundById(id=pci_addr)
return vf_num

31
vif_plug_ovs/ovs.py
View File

@ -79,7 +79,11 @@ class OvsPlugin(plugin.PluginBase):
objects.host_info.HostVIFInfo(
vif_object_name=objects.vif.VIFVHostUser.__name__,
min_version="1.0",
max_version="1.0")
max_version="1.0"),
objects.host_info.HostVIFInfo(
vif_object_name=objects.vif.VIFHostDevice.__name__,
min_version="1.0",
max_version="1.0"),
])
def _get_mtu(self, vif):
@ -151,6 +155,17 @@ class OvsPlugin(plugin.PluginBase):
constants.OVS_DATAPATH_SYSTEM)
self._create_vif_port(vif, vif.id, instance_info)
def _plug_vf_passthrough(self, vif, instance_info):
linux_net.ensure_ovs_bridge(
vif.network.bridge, constants.OVS_DATAPATH_SYSTEM)
pci_slot = vif.dev_address
pf_ifname = linux_net.get_ifname_by_pci_address(
pci_slot, pf_interface=True)
vf_num = linux_net.get_vf_num_by_pci_address(pci_slot)
representor = linux_net.get_representor_port(pf_ifname, vf_num)
linux_net.set_interface_state(representor, 'up')
self._create_vif_port(vif, representor, instance_info)
def plug(self, vif, instance_info):
if not hasattr(vif, "port_profile"):
raise exception.MissingPortProfile()
@ -172,6 +187,8 @@ class OvsPlugin(plugin.PluginBase):
self._plug_vif_windows(vif, instance_info)
elif isinstance(vif, objects.vif.VIFVHostUser):
self._plug_vhostuser(vif, instance_info)
elif isinstance(vif, objects.vif.VIFHostDevice):
self._plug_vf_passthrough(vif, instance_info)
def _unplug_vhostuser(self, vif, instance_info):
linux_net.delete_ovs_vif_port(vif.network.bridge,
@ -200,6 +217,16 @@ class OvsPlugin(plugin.PluginBase):
linux_net.delete_ovs_vif_port(vif.network.bridge, vif.id,
timeout=self.config.ovs_vsctl_timeout)
def _unplug_vf_passthrough(self, vif, instance_info):
"""Remove port from OVS."""
pci_slot = vif.dev_address
pf_ifname = linux_net.get_ifname_by_pci_address(pci_slot,
pf_interface=True)
vf_num = linux_net.get_vf_num_by_pci_address(pci_slot)
representor = linux_net.get_representor_port(pf_ifname, vf_num)
linux_net.delete_ovs_vif_port(vif.network.bridge, representor)
linux_net.set_interface_state(representor, 'down')
def unplug(self, vif, instance_info):
if not hasattr(vif, "port_profile"):
raise exception.MissingPortProfile()
@ -218,3 +245,5 @@ class OvsPlugin(plugin.PluginBase):
self._unplug_vif_windows(vif, instance_info)
elif isinstance(vif, objects.vif.VIFVHostUser):
self._unplug_vhostuser(vif, instance_info)
elif isinstance(vif, objects.vif.VIFHostDevice):
self._unplug_vf_passthrough(vif, instance_info)

122
vif_plug_ovs/tests/test_linux_net.py
View File

@ -10,6 +10,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import glob
import mock
import os.path
import testtools
@ -17,6 +18,7 @@ import testtools
from oslo_concurrency import processutils
from vif_plug_ovs import constants
from vif_plug_ovs import exception
from vif_plug_ovs import linux_net
from vif_plug_ovs import privsep
@ -34,7 +36,8 @@ class LinuxNetTest(testtools.TestCase):
linux_net.ensure_bridge("br0")
self.assertEqual(mock_execute.mock_calls, [
mock.call('ip', 'link', 'set', 'br0', 'up'),
mock.call('ip', 'link', 'set', 'br0', 'up',
check_exit_code=[0, 2, 254]),
])
self.assertEqual(mock_dev_exists.mock_calls, [
mock.call("br0"),
@ -53,7 +56,8 @@ class LinuxNetTest(testtools.TestCase):
mock.call('brctl', 'stp', 'br0', "off"),
mock.call('tee', '/sys/class/net/br0/bridge/multicast_snooping',
check_exit_code=[0, 1], process_input='0'),
mock.call('ip', 'link', 'set', 'br0', 'up'),
mock.call('ip', 'link', 'set', 'br0', 'up',
check_exit_code=[0, 2, 254]),
])
self.assertEqual(mock_dev_exists.mock_calls, [
mock.call("br0")
@ -74,7 +78,8 @@ class LinuxNetTest(testtools.TestCase):
check_exit_code=[0, 1], process_input='0'),
mock.call('tee', '/proc/sys/net/ipv6/conf/br0/disable_ipv6',
check_exit_code=[0, 1], process_input='1'),
mock.call('ip', 'link', 'set', 'br0', 'up'),
mock.call('ip', 'link', 'set', 'br0', 'up',
check_exit_code=[0, 2, 254]),
])
self.assertEqual(mock_dev_exists.mock_calls, [
mock.call("br0")
@ -296,3 +301,114 @@ class LinuxNetTest(testtools.TestCase):
result = linux_net._ovs_supports_mtu_requests(timeout=timeout)
mock_vsctl.assert_called_with(args, timeout=timeout)
self.assertTrue(result)
@mock.patch('six.moves.builtins.open')
@mock.patch.object(os.path, 'isfile')
@mock.patch.object(os, 'listdir')
def test_get_representor_port(self, mock_listdir, mock_isfile, mock_open):
mock_listdir.return_value = [
'pf_ifname', 'vf1_ifname', 'vf2_ifname', 'rep_vf_1', 'rep_vf_2'
]
mock_isfile.side_effect = [True, True, True, True, True]
mock_open.return_value.__enter__ = lambda s: s
mock_open.return_value.__exit__ = mock.Mock()
readline_mock = mock_open.return_value.readline
readline_mock.side_effect = ['', '', '1', '2']
ifname = linux_net.get_representor_port('pf_ifname', '2')
self.assertEqual('rep_vf_2', ifname)
@mock.patch('six.moves.builtins.open')
@mock.patch.object(os.path, 'isfile')
@mock.patch.object(os, 'listdir')
def test_get_representor_port_not_found(
self, mock_listdir, mock_isfile, mock_open):
mock_listdir.return_value = [
'pf_ifname', 'vf1_ifname', 'vf2_ifname', 'rep_vf_1', 'rep_vf_2'
]
mock_isfile.side_effect = [True, True, True, True, True]
mock_open.return_value.__enter__ = lambda s: s
mock_open.return_value.__exit__ = mock.Mock()
readline_mock = mock_open.return_value.readline
readline_mock.side_effect = ['', '', '1', '2']
self.assertRaises(
exception.RepresentorNotFound,
linux_net.get_representor_port,
'pf_ifname', '3'),
@mock.patch('six.moves.builtins.open')
@mock.patch.object(os.path, 'isfile')
@mock.patch.object(os, 'listdir')
def test_get_representor_port_exception(
self, mock_listdir, mock_isfile, mock_open):
mock_listdir.return_value = [
'pf_ifname', 'vf1_ifname', 'vf2_ifname', 'rep_vf_1', 'rep_vf_2'
]
mock_isfile.side_effect = [True, True, True, True, True]
mock_open.return_value.__enter__ = lambda s: s
mock_open.return_value.__exit__ = mock.Mock()
readline_mock = mock_open.return_value.readline
readline_mock.side_effect = ['', IOError(), '1', '2']
self.assertRaises(
exception.RepresentorNotFound,
linux_net.get_representor_port,
'pf_ifname', '3'),
@mock.patch.object(os, 'listdir')
def test_physical_function_inferface_name(self, mock_listdir):
mock_listdir.return_value = ['foo', 'bar']
ifname = linux_net.get_ifname_by_pci_address(
'0000:00:00.1', pf_interface=True)
self.assertEqual(ifname, 'bar')
@mock.patch.object(os, 'listdir')
def test_virtual_function_inferface_name(self, mock_listdir):
mock_listdir.return_value = ['foo', 'bar']
ifname = linux_net.get_ifname_by_pci_address(
'0000:00:00.1', pf_interface=False)
self.assertEqual(ifname, 'bar')
@mock.patch.object(os, 'listdir')
def test_get_ifname_by_pci_address_exception(self, mock_listdir):
mock_listdir.side_effect = OSError('No such file or directory')
self.assertRaises(
exception.PciDeviceNotFoundById,
linux_net.get_ifname_by_pci_address,
'0000:00:00.1'
)
@mock.patch.object(os, 'readlink')
@mock.patch.object(glob, 'iglob')
def test_vf_number_found(self, mock_iglob, mock_readlink):
mock_iglob.return_value = [
'/sys/bus/pci/devices/0000:00:00.1/physfn/virtfn3',
]
mock_readlink.return_value = '../../0000:00:00.1'
vf_num = linux_net.get_vf_num_by_pci_address('0000:00:00.1')
self.assertEqual(vf_num, '3')
@mock.patch.object(os, 'readlink')
@mock.patch.object(glob, 'iglob')
def test_vf_number_not_found(self, mock_iglob, mock_readlink):
mock_iglob.return_value = [
'/sys/bus/pci/devices/0000:00:00.1/physfn/virtfn3',
]
mock_readlink.return_value = '../../0000:00:00.2'
self.assertRaises(
exception.PciDeviceNotFoundById,
linux_net.get_vf_num_by_pci_address,
'0000:00:00.1'
)
@mock.patch.object(os, 'readlink')
@mock.patch.object(glob, 'iglob')
def test_get_vf_num_by_pci_address_exception(
self, mock_iglob, mock_readlink):
mock_iglob.return_value = [
'/sys/bus/pci/devices/0000:00:00.1/physfn/virtfn3',
]
mock_readlink.side_effect = OSError('No such file or directory')
self.assertRaises(
exception.PciDeviceNotFoundById,
linux_net.get_vf_num_by_pci_address,
'0000:00:00.1'
)

92
vif_plug_ovs/tests/test_plugin.py
View File

@ -14,6 +14,7 @@ import mock
import testtools
from os_vif import objects
from os_vif.objects import fields
from vif_plug_ovs import constants
from vif_plug_ovs import linux_net
@ -88,6 +89,15 @@ class PluginTest(testtools.TestCase):
mode='server', # qemu server mode <=> ovs client mode
port_profile=self.profile_ovs)
self.vif_ovs_vf_passthrough = objects.vif.VIFHostDevice(
id='b679325f-ca89-4ee0-a8be-6db1409b69ea',
address='ca:fe:de:ad:be:ef',
network=self.network_ovs,
dev_type=fields.VIFVIFHostDeviceDevType.ETHERNET,
dev_address='0002:24:12.3',
bridge_name='br-int',
port_profile=self.profile_ovs)
self.instance = objects.instance_info.InstanceInfo(
name='demo',
uuid='f0000000-0000-0000-0000-000000000001')
@ -133,6 +143,7 @@ class PluginTest(testtools.TestCase):
ensure_ovs_bridge.assert_called_once_with(
self.vif_ovs.network.bridge, constants.OVS_DATAPATH_SYSTEM)
@mock.patch.object(linux_net, 'set_interface_state')
@mock.patch.object(linux_net, 'ensure_ovs_bridge')
@mock.patch.object(ovs.OvsPlugin, '_update_vif_port')
@mock.patch.object(ovs.OvsPlugin, '_create_vif_port')
@ -145,7 +156,8 @@ class PluginTest(testtools.TestCase):
def test_plug_ovs_bridge(self, mock_sys, ensure_bridge, device_exists,
create_veth_pair, update_veth_pair,
add_bridge_port, _create_vif_port,
_update_vif_port, ensure_ovs_bridge):
_update_vif_port, ensure_ovs_bridge,
set_interface_state):
calls = {
'device_exists': [mock.call('qvob679325f-ca')],
'create_veth_pair': [mock.call('qvbb679325f-ca',
@ -155,6 +167,8 @@ class PluginTest(testtools.TestCase):
'qvob679325f-ca',
1500)],
'ensure_bridge': [mock.call('qbrvif-xxx-yyy')],
'set_interface_state': [mock.call('qbrvif-xxx-yyy',
'up')],
'add_bridge_port': [mock.call('qbrvif-xxx-yyy',
'qvbb679325f-ca')],
'_update_vif_port': [mock.call(self.vif_ovs_hybrid,
@ -309,3 +323,79 @@ class PluginTest(testtools.TestCase):
plugin = ovs.OvsPlugin.load("ovs")
plugin.unplug(self.vif_vhostuser, self.instance)
delete_ovs_vif_port.assert_has_calls(calls['delete_ovs_vif_port'])
@mock.patch.object(linux_net, 'ensure_ovs_bridge')
@mock.patch.object(linux_net, 'get_ifname_by_pci_address')
@mock.patch.object(linux_net, 'get_vf_num_by_pci_address')
@mock.patch.object(linux_net, 'get_representor_port')
@mock.patch.object(linux_net, 'set_interface_state')
@mock.patch.object(ovs.OvsPlugin, '_create_vif_port')
def test_plug_ovs_vf_passthrough(self, _create_vif_port,
set_interface_state,
get_representor_port,
get_vf_num_by_pci_address,
get_ifname_by_pci_address,
ensure_ovs_bridge):
get_ifname_by_pci_address.return_value = 'eth0'
get_vf_num_by_pci_address.return_value = '2'
get_representor_port.return_value = 'eth0_2'
calls = {
'ensure_ovs_bridge': [mock.call('br0',
constants.OVS_DATAPATH_SYSTEM)],
'get_ifname_by_pci_address': [mock.call('0002:24:12.3',
pf_interface=True)],
'get_vf_num_by_pci_address': [mock.call('0002:24:12.3')],
'get_representor_port': [mock.call('eth0', '2')],
'set_interface_state': [mock.call('eth0_2', 'up')],
'_create_vif_port': [mock.call(
self.vif_ovs_vf_passthrough, 'eth0_2',
self.instance)]
}
plugin = ovs.OvsPlugin.load("ovs")
plugin.plug(self.vif_ovs_vf_passthrough, self.instance)
ensure_ovs_bridge.assert_has_calls(calls['ensure_ovs_bridge'])
get_ifname_by_pci_address.assert_has_calls(
calls['get_ifname_by_pci_address'])
get_vf_num_by_pci_address.assert_has_calls(
calls['get_vf_num_by_pci_address'])
get_representor_port.assert_has_calls(
calls['get_representor_port'])
set_interface_state.assert_has_calls(calls['set_interface_state'])
_create_vif_port.assert_has_calls(calls['_create_vif_port'])
@mock.patch.object(linux_net, 'get_ifname_by_pci_address')
@mock.patch.object(linux_net, 'get_vf_num_by_pci_address')
@mock.patch.object(linux_net, 'get_representor_port')
@mock.patch.object(linux_net, 'set_interface_state')
@mock.patch.object(linux_net, 'delete_ovs_vif_port')
def test_unplug_ovs_vf_passthrough(self, delete_ovs_vif_port,
set_interface_state,
get_representor_port,
get_vf_num_by_pci_address,
get_ifname_by_pci_address):
calls = {
'get_ifname_by_pci_address': [mock.call('0002:24:12.3',
pf_interface=True)],
'get_vf_num_by_pci_address': [mock.call('0002:24:12.3')],
'get_representor_port': [mock.call('eth0', '2')],
'set_interface_state': [mock.call('eth0_2', 'down')],
'delete_ovs_vif_port': [mock.call('br0', 'eth0_2')]
}
get_ifname_by_pci_address.return_value = 'eth0'
get_vf_num_by_pci_address.return_value = '2'
get_representor_port.return_value = 'eth0_2'
plugin = ovs.OvsPlugin.load("ovs")
plugin.unplug(self.vif_ovs_vf_passthrough, self.instance)
get_ifname_by_pci_address.assert_has_calls(
calls['get_ifname_by_pci_address'])
get_vf_num_by_pci_address.assert_has_calls(
calls['get_vf_num_by_pci_address'])
get_representor_port.assert_has_calls(
calls['get_representor_port'])
delete_ovs_vif_port.assert_has_calls(calls['delete_ovs_vif_port'])
set_interface_state.assert_has_calls(calls['set_interface_state'])