Add SR-IOV binding driver to CNI

This commit includes a binding driver for SR-IOV interfaces. The driver
scans VFs of a PF for each SR-IOV interface requested and assignes them
to the Pod.

This commit adds new config parameter `physical_device_mappings`.
It is similar to neutron-sriov-nic-agent and helps manage PFs and physnets.

Implements: blueprint kuryr-kubernetes-sriov-support
Change-Id: Icda852cef35efdb75daeae78f7a093fe516f4c02
Signed-off-by: Danil Golov <d.golov@partner.samsung.com>
Signed-off-by: Alexey Perevalov <a.perevalov@samsung.com>
Signed-off-by: Vladimir Kuramshin <v.kuramshin@samsung.com>
This commit is contained in:
Danil Golov 2017-10-10 15:06:12 +03:00
parent 8e60dcc4aa
commit 4d53db2fdf
3 changed files with 203 additions and 0 deletions

View File

@ -0,0 +1,194 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import os
from kuryr.lib._i18n import _
from oslo_concurrency import lockutils
from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_log import log as logging
from kuryr_kubernetes.cni.binding import base as b_base
from kuryr_kubernetes import config
from kuryr_kubernetes import exceptions
from kuryr_kubernetes import utils
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class VIFSriovDriver(object):
def __init__(self):
self._lock = None
self._device_pf_mapping = self._get_device_pf_mapping()
def release_lock_object(func):
def wrapped(self, *args, **kwargs):
try:
func(self, *args, **kwargs)
finally:
if self._lock.acquired:
self._lock.release()
return
return wrapped
@release_lock_object
def connect(self, vif, ifname, netns):
physnet = vif.physnet
h_ipdb = b_base.get_ipdb()
c_ipdb = b_base.get_ipdb(netns)
pf_names = self._get_host_pf_names(physnet)
vf_name, vf_index, pf = self._get_available_vf_info(pf_names)
if not vf_name:
error_msg = "No free interfaces for pfysnet {} available".format(
physnet)
LOG.error(error_msg)
raise exceptions.CNIError(error_msg)
if vif.network.should_provide_vlan:
vlan_id = vif.network.vlan
self._set_vf_vlan(pf, vf_index, vlan_id)
with h_ipdb.interfaces[vf_name] as host_iface:
host_iface.net_ns_fd = utils.convert_netns(netns)
with c_ipdb.interfaces[vf_name] as iface:
iface.ifname = ifname
iface.address = vif.address
iface.mtu = vif.network.mtu
iface.up()
def disconnect(self, vif, ifname, netns):
# NOTE(k.zaitsev): when netns is deleted the interface is
# returned automatically to host netns. We may reset
# it to all-zero state
pass
def _get_host_pf_names(self, physnet):
"""Return a list of PFs, that belong to a physnet"""
if physnet not in self._device_pf_mapping:
raise cfg.Error(
"No mapping for physnet {} in {}".format(
physnet, self._device_pf_mapping))
return self._device_pf_mapping[physnet]
def _get_available_vf_info(self, pf_names):
"""Scan /sys for unacquired VF among PFs in pf_names"""
for pf in pf_names:
pf_sys_path = '/sys/class/net/{}/device'.format(pf)
nvfs = self._get_total_vfs(pf)
for vf_index in range(nvfs):
vf_sys_path = os.path.join(pf_sys_path,
'virtfn{}'.format(vf_index),
'net')
# TODO(kzaitsev): use /var/run/kuryr/smth
lock_path = os.path.join("/tmp",
"{}.{}".format(pf, vf_index))
self._acquire(lock_path)
LOG.debug("Aquired %s lock", lock_path)
try:
vf_names = os.listdir(vf_sys_path)
except OSError:
LOG.debug("Could not open %s. "
"Skipping vf %s for pf %s", vf_sys_path,
vf_index, pf)
self._release()
continue
if not vf_names:
LOG.debug("No interfaces in %s"
"Skipping vf %s for pf %s", vf_sys_path,
vf_index, pf)
self._release()
continue
vf_name = vf_names[0]
LOG.debug("Aquiring vf %s of pf %s", vf_index, pf)
return vf_name, vf_index, pf
return None, None, None
def _acquire(self, path):
if self._lock and self._lock.acquired:
raise RuntimeError(_("Attempting to lock {} when {} "
"is already locked.").format(path, self._lock))
self._lock = lockutils.InterProcessLock(path=path)
return self._lock.acquire()
def _release(self):
if not self._lock:
raise RuntimeError(_("Attempting release an empty lock"))
return self._lock.release()
def _get_total_vfs(self, pf):
"""Read /sys information for configured number of VFs of a PF"""
pf_sys_path = '/sys/class/net/{}/device'.format(pf)
total_fname = os.path.join(pf_sys_path, 'sriov_numvfs')
try:
with open(total_fname) as total_f:
data = total_f.read()
except IOError:
LOG.warning("Could not open %s. No VFs for %s", total_fname, pf)
return 0
nvfs = 0
try:
nvfs = int(data.strip())
except ValueError:
LOG.warning("Could not parse %s from %s. No VFs for %s", data,
total_fname, pf)
return 0
LOG.debug("PF %s has %s VFs", pf, nvfs)
return nvfs
def _set_vf_vlan(self, pf, vf_index, vlan_id):
"""Call `ip link set enp1s0f0 vf 5 vlan 10`"""
cmd = [
'ip', 'link',
'set', pf, 'vf', vf_index, 'vlan', vlan_id
]
try:
return processutils.execute(*cmd, run_as_root=True)
except Exception:
LOG.exception("Unable to execute %s", cmd)
raise
def is_healthy(self):
bridge_name = CONF.neutron_defaults.ovs_bridge
try:
with b_base.get_ipdb() as h_ipdb:
h_ipdb.interfaces[bridge_name]
return True
except Exception:
LOG.warning("Default OVS bridge %s does not exist on "
"the host.", bridge_name)
return False
def _get_device_pf_mapping(self):
"""Return a mapping in format {<physnet_name>:[<pf_name>, ...]}"""
phys_mappings = config.CONF.sriov.physical_device_mappings
physnets = collections.defaultdict(list)
for phys_map in phys_mappings:
try:
netname, ifname = phys_map.split(':', 1)
except ValueError:
raise cfg.Error(
"Invalid mapping {}".format(phys_map))
physnets[netname].append(ifname)
return physnets

View File

@ -238,11 +238,19 @@ nested_vif_driver_opts = [
]
DEFAULT_PHYSNET_SUBNET_MAPPINGS = {}
DEFAULT_DEVICE_MAPPINGS = []
sriov_opts = [
cfg.DictOpt('default_physnet_subnets',
help=_("A mapping of default subnets for certain physnets "
"in a form of physnet-name:<SUBNET-ID>"),
default=DEFAULT_PHYSNET_SUBNET_MAPPINGS),
cfg.ListOpt('physical_device_mappings',
default=DEFAULT_DEVICE_MAPPINGS,
help=_("Comma-separated list of "
"<physical_network>:<network_device> tuples mapping "
"physical network names to the agent's node-specific "
"physical network device interfaces of SR-IOV physical "
"function to be used for VLAN networks.")),
]

View File

@ -41,6 +41,7 @@ kuryr_kubernetes.cni.binding =
VIFOpenVSwitch = kuryr_kubernetes.cni.binding.bridge:VIFOpenVSwitchDriver
VIFVlanNested = kuryr_kubernetes.cni.binding.nested:VlanDriver
VIFMacvlanNested = kuryr_kubernetes.cni.binding.nested:MacvlanDriver
VIFSriov = kuryr_kubernetes.cni.binding.sriov:VIFSriovDriver
kuryr_kubernetes.controller.drivers.pod_project =
default = kuryr_kubernetes.controller.drivers.default_project:DefaultPodProjectDriver