Add SR-IOV binding driver to CNI
This commit includes a binding driver for SR-IOV interfaces. The driver scans VFs of a PF for each SR-IOV interface requested and assignes them to the Pod. This commit adds new config parameter `physical_device_mappings`. It is similar to neutron-sriov-nic-agent and helps manage PFs and physnets. Implements: blueprint kuryr-kubernetes-sriov-support Change-Id: Icda852cef35efdb75daeae78f7a093fe516f4c02 Signed-off-by: Danil Golov <d.golov@partner.samsung.com> Signed-off-by: Alexey Perevalov <a.perevalov@samsung.com> Signed-off-by: Vladimir Kuramshin <v.kuramshin@samsung.com>
This commit is contained in:
parent
8e60dcc4aa
commit
4d53db2fdf
|
@ -0,0 +1,194 @@
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import collections
|
||||||
|
import os
|
||||||
|
|
||||||
|
from kuryr.lib._i18n import _
|
||||||
|
from oslo_concurrency import lockutils
|
||||||
|
from oslo_concurrency import processutils
|
||||||
|
from oslo_config import cfg
|
||||||
|
from oslo_log import log as logging
|
||||||
|
|
||||||
|
from kuryr_kubernetes.cni.binding import base as b_base
|
||||||
|
from kuryr_kubernetes import config
|
||||||
|
from kuryr_kubernetes import exceptions
|
||||||
|
from kuryr_kubernetes import utils
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
CONF = cfg.CONF
|
||||||
|
|
||||||
|
|
||||||
|
class VIFSriovDriver(object):
|
||||||
|
def __init__(self):
|
||||||
|
self._lock = None
|
||||||
|
self._device_pf_mapping = self._get_device_pf_mapping()
|
||||||
|
|
||||||
|
def release_lock_object(func):
|
||||||
|
def wrapped(self, *args, **kwargs):
|
||||||
|
try:
|
||||||
|
func(self, *args, **kwargs)
|
||||||
|
finally:
|
||||||
|
if self._lock.acquired:
|
||||||
|
self._lock.release()
|
||||||
|
return
|
||||||
|
return wrapped
|
||||||
|
|
||||||
|
@release_lock_object
|
||||||
|
def connect(self, vif, ifname, netns):
|
||||||
|
physnet = vif.physnet
|
||||||
|
|
||||||
|
h_ipdb = b_base.get_ipdb()
|
||||||
|
c_ipdb = b_base.get_ipdb(netns)
|
||||||
|
|
||||||
|
pf_names = self._get_host_pf_names(physnet)
|
||||||
|
vf_name, vf_index, pf = self._get_available_vf_info(pf_names)
|
||||||
|
|
||||||
|
if not vf_name:
|
||||||
|
error_msg = "No free interfaces for pfysnet {} available".format(
|
||||||
|
physnet)
|
||||||
|
LOG.error(error_msg)
|
||||||
|
raise exceptions.CNIError(error_msg)
|
||||||
|
|
||||||
|
if vif.network.should_provide_vlan:
|
||||||
|
vlan_id = vif.network.vlan
|
||||||
|
self._set_vf_vlan(pf, vf_index, vlan_id)
|
||||||
|
|
||||||
|
with h_ipdb.interfaces[vf_name] as host_iface:
|
||||||
|
host_iface.net_ns_fd = utils.convert_netns(netns)
|
||||||
|
|
||||||
|
with c_ipdb.interfaces[vf_name] as iface:
|
||||||
|
iface.ifname = ifname
|
||||||
|
iface.address = vif.address
|
||||||
|
iface.mtu = vif.network.mtu
|
||||||
|
iface.up()
|
||||||
|
|
||||||
|
def disconnect(self, vif, ifname, netns):
|
||||||
|
# NOTE(k.zaitsev): when netns is deleted the interface is
|
||||||
|
# returned automatically to host netns. We may reset
|
||||||
|
# it to all-zero state
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _get_host_pf_names(self, physnet):
|
||||||
|
"""Return a list of PFs, that belong to a physnet"""
|
||||||
|
|
||||||
|
if physnet not in self._device_pf_mapping:
|
||||||
|
raise cfg.Error(
|
||||||
|
"No mapping for physnet {} in {}".format(
|
||||||
|
physnet, self._device_pf_mapping))
|
||||||
|
return self._device_pf_mapping[physnet]
|
||||||
|
|
||||||
|
def _get_available_vf_info(self, pf_names):
|
||||||
|
"""Scan /sys for unacquired VF among PFs in pf_names"""
|
||||||
|
|
||||||
|
for pf in pf_names:
|
||||||
|
pf_sys_path = '/sys/class/net/{}/device'.format(pf)
|
||||||
|
nvfs = self._get_total_vfs(pf)
|
||||||
|
for vf_index in range(nvfs):
|
||||||
|
vf_sys_path = os.path.join(pf_sys_path,
|
||||||
|
'virtfn{}'.format(vf_index),
|
||||||
|
'net')
|
||||||
|
# TODO(kzaitsev): use /var/run/kuryr/smth
|
||||||
|
lock_path = os.path.join("/tmp",
|
||||||
|
"{}.{}".format(pf, vf_index))
|
||||||
|
self._acquire(lock_path)
|
||||||
|
LOG.debug("Aquired %s lock", lock_path)
|
||||||
|
try:
|
||||||
|
vf_names = os.listdir(vf_sys_path)
|
||||||
|
except OSError:
|
||||||
|
LOG.debug("Could not open %s. "
|
||||||
|
"Skipping vf %s for pf %s", vf_sys_path,
|
||||||
|
vf_index, pf)
|
||||||
|
self._release()
|
||||||
|
continue
|
||||||
|
if not vf_names:
|
||||||
|
LOG.debug("No interfaces in %s"
|
||||||
|
"Skipping vf %s for pf %s", vf_sys_path,
|
||||||
|
vf_index, pf)
|
||||||
|
self._release()
|
||||||
|
continue
|
||||||
|
vf_name = vf_names[0]
|
||||||
|
LOG.debug("Aquiring vf %s of pf %s", vf_index, pf)
|
||||||
|
return vf_name, vf_index, pf
|
||||||
|
return None, None, None
|
||||||
|
|
||||||
|
def _acquire(self, path):
|
||||||
|
if self._lock and self._lock.acquired:
|
||||||
|
raise RuntimeError(_("Attempting to lock {} when {} "
|
||||||
|
"is already locked.").format(path, self._lock))
|
||||||
|
self._lock = lockutils.InterProcessLock(path=path)
|
||||||
|
return self._lock.acquire()
|
||||||
|
|
||||||
|
def _release(self):
|
||||||
|
if not self._lock:
|
||||||
|
raise RuntimeError(_("Attempting release an empty lock"))
|
||||||
|
return self._lock.release()
|
||||||
|
|
||||||
|
def _get_total_vfs(self, pf):
|
||||||
|
"""Read /sys information for configured number of VFs of a PF"""
|
||||||
|
|
||||||
|
pf_sys_path = '/sys/class/net/{}/device'.format(pf)
|
||||||
|
total_fname = os.path.join(pf_sys_path, 'sriov_numvfs')
|
||||||
|
try:
|
||||||
|
with open(total_fname) as total_f:
|
||||||
|
data = total_f.read()
|
||||||
|
except IOError:
|
||||||
|
LOG.warning("Could not open %s. No VFs for %s", total_fname, pf)
|
||||||
|
return 0
|
||||||
|
nvfs = 0
|
||||||
|
try:
|
||||||
|
nvfs = int(data.strip())
|
||||||
|
except ValueError:
|
||||||
|
LOG.warning("Could not parse %s from %s. No VFs for %s", data,
|
||||||
|
total_fname, pf)
|
||||||
|
return 0
|
||||||
|
LOG.debug("PF %s has %s VFs", pf, nvfs)
|
||||||
|
return nvfs
|
||||||
|
|
||||||
|
def _set_vf_vlan(self, pf, vf_index, vlan_id):
|
||||||
|
"""Call `ip link set enp1s0f0 vf 5 vlan 10`"""
|
||||||
|
cmd = [
|
||||||
|
'ip', 'link',
|
||||||
|
'set', pf, 'vf', vf_index, 'vlan', vlan_id
|
||||||
|
]
|
||||||
|
try:
|
||||||
|
return processutils.execute(*cmd, run_as_root=True)
|
||||||
|
except Exception:
|
||||||
|
LOG.exception("Unable to execute %s", cmd)
|
||||||
|
raise
|
||||||
|
|
||||||
|
def is_healthy(self):
|
||||||
|
bridge_name = CONF.neutron_defaults.ovs_bridge
|
||||||
|
try:
|
||||||
|
with b_base.get_ipdb() as h_ipdb:
|
||||||
|
h_ipdb.interfaces[bridge_name]
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
LOG.warning("Default OVS bridge %s does not exist on "
|
||||||
|
"the host.", bridge_name)
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _get_device_pf_mapping(self):
|
||||||
|
"""Return a mapping in format {<physnet_name>:[<pf_name>, ...]}"""
|
||||||
|
|
||||||
|
phys_mappings = config.CONF.sriov.physical_device_mappings
|
||||||
|
physnets = collections.defaultdict(list)
|
||||||
|
for phys_map in phys_mappings:
|
||||||
|
try:
|
||||||
|
netname, ifname = phys_map.split(':', 1)
|
||||||
|
except ValueError:
|
||||||
|
raise cfg.Error(
|
||||||
|
"Invalid mapping {}".format(phys_map))
|
||||||
|
physnets[netname].append(ifname)
|
||||||
|
return physnets
|
|
@ -238,11 +238,19 @@ nested_vif_driver_opts = [
|
||||||
]
|
]
|
||||||
|
|
||||||
DEFAULT_PHYSNET_SUBNET_MAPPINGS = {}
|
DEFAULT_PHYSNET_SUBNET_MAPPINGS = {}
|
||||||
|
DEFAULT_DEVICE_MAPPINGS = []
|
||||||
sriov_opts = [
|
sriov_opts = [
|
||||||
cfg.DictOpt('default_physnet_subnets',
|
cfg.DictOpt('default_physnet_subnets',
|
||||||
help=_("A mapping of default subnets for certain physnets "
|
help=_("A mapping of default subnets for certain physnets "
|
||||||
"in a form of physnet-name:<SUBNET-ID>"),
|
"in a form of physnet-name:<SUBNET-ID>"),
|
||||||
default=DEFAULT_PHYSNET_SUBNET_MAPPINGS),
|
default=DEFAULT_PHYSNET_SUBNET_MAPPINGS),
|
||||||
|
cfg.ListOpt('physical_device_mappings',
|
||||||
|
default=DEFAULT_DEVICE_MAPPINGS,
|
||||||
|
help=_("Comma-separated list of "
|
||||||
|
"<physical_network>:<network_device> tuples mapping "
|
||||||
|
"physical network names to the agent's node-specific "
|
||||||
|
"physical network device interfaces of SR-IOV physical "
|
||||||
|
"function to be used for VLAN networks.")),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -41,6 +41,7 @@ kuryr_kubernetes.cni.binding =
|
||||||
VIFOpenVSwitch = kuryr_kubernetes.cni.binding.bridge:VIFOpenVSwitchDriver
|
VIFOpenVSwitch = kuryr_kubernetes.cni.binding.bridge:VIFOpenVSwitchDriver
|
||||||
VIFVlanNested = kuryr_kubernetes.cni.binding.nested:VlanDriver
|
VIFVlanNested = kuryr_kubernetes.cni.binding.nested:VlanDriver
|
||||||
VIFMacvlanNested = kuryr_kubernetes.cni.binding.nested:MacvlanDriver
|
VIFMacvlanNested = kuryr_kubernetes.cni.binding.nested:MacvlanDriver
|
||||||
|
VIFSriov = kuryr_kubernetes.cni.binding.sriov:VIFSriovDriver
|
||||||
|
|
||||||
kuryr_kubernetes.controller.drivers.pod_project =
|
kuryr_kubernetes.controller.drivers.pod_project =
|
||||||
default = kuryr_kubernetes.controller.drivers.default_project:DefaultPodProjectDriver
|
default = kuryr_kubernetes.controller.drivers.default_project:DefaultPodProjectDriver
|
||||||
|
|
Loading…
Reference in New Issue