Add SR-IOV pod vif driver
This commit adds SR-IOV driver and new type of VIF to handle SR-IOV requests. This driver can work as a primary driver and only one driver, but only when kubernetes will fully support CNI specification. Now this driver can work in couple with multi vif driver, e.g. NPWGMultiVIFDriver. (see doc/source/installation/multi_vif_with_npwg_spec.rst) Also this driver relies on kubernetes SRIOV device plugin. This commit also adds 'default_physnet_subnets' setting, that should include a mapping of physnets to neutron subnet IDs, it's necessary to specify VIF's physnet (subnet id comes from annotation). To get details how to create pods with sriov interfaces see doc/source/installation/sriov.rst Target bp: kuryr-kubernetes-sriov-support Change-Id: I45c5f1a7fb423ee68731d0ae85f7171e33d0aeeb Signed-off-by: Danil Golov <d.golov@partner.samsung.com> Signed-off-by: Vladimir Kuramshin <v.kuramshin@samsung.com> Signed-off-by: Alexey Perevalov <a.perevalov@samsung.com>
This commit is contained in:
Danil Golov
parent
c9f03a6a0b
commit
8e60dcc4aa
|
|
@ -237,6 +237,15 @@ nested_vif_driver_opts = [
|
|||
default=''),
|
||||
]
|
||||
|
||||
DEFAULT_PHYSNET_SUBNET_MAPPINGS = {}
|
||||
sriov_opts = [
|
||||
cfg.DictOpt('default_physnet_subnets',
|
||||
help=_("A mapping of default subnets for certain physnets "
|
||||
"in a form of physnet-name:<SUBNET-ID>"),
|
||||
default=DEFAULT_PHYSNET_SUBNET_MAPPINGS),
|
||||
]
|
||||
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(kuryr_k8s_opts)
|
||||
CONF.register_opts(daemon_opts, group='cni_daemon')
|
||||
|
|
@ -246,6 +255,7 @@ CONF.register_opts(octavia_defaults, group='octavia_defaults')
|
|||
CONF.register_opts(cache_defaults, group='cache_defaults')
|
||||
CONF.register_opts(ingress, group='ingress')
|
||||
CONF.register_opts(nested_vif_driver_opts, group='pod_vif_nested')
|
||||
CONF.register_opts(sriov_opts, group='sriov')
|
||||
|
||||
CONF.register_opts(lib_config.core_opts)
|
||||
CONF.register_opts(lib_config.binding_opts, 'binding')
|
||||
|
|
|
|||
|
|
@ -65,3 +65,4 @@ VIF_POOL_SHOW = '/showPool'
|
|||
DEFAULT_IFNAME = 'eth0'
|
||||
|
||||
ADDITIONAL_IFNAME_PREFIX = 'eth'
|
||||
K8S_NPWG_SRIOV_PREFIX = "intel.com/sriov"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,132 @@
|
|||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from kuryr.lib import constants as kl_const
|
||||
from oslo_log import log as logging
|
||||
|
||||
from kuryr_kubernetes import clients
|
||||
from kuryr_kubernetes import config
|
||||
from kuryr_kubernetes import constants
|
||||
from kuryr_kubernetes.controller.drivers import neutron_vif
|
||||
from kuryr_kubernetes.controller.drivers import utils as c_utils
|
||||
from kuryr_kubernetes import os_vif_util as ovu
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SriovVIFDriver(neutron_vif.NeutronPodVIFDriver):
|
||||
"""Provides VIFs for SRIOV VF interfaces."""
|
||||
|
||||
ALIAS = 'sriov_pod_vif'
|
||||
|
||||
def __init__(self):
|
||||
self._physnet_mapping = self._get_physnet_mapping()
|
||||
|
||||
def request_vif(self, pod, project_id, subnets, security_groups):
|
||||
amount = self._get_remaining_sriov_vfs(pod)
|
||||
if not amount:
|
||||
LOG.error("SRIOV VIF request failed due to lack of "
|
||||
"available VFs for the current pod creation")
|
||||
return None
|
||||
|
||||
pod_name = pod['metadata']['name']
|
||||
neutron = clients.get_neutron_client()
|
||||
vif_plugin = 'sriov'
|
||||
subnet_id = next(iter(subnets))
|
||||
physnet = self._get_physnet_for_subnet_id(subnet_id)
|
||||
LOG.debug("Pod {} handling {}".format(pod_name, physnet))
|
||||
rq = self._get_port_request(pod, project_id,
|
||||
subnets, security_groups)
|
||||
|
||||
port = neutron.create_port(rq).get('port')
|
||||
vif = ovu.neutron_to_osvif_vif(vif_plugin, port, subnets)
|
||||
vif.physnet = physnet
|
||||
|
||||
LOG.debug("{} vifs are available for the pod {}".format(
|
||||
amount, pod_name))
|
||||
|
||||
self._reduce_remaining_sriov_vfs(pod)
|
||||
return vif
|
||||
|
||||
def activate_vif(self, pod, vif):
|
||||
vif.active = True
|
||||
|
||||
def _get_physnet_mapping(self):
|
||||
physnets = config.CONF.sriov.default_physnet_subnets
|
||||
|
||||
result = {}
|
||||
for name, subnet_id in physnets.items():
|
||||
result[subnet_id] = name
|
||||
return result
|
||||
|
||||
def _get_physnet_for_subnet_id(self, subnet_id):
|
||||
"""Returns an appropriate physnet for exact subnet_id from mapping"""
|
||||
try:
|
||||
physnet = self._physnet_mapping[subnet_id]
|
||||
except KeyError as ex:
|
||||
LOG.error("No mapping for subnet {} in {}".format(
|
||||
subnet_id, self._physnet_mapping))
|
||||
raise ex
|
||||
return physnet
|
||||
|
||||
def _get_remaining_sriov_vfs(self, pod):
|
||||
"""Returns the number of remaining vfs.
|
||||
|
||||
Returns the number of remaining vfs from the initial number that
|
||||
got allocated for the current pod. This information is stored in
|
||||
pod object.
|
||||
"""
|
||||
containers = pod['spec']['containers']
|
||||
total_amount = 0
|
||||
for container in containers:
|
||||
try:
|
||||
requests = container['resources']['requests']
|
||||
amount_value = requests[constants.K8S_NPWG_SRIOV_PREFIX]
|
||||
total_amount += int(amount_value)
|
||||
except KeyError:
|
||||
continue
|
||||
|
||||
return total_amount
|
||||
|
||||
def _reduce_remaining_sriov_vfs(self, pod):
|
||||
"""Reduces number of avaliable vfs for request"""
|
||||
containers = pod['spec']['containers']
|
||||
for container in containers:
|
||||
try:
|
||||
requests = container['resources']['requests']
|
||||
num_of_sriov = int(requests[constants.K8S_NPWG_SRIOV_PREFIX])
|
||||
if num_of_sriov == 0:
|
||||
continue
|
||||
requests[constants.K8S_NPWG_SRIOV_PREFIX] = (
|
||||
str(num_of_sriov - 1))
|
||||
except KeyError:
|
||||
continue
|
||||
|
||||
def _get_port_request(self, pod, project_id, subnets, security_groups):
|
||||
port_req_body = {
|
||||
'project_id': project_id,
|
||||
'name': c_utils.get_port_name(pod),
|
||||
'network_id': c_utils.get_network_id(subnets),
|
||||
'fixed_ips': ovu.osvif_to_neutron_fixed_ips(subnets),
|
||||
'device_owner': kl_const.DEVICE_OWNER + ':sriov',
|
||||
'device_id': c_utils.get_device_id(pod),
|
||||
'admin_state_up': True,
|
||||
'binding:vnic_type': 'direct',
|
||||
'binding:host_id': c_utils.get_host_id(pod),
|
||||
}
|
||||
|
||||
if security_groups:
|
||||
port_req_body['security_groups'] = security_groups
|
||||
|
||||
return {'port': port_req_body}
|
||||
|
|
@ -85,7 +85,8 @@ VIF_TYPE_TO_DRIVER_MAPPING = {
|
|||
'VIFOpenVSwitch': 'neutron-vif',
|
||||
'VIFBridge': 'neutron-vif',
|
||||
'VIFVlanNested': 'nested-vlan',
|
||||
'VIFMacvlanNested': 'nested-macvlan'
|
||||
'VIFMacvlanNested': 'nested-macvlan',
|
||||
'VIFSriov': 'sriov'
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -68,3 +68,15 @@ class VIFMacvlanNested(obj_osvif.VIFBase):
|
|||
# Name of the device to create
|
||||
'vif_name': obj_fields.StringField(),
|
||||
}
|
||||
|
||||
|
||||
@obj_base.VersionedObjectRegistry.register
|
||||
class VIFSriov(obj_osvif.VIFDirect):
|
||||
# This is OVO based SRIOV vif.
|
||||
|
||||
VERSION = '1.0'
|
||||
|
||||
fields = {
|
||||
# physnet of the VIF
|
||||
'physnet': obj_fields.StringField(),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ _kuryr_k8s_opts = [
|
|||
('namespace_subnet', namespace_subnet.namespace_subnet_driver_opts),
|
||||
('namespace_sg', namespace_security_groups.namespace_sg_driver_opts),
|
||||
('ingress', config.ingress),
|
||||
('sriov', config.sriov_opts),
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -38,3 +38,23 @@ class NoOpPlugin(PluginBase):
|
|||
|
||||
def unplug(self, vif, instance_info):
|
||||
pass
|
||||
|
||||
|
||||
class SriovPlugin(PluginBase):
|
||||
"""Sriov Plugin to be used with sriov VIFS"""
|
||||
|
||||
def describe(self):
|
||||
return objects.host_info.HostPluginInfo(
|
||||
plugin_name='sriov',
|
||||
vif_info=[
|
||||
objects.host_info.HostVIFInfo(
|
||||
vif_object_name=objects.vif.VIFDirect.__name__,
|
||||
min_version="1.0",
|
||||
max_version="1.0"),
|
||||
])
|
||||
|
||||
def plug(self, vif, instance_info):
|
||||
pass
|
||||
|
||||
def unplug(self, vif, instance_info):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -52,6 +52,11 @@ def neutron_to_osvif_network(neutron_network):
|
|||
if neutron_network.get('mtu') is not None:
|
||||
obj.mtu = neutron_network['mtu']
|
||||
|
||||
# Vlan information will be used later in Sriov binding driver
|
||||
if neutron_network.get('provider:network_type') == 'vlan':
|
||||
obj.should_provide_vlan = True
|
||||
obj.vlan = neutron_network['provider:segmentation_id']
|
||||
|
||||
return obj
|
||||
|
||||
|
||||
|
|
@ -295,6 +300,35 @@ def neutron_to_osvif_vif_nested_macvlan(neutron_port, subnets):
|
|||
vif_name=_get_vif_name(neutron_port))
|
||||
|
||||
|
||||
def neutron_to_osvif_vif_sriov(vif_plugin, neutron_port, subnets):
|
||||
"""Converts Neutron port to VIF object for SRIOV containers.
|
||||
|
||||
:param vif_plugin: name of the os-vif plugin to use (i.e. 'noop')
|
||||
:param neutron_port: dict containing port information as returned by
|
||||
neutron client's 'show_port'
|
||||
:param subnets: subnet mapping as returned by PodSubnetsDriver.get_subnets
|
||||
:return: osv_vif VIFSriov object
|
||||
"""
|
||||
|
||||
details = neutron_port.get('binding:vif_details', {})
|
||||
network = _make_vif_network(neutron_port, subnets)
|
||||
vlan_name = network.vlan if network.should_provide_vlan else ''
|
||||
vif = k_vif.VIFSriov(
|
||||
id=neutron_port['id'],
|
||||
address=neutron_port['mac_address'],
|
||||
network=network,
|
||||
has_traffic_filtering=details.get('port_filter', False),
|
||||
preserve_on_delete=False,
|
||||
active=_is_port_active(neutron_port),
|
||||
plugin=vif_plugin,
|
||||
mode='passthrough',
|
||||
vlan_name=vlan_name,
|
||||
vif_name=_get_vif_name(neutron_port),
|
||||
)
|
||||
|
||||
return vif
|
||||
|
||||
|
||||
def neutron_to_osvif_vif(vif_translator, neutron_port, subnets):
|
||||
"""Converts Neutron port to os-vif VIF object.
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,159 @@
|
|||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import mock
|
||||
|
||||
from kuryr_kubernetes.controller.drivers import sriov as sriov_drivers
|
||||
from kuryr_kubernetes.tests import base as test_base
|
||||
from kuryr_kubernetes.tests.unit import kuryr_fixtures as k_fix
|
||||
|
||||
from kuryr_kubernetes import constants as k_const
|
||||
from kuryr_kubernetes import os_vif_util as ovu
|
||||
from kuryr_kubernetes import utils
|
||||
|
||||
from oslo_config import cfg as oslo_cfg
|
||||
from oslo_utils import uuidutils
|
||||
|
||||
|
||||
class TestSriovVIFDriver(test_base.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestSriovVIFDriver, self).setUp()
|
||||
self._pod = {
|
||||
'metadata': {
|
||||
'resourceVersion': mock.sentinel.pod_version,
|
||||
'selfLink': mock.sentinel.pod_link,
|
||||
'name': 'podname'},
|
||||
'status': {'phase': k_const.K8S_POD_STATUS_PENDING},
|
||||
'spec': {
|
||||
'hostNetwork': False,
|
||||
'nodeName': 'hostname',
|
||||
'containers': [{
|
||||
'resources': {
|
||||
'requests': {
|
||||
k_const.K8S_NPWG_SRIOV_PREFIX: "2"
|
||||
}
|
||||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
|
||||
def test_activate_vif(self):
|
||||
cls = sriov_drivers.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
pod = mock.sentinel.pod
|
||||
vif = mock.Mock()
|
||||
vif.active = False
|
||||
|
||||
cls.activate_vif(m_driver, pod, vif)
|
||||
self.assertEqual(True, vif.active)
|
||||
|
||||
@mock.patch('kuryr_kubernetes.os_vif_util.osvif_to_neutron_fixed_ips')
|
||||
@mock.patch.object(ovu, 'neutron_to_osvif_vif')
|
||||
def test_request_vif(self, m_to_vif, m_to_fips):
|
||||
cls = sriov_drivers.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
neutron = self.useFixture(k_fix.MockNeutronClient()).client
|
||||
project_id = mock.sentinel.project_id
|
||||
fixed_ips = mock.sentinel.fixed_ips
|
||||
m_to_fips.return_value = fixed_ips
|
||||
network = mock.sentinel.Network
|
||||
subnet_id = uuidutils.generate_uuid()
|
||||
subnets = {subnet_id: network}
|
||||
security_groups = mock.sentinel.security_groups
|
||||
port_fixed_ips = mock.sentinel.port_fixed_ips
|
||||
port_id = mock.sentinel.port_id
|
||||
port = {
|
||||
'fixed_ips': port_fixed_ips,
|
||||
'id': port_id
|
||||
}
|
||||
port_request = mock.sentinel.port_request
|
||||
m_driver._get_port_request.return_value = port_request
|
||||
vif = mock.sentinel.vif
|
||||
m_to_vif.return_value = vif
|
||||
neutron.create_port.return_value = {'port': port}
|
||||
utils.get_subnet.return_value = subnets
|
||||
|
||||
self.assertEqual(vif, cls.request_vif(m_driver, self._pod, project_id,
|
||||
subnets, security_groups))
|
||||
|
||||
neutron.create_port.assert_called_once_with(port_request)
|
||||
|
||||
@mock.patch('kuryr_kubernetes.os_vif_util.osvif_to_neutron_fixed_ips')
|
||||
@mock.patch.object(ovu, 'neutron_to_osvif_vif')
|
||||
def test_request_vif_not_enough_vfs(self, m_to_vif, m_to_fips):
|
||||
cls = sriov_drivers.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
m_driver._get_remaining_sriov_vfs.return_value = 0
|
||||
neutron = self.useFixture(k_fix.MockNeutronClient()).client
|
||||
project_id = mock.sentinel.project_id
|
||||
network = mock.sentinel.Network
|
||||
subnet_id = uuidutils.generate_uuid()
|
||||
subnets = {subnet_id: network}
|
||||
security_groups = mock.sentinel.security_groups
|
||||
|
||||
self.assertIsNone(cls.request_vif(m_driver, self._pod, project_id,
|
||||
subnets, security_groups))
|
||||
|
||||
neutron.create_port.assert_not_called()
|
||||
|
||||
def test_get_sriov_num_vf(self):
|
||||
cls = sriov_drivers.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
amount = cls._get_remaining_sriov_vfs(m_driver, self._pod)
|
||||
self.assertEqual(amount, 2)
|
||||
|
||||
def test_reduce_remaining_sriov_vfs(self):
|
||||
cls = sriov_drivers.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
cls._reduce_remaining_sriov_vfs(m_driver, self._pod)
|
||||
amount = cls._get_remaining_sriov_vfs(m_driver, self._pod)
|
||||
self.assertEqual(amount, 1)
|
||||
|
||||
def test_get_physnet_mapping(self):
|
||||
cls = sriov_drivers.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
subnet_id = uuidutils.generate_uuid()
|
||||
oslo_cfg.CONF.set_override('default_physnet_subnets',
|
||||
'physnet10_4:'+str(subnet_id),
|
||||
group='sriov')
|
||||
|
||||
mapping = cls._get_physnet_mapping(m_driver)
|
||||
self.assertEqual(mapping, {subnet_id: 'physnet10_4'})
|
||||
|
||||
def test_get_physnet_for_subnet_id(self):
|
||||
cls = sriov_drivers.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
subnet_id = uuidutils.generate_uuid()
|
||||
m_driver._physnet_mapping = {subnet_id: 'physnet10_4'}
|
||||
|
||||
physnet = cls._get_physnet_for_subnet_id(m_driver, subnet_id)
|
||||
self.assertEqual(physnet, 'physnet10_4')
|
||||
|
||||
def test_get_physnet_for_subnet_id_error(self):
|
||||
cls = sriov_drivers.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
subnet_id = uuidutils.generate_uuid()
|
||||
m_driver._physnet_mapping = {}
|
||||
|
||||
self.assertRaises(KeyError, cls._get_physnet_for_subnet_id,
|
||||
m_driver, subnet_id)
|
||||
|
|
@ -33,7 +33,8 @@ LOG = log.getLogger(__name__)
|
|||
|
||||
VALID_MULTI_POD_POOLS_OPTS = {'noop': ['neutron-vif',
|
||||
'nested-vlan',
|
||||
'nested-macvlan'],
|
||||
'nested-macvlan',
|
||||
'sriov'],
|
||||
'neutron': ['neutron-vif'],
|
||||
'nested': ['nested-vlan'],
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ oslo.config.opts =
|
|||
|
||||
os_vif =
|
||||
noop = kuryr_kubernetes.os_vif_plug_noop:NoOpPlugin
|
||||
sriov = kuryr_kubernetes.os_vif_plug_noop:SriovPlugin
|
||||
|
||||
console_scripts =
|
||||
kuryr-k8s-controller = kuryr_kubernetes.cmd.eventlet.controller:start
|
||||
|
|
@ -33,6 +34,7 @@ console_scripts =
|
|||
|
||||
kuryr_kubernetes.vif_translators =
|
||||
ovs = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_ovs
|
||||
sriov = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_sriov
|
||||
|
||||
kuryr_kubernetes.cni.binding =
|
||||
VIFBridge = kuryr_kubernetes.cni.binding.bridge:BridgeDriver
|
||||
|
|
@ -74,6 +76,7 @@ kuryr_kubernetes.controller.drivers.pod_vif =
|
|||
neutron-vif = kuryr_kubernetes.controller.drivers.neutron_vif:NeutronPodVIFDriver
|
||||
nested-vlan = kuryr_kubernetes.controller.drivers.nested_vlan_vif:NestedVlanPodVIFDriver
|
||||
nested-macvlan = kuryr_kubernetes.controller.drivers.nested_macvlan_vif:NestedMacvlanPodVIFDriver
|
||||
sriov = kuryr_kubernetes.controller.drivers.sriov:SriovVIFDriver
|
||||
|
||||
kuryr_kubernetes.controller.drivers.endpoints_lbaas =
|
||||
lbaasv2 = kuryr_kubernetes.controller.drivers.lbaasv2:LBaaSv2Driver
|
||||
|
|
|
|||
Loading…
Reference in New Issue