From 8e60dcc4aa4cc41a4c0276c4b8fdcb775d2eb6f1 Mon Sep 17 00:00:00 2001 From: Danil Golov Date: Tue, 10 Oct 2017 14:30:59 +0300 Subject: [PATCH] Add SR-IOV pod vif driver This commit adds SR-IOV driver and new type of VIF to handle SR-IOV requests. This driver can work as a primary driver and only one driver, but only when kubernetes will fully support CNI specification. Now this driver can work in couple with multi vif driver, e.g. NPWGMultiVIFDriver. (see doc/source/installation/multi_vif_with_npwg_spec.rst) Also this driver relies on kubernetes SRIOV device plugin. This commit also adds 'default_physnet_subnets' setting, that should include a mapping of physnets to neutron subnet IDs, it's necessary to specify VIF's physnet (subnet id comes from annotation). To get details how to create pods with sriov interfaces see doc/source/installation/sriov.rst Target bp: kuryr-kubernetes-sriov-support Change-Id: I45c5f1a7fb423ee68731d0ae85f7171e33d0aeeb Signed-off-by: Danil Golov Signed-off-by: Vladimir Kuramshin Signed-off-by: Alexey Perevalov --- kuryr_kubernetes/config.py | 10 ++ kuryr_kubernetes/constants.py | 1 + kuryr_kubernetes/controller/drivers/sriov.py | 132 +++++++++++++++ .../controller/drivers/vif_pool.py | 3 +- kuryr_kubernetes/objects/vif.py | 12 ++ kuryr_kubernetes/opts.py | 1 + kuryr_kubernetes/os_vif_plug_noop.py | 20 +++ kuryr_kubernetes/os_vif_util.py | 34 ++++ .../unit/controller/drivers/test_sriov.py | 159 ++++++++++++++++++ kuryr_kubernetes/utils.py | 3 +- setup.cfg | 3 + 11 files changed, 376 insertions(+), 2 deletions(-) create mode 100644 kuryr_kubernetes/controller/drivers/sriov.py create mode 100644 kuryr_kubernetes/tests/unit/controller/drivers/test_sriov.py diff --git a/kuryr_kubernetes/config.py b/kuryr_kubernetes/config.py index 12be9e992..4d388554a 100644 --- a/kuryr_kubernetes/config.py +++ b/kuryr_kubernetes/config.py @@ -237,6 +237,15 @@ nested_vif_driver_opts = [ default=''), ] +DEFAULT_PHYSNET_SUBNET_MAPPINGS = {} +sriov_opts = [ + cfg.DictOpt('default_physnet_subnets', + help=_("A mapping of default subnets for certain physnets " + "in a form of physnet-name:"), + default=DEFAULT_PHYSNET_SUBNET_MAPPINGS), +] + + CONF = cfg.CONF CONF.register_opts(kuryr_k8s_opts) CONF.register_opts(daemon_opts, group='cni_daemon') @@ -246,6 +255,7 @@ CONF.register_opts(octavia_defaults, group='octavia_defaults') CONF.register_opts(cache_defaults, group='cache_defaults') CONF.register_opts(ingress, group='ingress') CONF.register_opts(nested_vif_driver_opts, group='pod_vif_nested') +CONF.register_opts(sriov_opts, group='sriov') CONF.register_opts(lib_config.core_opts) CONF.register_opts(lib_config.binding_opts, 'binding') diff --git a/kuryr_kubernetes/constants.py b/kuryr_kubernetes/constants.py index 304ef9428..e384d74ed 100644 --- a/kuryr_kubernetes/constants.py +++ b/kuryr_kubernetes/constants.py @@ -65,3 +65,4 @@ VIF_POOL_SHOW = '/showPool' DEFAULT_IFNAME = 'eth0' ADDITIONAL_IFNAME_PREFIX = 'eth' +K8S_NPWG_SRIOV_PREFIX = "intel.com/sriov" diff --git a/kuryr_kubernetes/controller/drivers/sriov.py b/kuryr_kubernetes/controller/drivers/sriov.py new file mode 100644 index 000000000..d54e18125 --- /dev/null +++ b/kuryr_kubernetes/controller/drivers/sriov.py @@ -0,0 +1,132 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from kuryr.lib import constants as kl_const +from oslo_log import log as logging + +from kuryr_kubernetes import clients +from kuryr_kubernetes import config +from kuryr_kubernetes import constants +from kuryr_kubernetes.controller.drivers import neutron_vif +from kuryr_kubernetes.controller.drivers import utils as c_utils +from kuryr_kubernetes import os_vif_util as ovu + +LOG = logging.getLogger(__name__) + + +class SriovVIFDriver(neutron_vif.NeutronPodVIFDriver): + """Provides VIFs for SRIOV VF interfaces.""" + + ALIAS = 'sriov_pod_vif' + + def __init__(self): + self._physnet_mapping = self._get_physnet_mapping() + + def request_vif(self, pod, project_id, subnets, security_groups): + amount = self._get_remaining_sriov_vfs(pod) + if not amount: + LOG.error("SRIOV VIF request failed due to lack of " + "available VFs for the current pod creation") + return None + + pod_name = pod['metadata']['name'] + neutron = clients.get_neutron_client() + vif_plugin = 'sriov' + subnet_id = next(iter(subnets)) + physnet = self._get_physnet_for_subnet_id(subnet_id) + LOG.debug("Pod {} handling {}".format(pod_name, physnet)) + rq = self._get_port_request(pod, project_id, + subnets, security_groups) + + port = neutron.create_port(rq).get('port') + vif = ovu.neutron_to_osvif_vif(vif_plugin, port, subnets) + vif.physnet = physnet + + LOG.debug("{} vifs are available for the pod {}".format( + amount, pod_name)) + + self._reduce_remaining_sriov_vfs(pod) + return vif + + def activate_vif(self, pod, vif): + vif.active = True + + def _get_physnet_mapping(self): + physnets = config.CONF.sriov.default_physnet_subnets + + result = {} + for name, subnet_id in physnets.items(): + result[subnet_id] = name + return result + + def _get_physnet_for_subnet_id(self, subnet_id): + """Returns an appropriate physnet for exact subnet_id from mapping""" + try: + physnet = self._physnet_mapping[subnet_id] + except KeyError as ex: + LOG.error("No mapping for subnet {} in {}".format( + subnet_id, self._physnet_mapping)) + raise ex + return physnet + + def _get_remaining_sriov_vfs(self, pod): + """Returns the number of remaining vfs. + + Returns the number of remaining vfs from the initial number that + got allocated for the current pod. This information is stored in + pod object. + """ + containers = pod['spec']['containers'] + total_amount = 0 + for container in containers: + try: + requests = container['resources']['requests'] + amount_value = requests[constants.K8S_NPWG_SRIOV_PREFIX] + total_amount += int(amount_value) + except KeyError: + continue + + return total_amount + + def _reduce_remaining_sriov_vfs(self, pod): + """Reduces number of avaliable vfs for request""" + containers = pod['spec']['containers'] + for container in containers: + try: + requests = container['resources']['requests'] + num_of_sriov = int(requests[constants.K8S_NPWG_SRIOV_PREFIX]) + if num_of_sriov == 0: + continue + requests[constants.K8S_NPWG_SRIOV_PREFIX] = ( + str(num_of_sriov - 1)) + except KeyError: + continue + + def _get_port_request(self, pod, project_id, subnets, security_groups): + port_req_body = { + 'project_id': project_id, + 'name': c_utils.get_port_name(pod), + 'network_id': c_utils.get_network_id(subnets), + 'fixed_ips': ovu.osvif_to_neutron_fixed_ips(subnets), + 'device_owner': kl_const.DEVICE_OWNER + ':sriov', + 'device_id': c_utils.get_device_id(pod), + 'admin_state_up': True, + 'binding:vnic_type': 'direct', + 'binding:host_id': c_utils.get_host_id(pod), + } + + if security_groups: + port_req_body['security_groups'] = security_groups + + return {'port': port_req_body} diff --git a/kuryr_kubernetes/controller/drivers/vif_pool.py b/kuryr_kubernetes/controller/drivers/vif_pool.py index 63f6e3ad8..ef6e94db3 100644 --- a/kuryr_kubernetes/controller/drivers/vif_pool.py +++ b/kuryr_kubernetes/controller/drivers/vif_pool.py @@ -85,7 +85,8 @@ VIF_TYPE_TO_DRIVER_MAPPING = { 'VIFOpenVSwitch': 'neutron-vif', 'VIFBridge': 'neutron-vif', 'VIFVlanNested': 'nested-vlan', - 'VIFMacvlanNested': 'nested-macvlan' + 'VIFMacvlanNested': 'nested-macvlan', + 'VIFSriov': 'sriov' } diff --git a/kuryr_kubernetes/objects/vif.py b/kuryr_kubernetes/objects/vif.py index 730c0e8fa..c2d7cd844 100644 --- a/kuryr_kubernetes/objects/vif.py +++ b/kuryr_kubernetes/objects/vif.py @@ -68,3 +68,15 @@ class VIFMacvlanNested(obj_osvif.VIFBase): # Name of the device to create 'vif_name': obj_fields.StringField(), } + + +@obj_base.VersionedObjectRegistry.register +class VIFSriov(obj_osvif.VIFDirect): + # This is OVO based SRIOV vif. + + VERSION = '1.0' + + fields = { + # physnet of the VIF + 'physnet': obj_fields.StringField(), + } diff --git a/kuryr_kubernetes/opts.py b/kuryr_kubernetes/opts.py index 59286a6d2..789da513f 100644 --- a/kuryr_kubernetes/opts.py +++ b/kuryr_kubernetes/opts.py @@ -40,6 +40,7 @@ _kuryr_k8s_opts = [ ('namespace_subnet', namespace_subnet.namespace_subnet_driver_opts), ('namespace_sg', namespace_security_groups.namespace_sg_driver_opts), ('ingress', config.ingress), + ('sriov', config.sriov_opts), ] diff --git a/kuryr_kubernetes/os_vif_plug_noop.py b/kuryr_kubernetes/os_vif_plug_noop.py index 26dbe7f92..9d5137de3 100644 --- a/kuryr_kubernetes/os_vif_plug_noop.py +++ b/kuryr_kubernetes/os_vif_plug_noop.py @@ -38,3 +38,23 @@ class NoOpPlugin(PluginBase): def unplug(self, vif, instance_info): pass + + +class SriovPlugin(PluginBase): + """Sriov Plugin to be used with sriov VIFS""" + + def describe(self): + return objects.host_info.HostPluginInfo( + plugin_name='sriov', + vif_info=[ + objects.host_info.HostVIFInfo( + vif_object_name=objects.vif.VIFDirect.__name__, + min_version="1.0", + max_version="1.0"), + ]) + + def plug(self, vif, instance_info): + pass + + def unplug(self, vif, instance_info): + pass diff --git a/kuryr_kubernetes/os_vif_util.py b/kuryr_kubernetes/os_vif_util.py index 499e511af..30c411426 100644 --- a/kuryr_kubernetes/os_vif_util.py +++ b/kuryr_kubernetes/os_vif_util.py @@ -52,6 +52,11 @@ def neutron_to_osvif_network(neutron_network): if neutron_network.get('mtu') is not None: obj.mtu = neutron_network['mtu'] + # Vlan information will be used later in Sriov binding driver + if neutron_network.get('provider:network_type') == 'vlan': + obj.should_provide_vlan = True + obj.vlan = neutron_network['provider:segmentation_id'] + return obj @@ -295,6 +300,35 @@ def neutron_to_osvif_vif_nested_macvlan(neutron_port, subnets): vif_name=_get_vif_name(neutron_port)) +def neutron_to_osvif_vif_sriov(vif_plugin, neutron_port, subnets): + """Converts Neutron port to VIF object for SRIOV containers. + + :param vif_plugin: name of the os-vif plugin to use (i.e. 'noop') + :param neutron_port: dict containing port information as returned by + neutron client's 'show_port' + :param subnets: subnet mapping as returned by PodSubnetsDriver.get_subnets + :return: osv_vif VIFSriov object + """ + + details = neutron_port.get('binding:vif_details', {}) + network = _make_vif_network(neutron_port, subnets) + vlan_name = network.vlan if network.should_provide_vlan else '' + vif = k_vif.VIFSriov( + id=neutron_port['id'], + address=neutron_port['mac_address'], + network=network, + has_traffic_filtering=details.get('port_filter', False), + preserve_on_delete=False, + active=_is_port_active(neutron_port), + plugin=vif_plugin, + mode='passthrough', + vlan_name=vlan_name, + vif_name=_get_vif_name(neutron_port), + ) + + return vif + + def neutron_to_osvif_vif(vif_translator, neutron_port, subnets): """Converts Neutron port to os-vif VIF object. diff --git a/kuryr_kubernetes/tests/unit/controller/drivers/test_sriov.py b/kuryr_kubernetes/tests/unit/controller/drivers/test_sriov.py new file mode 100644 index 000000000..bcdfcf980 --- /dev/null +++ b/kuryr_kubernetes/tests/unit/controller/drivers/test_sriov.py @@ -0,0 +1,159 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from kuryr_kubernetes.controller.drivers import sriov as sriov_drivers +from kuryr_kubernetes.tests import base as test_base +from kuryr_kubernetes.tests.unit import kuryr_fixtures as k_fix + +from kuryr_kubernetes import constants as k_const +from kuryr_kubernetes import os_vif_util as ovu +from kuryr_kubernetes import utils + +from oslo_config import cfg as oslo_cfg +from oslo_utils import uuidutils + + +class TestSriovVIFDriver(test_base.TestCase): + + def setUp(self): + super(TestSriovVIFDriver, self).setUp() + self._pod = { + 'metadata': { + 'resourceVersion': mock.sentinel.pod_version, + 'selfLink': mock.sentinel.pod_link, + 'name': 'podname'}, + 'status': {'phase': k_const.K8S_POD_STATUS_PENDING}, + 'spec': { + 'hostNetwork': False, + 'nodeName': 'hostname', + 'containers': [{ + 'resources': { + 'requests': { + k_const.K8S_NPWG_SRIOV_PREFIX: "2" + } + } + }] + } + } + + def test_activate_vif(self): + cls = sriov_drivers.SriovVIFDriver + m_driver = mock.Mock(spec=cls) + + pod = mock.sentinel.pod + vif = mock.Mock() + vif.active = False + + cls.activate_vif(m_driver, pod, vif) + self.assertEqual(True, vif.active) + + @mock.patch('kuryr_kubernetes.os_vif_util.osvif_to_neutron_fixed_ips') + @mock.patch.object(ovu, 'neutron_to_osvif_vif') + def test_request_vif(self, m_to_vif, m_to_fips): + cls = sriov_drivers.SriovVIFDriver + m_driver = mock.Mock(spec=cls) + + neutron = self.useFixture(k_fix.MockNeutronClient()).client + project_id = mock.sentinel.project_id + fixed_ips = mock.sentinel.fixed_ips + m_to_fips.return_value = fixed_ips + network = mock.sentinel.Network + subnet_id = uuidutils.generate_uuid() + subnets = {subnet_id: network} + security_groups = mock.sentinel.security_groups + port_fixed_ips = mock.sentinel.port_fixed_ips + port_id = mock.sentinel.port_id + port = { + 'fixed_ips': port_fixed_ips, + 'id': port_id + } + port_request = mock.sentinel.port_request + m_driver._get_port_request.return_value = port_request + vif = mock.sentinel.vif + m_to_vif.return_value = vif + neutron.create_port.return_value = {'port': port} + utils.get_subnet.return_value = subnets + + self.assertEqual(vif, cls.request_vif(m_driver, self._pod, project_id, + subnets, security_groups)) + + neutron.create_port.assert_called_once_with(port_request) + + @mock.patch('kuryr_kubernetes.os_vif_util.osvif_to_neutron_fixed_ips') + @mock.patch.object(ovu, 'neutron_to_osvif_vif') + def test_request_vif_not_enough_vfs(self, m_to_vif, m_to_fips): + cls = sriov_drivers.SriovVIFDriver + m_driver = mock.Mock(spec=cls) + + m_driver._get_remaining_sriov_vfs.return_value = 0 + neutron = self.useFixture(k_fix.MockNeutronClient()).client + project_id = mock.sentinel.project_id + network = mock.sentinel.Network + subnet_id = uuidutils.generate_uuid() + subnets = {subnet_id: network} + security_groups = mock.sentinel.security_groups + + self.assertIsNone(cls.request_vif(m_driver, self._pod, project_id, + subnets, security_groups)) + + neutron.create_port.assert_not_called() + + def test_get_sriov_num_vf(self): + cls = sriov_drivers.SriovVIFDriver + m_driver = mock.Mock(spec=cls) + + amount = cls._get_remaining_sriov_vfs(m_driver, self._pod) + self.assertEqual(amount, 2) + + def test_reduce_remaining_sriov_vfs(self): + cls = sriov_drivers.SriovVIFDriver + m_driver = mock.Mock(spec=cls) + + cls._reduce_remaining_sriov_vfs(m_driver, self._pod) + amount = cls._get_remaining_sriov_vfs(m_driver, self._pod) + self.assertEqual(amount, 1) + + def test_get_physnet_mapping(self): + cls = sriov_drivers.SriovVIFDriver + m_driver = mock.Mock(spec=cls) + + subnet_id = uuidutils.generate_uuid() + oslo_cfg.CONF.set_override('default_physnet_subnets', + 'physnet10_4:'+str(subnet_id), + group='sriov') + + mapping = cls._get_physnet_mapping(m_driver) + self.assertEqual(mapping, {subnet_id: 'physnet10_4'}) + + def test_get_physnet_for_subnet_id(self): + cls = sriov_drivers.SriovVIFDriver + m_driver = mock.Mock(spec=cls) + + subnet_id = uuidutils.generate_uuid() + m_driver._physnet_mapping = {subnet_id: 'physnet10_4'} + + physnet = cls._get_physnet_for_subnet_id(m_driver, subnet_id) + self.assertEqual(physnet, 'physnet10_4') + + def test_get_physnet_for_subnet_id_error(self): + cls = sriov_drivers.SriovVIFDriver + m_driver = mock.Mock(spec=cls) + + subnet_id = uuidutils.generate_uuid() + m_driver._physnet_mapping = {} + + self.assertRaises(KeyError, cls._get_physnet_for_subnet_id, + m_driver, subnet_id) diff --git a/kuryr_kubernetes/utils.py b/kuryr_kubernetes/utils.py index 6fd14a100..81307dd97 100644 --- a/kuryr_kubernetes/utils.py +++ b/kuryr_kubernetes/utils.py @@ -33,7 +33,8 @@ LOG = log.getLogger(__name__) VALID_MULTI_POD_POOLS_OPTS = {'noop': ['neutron-vif', 'nested-vlan', - 'nested-macvlan'], + 'nested-macvlan', + 'sriov'], 'neutron': ['neutron-vif'], 'nested': ['nested-vlan'], } diff --git a/setup.cfg b/setup.cfg index 9276e2317..5e5ba5b8f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,6 +25,7 @@ oslo.config.opts = os_vif = noop = kuryr_kubernetes.os_vif_plug_noop:NoOpPlugin + sriov = kuryr_kubernetes.os_vif_plug_noop:SriovPlugin console_scripts = kuryr-k8s-controller = kuryr_kubernetes.cmd.eventlet.controller:start @@ -33,6 +34,7 @@ console_scripts = kuryr_kubernetes.vif_translators = ovs = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_ovs + sriov = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_sriov kuryr_kubernetes.cni.binding = VIFBridge = kuryr_kubernetes.cni.binding.bridge:BridgeDriver @@ -74,6 +76,7 @@ kuryr_kubernetes.controller.drivers.pod_vif = neutron-vif = kuryr_kubernetes.controller.drivers.neutron_vif:NeutronPodVIFDriver nested-vlan = kuryr_kubernetes.controller.drivers.nested_vlan_vif:NestedVlanPodVIFDriver nested-macvlan = kuryr_kubernetes.controller.drivers.nested_macvlan_vif:NestedMacvlanPodVIFDriver + sriov = kuryr_kubernetes.controller.drivers.sriov:SriovVIFDriver kuryr_kubernetes.controller.drivers.endpoints_lbaas = lbaasv2 = kuryr_kubernetes.controller.drivers.lbaasv2:LBaaSv2Driver