From 3c3bb020c87837cc6a326f5aba1226d1649ec681 Mon Sep 17 00:00:00 2001 From: Hongbin Lu Date: Mon, 4 Sep 2017 00:23:12 -0400 Subject: [PATCH] Introduce sriov port driver * Introduce a new config 'enabled_port_drivers'. This config allows operators to specify multiple port drivers to enable. This enables Kuryr to dynamically choose from a list of port drivers (Veth or SRIOV). * Introduce a new SRIOV port driver for performing SRIOV port binding * Choose port driver according to the type of the port. If the neutron port has SRIOV vnic type, choose the SRIOV port driver. Otherwise, choose the normal port driver (i.e. veth). * To use this feature, users are supposed to pre-create a SRIOV port and have the binding:profile populated. Furthermore, users should pass the SRIOV port to Kuryr (i.e. using --mac-address and/or --ip on 'docker run') so that Kuryr will choose the right driver. Implements: blueprint sriov-support Change-Id: I0d6552ce4a2c50edb164aff3de802e6239671c2c --- doc/source/config-sriov.rst | 124 ++++++++++++ doc/source/index.rst | 1 + kuryr_libnetwork/config.py | 9 +- kuryr_libnetwork/constants.py | 11 ++ kuryr_libnetwork/controllers.py | 41 +++- kuryr_libnetwork/port_driver/base.py | 6 +- kuryr_libnetwork/port_driver/driver.py | 36 ++-- kuryr_libnetwork/port_driver/drivers/sriov.py | 153 +++++++++++++++ kuryr_libnetwork/port_driver/drivers/veth.py | 6 +- kuryr_libnetwork/tests/unit/base.py | 6 +- .../unit/port_driver/drivers/test_nested.py | 5 +- .../unit/port_driver/drivers/test_sriov.py | 178 ++++++++++++++++++ .../unit/port_driver/drivers/test_veth.py | 5 +- .../unit/port_driver/drivers/test_vlan.py | 5 +- .../tests/unit/port_driver/test_base.py | 2 +- .../tests/unit/port_driver/test_driver.py | 37 +++- kuryr_libnetwork/tests/unit/test_kuryr.py | 31 +-- .../tests/unit/test_kuryr_endpoint.py | 8 +- 18 files changed, 611 insertions(+), 53 deletions(-) create mode 100644 doc/source/config-sriov.rst create mode 100644 kuryr_libnetwork/port_driver/drivers/sriov.py create mode 100644 kuryr_libnetwork/tests/unit/port_driver/drivers/test_sriov.py diff --git a/doc/source/config-sriov.rst b/doc/source/config-sriov.rst new file mode 100644 index 00000000..ad29c8ca --- /dev/null +++ b/doc/source/config-sriov.rst @@ -0,0 +1,124 @@ +====== +SR-IOV +====== + +The purpose of this page is to describe how to enable SR-IOV functionality +available in Kuryr-libnetwork. This page intends to serve as a guide for +how to configure OpenStack Networking and Kuryr-libnetwork to create SR-IOV +ports and leverage them for containers. + +The basics +~~~~~~~~~~ + +PCI-SIG Single Root I/O Virtualization and Sharing (SR-IOV) functionality is +available in OpenStack since the Juno release. The SR-IOV specification +defines a standardized mechanism to virtualize PCIe devices. This mechanism +can virtualize a single PCIe Ethernet controller to appear as multiple PCIe +devices. Each device can be directly assigned to an instance, bypassing the +virtual switch layer. As a result, users are able to achieve low latency and +near-line wire speed. + +The following terms are used throughout this document: + +.. list-table:: + :header-rows: 1 + :widths: 10 90 + + * - Term + - Definition + * - PF + - Physical Function. The physical Ethernet controller that supports + SR-IOV. + * - VF + - Virtual Function. The virtual PCIe device created from a physical + Ethernet controller. + +Using SR-IOV interfaces +~~~~~~~~~~~~~~~~~~~~~~~ + +In order to enable SR-IOV, the following steps are required: + +#. Create Virtual Functions (Compute) +#. Configure neutron-server (Controller) +#. Enable neutron sriov-agent (Compute) +#. Configure kuryr-libnetwork (Compute) + +Create Virtual Functions (Compute) +---------------------------------- + +Follow the session 'Create Virtual Functions' in the `networking guide +`_. + +Configure neutron-server (Controller) +------------------------------------- + +Follow the session 'Configure neutron-server' in the `networking guide +`_. + +Enable neutron sriov-agent (Compute) +------------------------------------- + +Follow the session 'Enable neutron sriov-agent' in the `networking guide +`_. + +Configure kuryr-libnetwork (Compute) +------------------------------------ + +#. On every compute node running the ``kuryr-libnetwork`` service, + edit kuryr-libnetwork config file (e.g. /etc/kuryr/kuryr.conf). Add + ``kuryr_libnetwork.port_driver.drivers.sriov`` to + ``enabled_port_drivers`` under ``[DEFAULT]`` and + add ``kuryr.lib.binding.drivers.hw_veb`` to ``enabled_drivers`` + under ``[binding]``. + + .. code-block:: ini + + [DEFAULT] + enabled_port_drivers = kuryr_libnetwork.port_driver.drivers.veth, kuryr_libnetwork.port_driver.drivers.sriov + + [binding] + enabled_drivers = kuryr.lib.binding.drivers.veth, kuryr.lib.binding.drivers.hw_veb + +#. Restart the ``kuryr-libnetwork`` service. + +Launching containers with SR-IOV ports +-------------------------------------- + +Once configuration is complete, you can launch containers with SR-IOV ports. + +#. Get the ``id`` of the network where you want the SR-IOV port to be created: + + .. code-block:: console + + $ net_id=`neutron net-show net04 | grep "\ id\ " | awk '{ print $4 }'` + +#. Create a kuryr network by specifying the name of the neutron network. + Replace ``10.10.0.0/24`` and ``10.10.0.1`` with the CIDR and gateway + of the subnet where you want the SR-IOV port to be created: + + .. code-block:: console + + $ docker network create -d kuryr --ipam-driver=kuryr --subnet=10.10.0.0/24 --gateway=10.10.0.1 \ + -o neutron.net.uuid=$net_id kuryr_net + +#. Create the SR-IOV port. ``vnic_type=direct`` is used here, but other options + include ``normal``, ``direct-physical``, and ``macvtap``. + The ``binding-profile`` is used by the Neutron SR-IOV driver [1]. + Replace ``physnet2``, ``1137:0047``, and ``0000:0a:00.1`` + with the correct value of the VF device: + + .. code-block:: console + + $ neutron port-create $net_id --name sriov_port --binding:vnic_type direct \ + --binding-profile '{"physical_network": "physnet2", "pci_vendor_info": "1137:0047", "pci_slot": "0000:0a:00.1"}' + +#. Create the container. Specify the SR-IOV port's IP address created in step + two: + + .. code-block:: console + + $ docker run -it --net=kuryr_net --ip=10.0.0.5 ubuntu + +Reference +--------- +[1] https://specs.openstack.org/openstack/neutron-specs/specs/juno/ml2-sriov-nic-switch.html diff --git a/doc/source/index.rst b/doc/source/index.rst index f58e8211..fecaa8ea 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -12,6 +12,7 @@ Contents: :maxdepth: 2 readme + config-sriov.rst Design and Developer Docs ========================== diff --git a/kuryr_libnetwork/config.py b/kuryr_libnetwork/config.py index 16e0a23b..bc6d37a2 100644 --- a/kuryr_libnetwork/config.py +++ b/kuryr_libnetwork/config.py @@ -44,7 +44,14 @@ core_opts = [ help=_('There is no address-space by default in neutron')), cfg.StrOpt('port_driver', default='kuryr_libnetwork.port_driver.drivers.veth', - help=_('Driver for the desired deployment model')), + deprecated_for_removal=True, + help=_('Default driver for the desired deployment model')), + cfg.StrOpt('default_port_driver', + default='kuryr_libnetwork.port_driver.drivers.veth', + help=_('Default driver for the desired deployment model')), + cfg.ListOpt('enabled_port_drivers', + default=['kuryr_libnetwork.port_driver.drivers.veth'], + help=_('Available port drivers')), cfg.StrOpt('ssl_cert_file', default='/var/lib/kuryr/certs/cert.pem', help=_('This option allows setting absolute path' diff --git a/kuryr_libnetwork/constants.py b/kuryr_libnetwork/constants.py index 28c45e47..84747f46 100644 --- a/kuryr_libnetwork/constants.py +++ b/kuryr_libnetwork/constants.py @@ -53,3 +53,14 @@ NEUTRON_UUID_OPTION = 'neutron.net.uuid' REQUEST_ADDRESS_TYPE = 'RequestAddressType' KURYR_UNBOUND_PORT = 'kuryr-unbound-port' NEUTRON_UNBOUND_PORT = 'neutron-unbound-port' +BINDING_PROFILE = 'binding:profile' + +# Define supported virtual NIC types. +VNIC_TYPE_NORMAL = 'normal' +VNIC_TYPE_DIRECT = 'direct' +VNIC_TYPE_MACVTAP = 'macvtap' +VNIC_TYPE_DIRECT_PHYSICAL = 'direct-physical' + +# Define list of virtual NIC types. +VNIC_TYPES_SRIOV = (VNIC_TYPE_DIRECT, VNIC_TYPE_MACVTAP, + VNIC_TYPE_DIRECT_PHYSICAL) diff --git a/kuryr_libnetwork/controllers.py b/kuryr_libnetwork/controllers.py index 33d748cd..e3e79b45 100644 --- a/kuryr_libnetwork/controllers.py +++ b/kuryr_libnetwork/controllers.py @@ -45,6 +45,17 @@ TAG_NEUTRON_EXTENSION = "tag" TAG_EXT_NEUTRON_EXTENSION = "tag-ext" SUBNET_POOLS_V4 = [] SUBNET_POOLS_V6 = [] +DEFAULT_DRIVER = driver.get_driver_instance() +try: + SRIOV_DRIVER = driver.get_driver_instance(name='sriov') +except exceptions.KuryrException: + SRIOV_DRIVER = None +VNIC_TYPES_DRIVERS_MAPPING = { + const.VNIC_TYPE_NORMAL: DEFAULT_DRIVER, + const.VNIC_TYPE_DIRECT: SRIOV_DRIVER, + const.VNIC_TYPE_MACVTAP: SRIOV_DRIVER, + const.VNIC_TYPE_DIRECT_PHYSICAL: SRIOV_DRIVER, +} def get_neutron_client(): @@ -97,10 +108,20 @@ def load_default_subnet_pools(): def load_port_driver(): - app.driver = driver.get_driver_instance() + app.driver = DEFAULT_DRIVER LOG.debug("Using port driver '%s'", str(app.driver)) +def get_driver(port): + vnic_type = port.get('binding:vnic_type', const.VNIC_TYPE_NORMAL) + driver = VNIC_TYPES_DRIVERS_MAPPING.get(vnic_type) + if driver is None: + raise exceptions.KuryrException( + "No port driver available for VNIC type %s" % vnic_type) + else: + return driver + + def _cache_default_subnetpool_ids(app): """Caches IDs of the default subnetpools as app.DEFAULT_POOL_IDS.""" if not hasattr(app, 'DEFAULT_POOL_IDS'): @@ -275,8 +296,9 @@ def _create_or_update_port(neutron_network_id, endpoint_id, interface_mac, fixed_ips) elif num_port == 1: port = filtered_ports['ports'][0] - response_port = app.driver.update_port(port, endpoint_id, - interface_mac) + port_driver = get_driver(port) + response_port = port_driver.update_port(port, endpoint_id, + interface_mac) # For the container boot from dual-net, request_address will # create two ports(v4 and v6 address), we should only allow one # for port bind. @@ -1113,7 +1135,8 @@ def network_driver_create_endpoint(): neutron_network_id, endpoint_id, interface_cidrv4, interface_cidrv6, interface_mac) try: - (stdout, stderr) = app.driver.create_host_iface( + port_driver = get_driver(neutron_port) + (stdout, stderr) = port_driver.create_host_iface( endpoint_id, neutron_port, subnets, filtered_networks[0]) LOG.debug(stdout) if stderr: @@ -1149,6 +1172,10 @@ def network_driver_create_endpoint(): if not interface_mac: response_interface['MacAddress'] = neutron_port['mac_address'] + vnic_type = neutron_port.get('binding:vnic_type') + if vnic_type in const.VNIC_TYPES_SRIOV: + response_interface.pop('MacAddress', None) + if not (interface_cidrv4 or interface_cidrv6): if 'ip_address' in neutron_port: _process_interface_address( @@ -1233,7 +1260,8 @@ def network_driver_delete_endpoint(): neutron_port = filtered_ports[0] try: - stdout, stderr = app.driver.delete_host_iface( + port_driver = get_driver(neutron_port) + stdout, stderr = port_driver.delete_host_iface( endpoint_id, neutron_port) LOG.debug(stdout) if stderr: @@ -1325,7 +1353,8 @@ def network_driver_join(): "Multiple Kuryr subnets exist for the network_id={0} " .format(neutron_network_id)) - iface_name = app.driver.get_container_iface_name(neutron_port['id']) + port_driver = get_driver(neutron_port) + iface_name = port_driver.get_container_iface_name(neutron_port) join_response = { "InterfaceName": { diff --git a/kuryr_libnetwork/port_driver/base.py b/kuryr_libnetwork/port_driver/base.py index fba99013..5f5c06eb 100644 --- a/kuryr_libnetwork/port_driver/base.py +++ b/kuryr_libnetwork/port_driver/base.py @@ -50,11 +50,11 @@ class BaseNestedDriver(driver.Driver): raise exceptions.KuryrException("Cannot find a Neutron port " "associated to interface name {0}".format(ifname)) - def get_container_iface_name(self, neutron_port_id): + def get_container_iface_name(self, neutron_port): """Returns interface name of a container in the default namespace. - :param neutron_port_id: The ID of a neutron port as string + :param neutron_port: The neutron port :returns: interface name as string. """ - _, container_iface_name = utils.get_veth_pair_names(neutron_port_id) + _, container_iface_name = utils.get_veth_pair_names(neutron_port['id']) return container_iface_name diff --git a/kuryr_libnetwork/port_driver/driver.py b/kuryr_libnetwork/port_driver/driver.py index 3ef039d8..a8ee954c 100644 --- a/kuryr_libnetwork/port_driver/driver.py +++ b/kuryr_libnetwork/port_driver/driver.py @@ -99,10 +99,10 @@ class Driver(object): raise NotImplementedError() @abc.abstractmethod - def get_container_iface_name(self, neutron_port_id): + def get_container_iface_name(self, neutron_port): """Returns interface name of a container in the default namespace. - :param neutron_port_id: The ID of a neutron port as string + :param neutron_port: The neutron port :returns: interface name as string """ raise NotImplementedError() @@ -150,13 +150,20 @@ class Driver(object): return self.__class__.__name__ -def get_driver_instance(): +def get_driver_instance(name=None): """Instantiate a driver instance accordingly to the file configuration. :returns: a Driver instance :raises: exceptions.KuryrException """ - module, name, classname = _parse_port_driver_config() + if name: + module, name, classname = _parse_port_driver_config(name) + else: + module, name, classname = _parse_port_driver_config() + + if (module not in config.CONF.enabled_port_drivers and + name not in config.CONF.enabled_port_drivers): + raise exceptions.KuryrException("No port driver available") # TODO(apuimedo): switch to the openstack/stevedore plugin system try: @@ -171,13 +178,14 @@ def get_driver_instance(): return driver -def _parse_port_driver_config(): +def _parse_port_driver_config(config_value=None): """Read the port driver related config value and parse it. :returns: the provided full module path as per config file, the name of the driver/module and the class name of the Driver class inside it """ - config_value = config.CONF.port_driver + if config_value is None: + config_value = config.CONF.default_port_driver config_tokens = config_value.rsplit('.', 1) if len(config_tokens) == 1: # not a path, just a name name = config_tokens[0] @@ -203,21 +211,23 @@ def _verify_port_driver_compliancy(driver, port_driver_name): def _verify_binding_driver_compatibility(driver, port_driver_name): - tokens = config.CONF.binding.default_driver.rsplit('.', 1) - binding_driver_name = tokens[0] if len(tokens) == 1 else tokens[1] - binding_driver_name.lower() + binding_drivers_names = [] + for binding_driver in config.CONF.binding.enabled_drivers: + tokens = binding_driver.rsplit('.', 1) + binding_driver_name = tokens[0] if len(tokens) == 1 else tokens[1] + binding_drivers_names.append(binding_driver_name.lower()) # TODO(mchiappe): find a clean way to test the binding driver # is also loadable before we start supported_bindings = driver.get_supported_bindings() - if binding_driver_name not in supported_bindings: + if not set(binding_drivers_names) & set(supported_bindings): raise exceptions.KuryrException("Configuration file error: " "port driver '{0}' is not compatible with binding driver '{1}'" - .format(port_driver_name, binding_driver_name)) + .format(port_driver_name, binding_drivers_names)) # Temporarily ban IPVLAN, to be removed in the future - if binding_driver_name == 'ipvlan': + if 'ipvlan' in binding_drivers_names: raise exceptions.KuryrException("Configuration file error: " "binding driver '{0}' is currently not supported with '{1}' " - "port driver".format(binding_driver_name, port_driver_name)) + "port driver".format(binding_drivers_names, port_driver_name)) diff --git a/kuryr_libnetwork/port_driver/drivers/sriov.py b/kuryr_libnetwork/port_driver/drivers/sriov.py new file mode 100644 index 00000000..1ca4210c --- /dev/null +++ b/kuryr_libnetwork/port_driver/drivers/sriov.py @@ -0,0 +1,153 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import glob +import os +import re + +from kuryr.lib import binding +from kuryr.lib import exceptions + +from kuryr_libnetwork import constants as const +from kuryr_libnetwork.port_driver import driver + + +def get_ifname_by_pci_address(pci_addr, pf_interface=False): + """Get the interface name based on a VF's pci address. + + The returned interface name is either the parent PF's or that of the VF + itself based on the argument of pf_interface. + """ + dev_path = _get_sysfs_netdev_path(pci_addr, pf_interface) + try: + dev_info = os.listdir(dev_path) + return dev_info.pop() + except Exception: + raise exceptions.KuryrException( + "PCI device %s not found" % pci_addr) + + +def _get_sysfs_netdev_path(pci_addr, pf_interface): + """Get the sysfs path based on the PCI address of the device. + + Assumes a networking device - will not check for the existence of the path. + """ + if pf_interface: + return "/sys/bus/pci/devices/%s/physfn/net" % pci_addr + return "/sys/bus/pci/devices/%s/net" % pci_addr + + +def get_vf_num_by_pci_address(pci_addr): + """Get the VF number based on a VF's pci address + + A VF is associated with an VF number, which ip link command uses to + configure it. This number can be obtained from the PCI device filesystem. + """ + VIRTFN_RE = re.compile("virtfn(\d+)") + virtfns_path = "/sys/bus/pci/devices/%s/physfn/virtfn*" % (pci_addr) + vf_num = None + try: + for vf_path in glob.iglob(virtfns_path): + if re.search(pci_addr, os.readlink(vf_path)): + t = VIRTFN_RE.search(vf_path) + vf_num = t.group(1) + break + except Exception: + pass + if vf_num is None: + raise exceptions.KuryrException( + "PCI device %s not found" % pci_addr) + return vf_num + + +class SriovDriver(driver.Driver): + """Driver supporting SR-IOV on Bare Metal""" + + BINDING_DRIVERS = ('hw_veb',) + + def get_supported_bindings(self): + """Returns a tuple of supported binding driver names for the driver. + + :returns: a tuple of strings + """ + return self.BINDING_DRIVERS + + def get_default_network_id(self): + """Returns a Neutron network ID as per driver logic, if any. + + This driver does not make use of any specific network and will thus + return None. + + :returns: None + """ + return None + + def create_host_iface(self, endpoint_id, neutron_port, subnets, + network=None): + """Instantiates a host interface and bind it to the host. + + :param endpoint_id: the ID of the endpoint as string + :param neutron_port: the container Neutron port dictionary as returned + by python-neutronclient + :param subnets: an iterable of all the Neutron subnets which the + endpoint is trying to join + :param network: the Neutron network which the endpoint is trying + to join + :returns: the tuple of stdout and stderr returned by + processutils.execute invoked with the executable script for + unbinding + :raises: kuryr.lib.exceptions.BindingNotSupportedFailure + processutils.ProcessExecutionError + """ + binding_driver = 'kuryr.lib.binding.drivers.hw_veb' + pci_addr = neutron_port[const.BINDING_PROFILE]['pci_slot'] + pf_ifname = get_ifname_by_pci_address(pci_addr, + pf_interface=True) + vf_num = get_vf_num_by_pci_address(pci_addr) + _, _, (stdout, stderr) = binding.port_bind( + endpoint_id, neutron_port, subnets, pf_ifname=pf_ifname, + vf_num=vf_num, driver=binding_driver) + return (stdout, stderr) + + def delete_host_iface(self, endpoint_id, neutron_port): + """Deletes a host interface after unbinding it from the host. + + The host veth interface associated to the Neutron port will be unbound + from its vitual bridge and deleted by delegating to the selected + kuryr-lib driver. + + :param endpoint_id: the ID of the Docker container as string + :param neutron_port: a port dictionary returned from + python-neutronclient + :returns: the tuple of stdout and stderr returned by + processutils.execute invoked with the executable script for + unbinding + :raises: processutils.ProcessExecutionError + """ + binding_driver = 'kuryr.lib.binding.drivers.hw_veb' + pci_addr = neutron_port[const.BINDING_PROFILE]['pci_slot'] + pf_ifname = get_ifname_by_pci_address(pci_addr, + pf_interface=True) + vf_num = get_vf_num_by_pci_address(pci_addr) + return binding.port_unbind(endpoint_id, neutron_port, + pf_ifname=pf_ifname, + vf_num=vf_num, driver=binding_driver) + + def get_container_iface_name(self, neutron_port): + """Returns interface name of a container in the default namespace. + + :param neutron_port_id: The ID of a neutron port as string + :returns: interface name as string + """ + pci_addr = neutron_port[const.BINDING_PROFILE]['pci_slot'] + vf_ifname = get_ifname_by_pci_address(pci_addr) + return vf_ifname diff --git a/kuryr_libnetwork/port_driver/drivers/veth.py b/kuryr_libnetwork/port_driver/drivers/veth.py index 32da0675..0f7e6204 100644 --- a/kuryr_libnetwork/port_driver/drivers/veth.py +++ b/kuryr_libnetwork/port_driver/drivers/veth.py @@ -80,11 +80,11 @@ class VethDriver(driver.Driver): """ return binding.port_unbind(endpoint_id, neutron_port) - def get_container_iface_name(self, neutron_port_id): + def get_container_iface_name(self, neutron_port): """Returns interface name of a container in the default namespace. - :param neutron_port_id: The ID of a neutron port as string + :param neutron_port_id: The neutron port :returns: interface name as string """ - _, container_iface_name = utils.get_veth_pair_names(neutron_port_id) + _, container_iface_name = utils.get_veth_pair_names(neutron_port['id']) return container_iface_name diff --git a/kuryr_libnetwork/tests/unit/base.py b/kuryr_libnetwork/tests/unit/base.py index 5465bf3c..4009a4b2 100644 --- a/kuryr_libnetwork/tests/unit/base.py +++ b/kuryr_libnetwork/tests/unit/base.py @@ -169,7 +169,8 @@ class TestKuryrBase(TestCase): device_owner=None, neutron_trunk_id=None, tags=None, - name=None): + name=None, + binding_profile=None): # The following fake response is retrieved from the Neutron doc: # http://developer.openstack.org/api-ref-networking-v2.html#createPort # noqa if not name: @@ -192,6 +193,9 @@ class TestKuryrBase(TestCase): } } + if binding_profile is not None: + fake_port['port']['binding:profile'] = binding_profile + if neutron_subnet_v4_id: fake_port['port']['fixed_ips'].append({ "subnet_id": neutron_subnet_v4_id, diff --git a/kuryr_libnetwork/tests/unit/port_driver/drivers/test_nested.py b/kuryr_libnetwork/tests/unit/port_driver/drivers/test_nested.py index ee2847ff..6e340cc5 100644 --- a/kuryr_libnetwork/tests/unit/port_driver/drivers/test_nested.py +++ b/kuryr_libnetwork/tests/unit/port_driver/drivers/test_nested.py @@ -181,7 +181,10 @@ class TestNestedDriver(base.TestKuryrBase): def test_get_container_iface_name(self, mock_get_pair_names): nested_driver = nested.NestedDriver() fake_neutron_port_id = uuidutils.generate_uuid() - response = nested_driver.get_container_iface_name(fake_neutron_port_id) + fake_neutron_port = self._get_fake_port( + uuidutils.generate_uuid(), uuidutils.generate_uuid(), + fake_neutron_port_id)['port'] + response = nested_driver.get_container_iface_name(fake_neutron_port) mock_get_pair_names.assert_called_with(fake_neutron_port_id) self.assertEqual(response, "fake_container_name") diff --git a/kuryr_libnetwork/tests/unit/port_driver/drivers/test_sriov.py b/kuryr_libnetwork/tests/unit/port_driver/drivers/test_sriov.py new file mode 100644 index 00000000..766db323 --- /dev/null +++ b/kuryr_libnetwork/tests/unit/port_driver/drivers/test_sriov.py @@ -0,0 +1,178 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import glob +import mock +import os + +from oslo_utils import uuidutils + +from kuryr.lib import binding +from kuryr.lib import exceptions +from kuryr.lib import utils as lib_utils +from kuryr_libnetwork.port_driver.drivers import sriov +from kuryr_libnetwork.tests.unit import base + + +class TestSriovDriver(base.TestKuryrBase): + """Unit tests for veth driver""" + + def test_get_supported_bindings(self): + sriov_driver = sriov.SriovDriver() + supported_bindings = sriov_driver.get_supported_bindings() + self.assertEqual(supported_bindings, sriov.SriovDriver.BINDING_DRIVERS) + + @mock.patch.object(os, 'readlink') + @mock.patch.object(glob, 'iglob') + @mock.patch.object(os, 'listdir') + @mock.patch.object(binding, 'port_bind') + def test_create_host_iface(self, mock_port_bind, mock_listdir, + mock_iglob, mock_readlink): + sriov_driver = sriov.SriovDriver() + fake_endpoint_id = lib_utils.get_hash() + fake_neutron_port_id = uuidutils.generate_uuid() + fake_neutron_port = self._get_fake_port( + fake_endpoint_id, uuidutils.generate_uuid(), + fake_neutron_port_id, + binding_profile={'pci_slot': '0000:0a:00.1'})['port'] + fake_subnets = mock.sentinel.binding_subnets + fake_pf_ifname = 'eth3' + mock_listdir.return_value = [fake_pf_ifname] + mock_iglob.return_value = [ + '/sys/bus/pci/devices/0000:0a:00.1/physfn/virtfn3', + ] + mock_readlink.return_value = '../../0000:0a:00.1' + fake_exec_response = ('fake_stdout', '') + mock_port_bind.return_value = ('fake_host_ifname', + 'fake_container_ifname', fake_exec_response) + + response = sriov_driver.create_host_iface(fake_endpoint_id, + fake_neutron_port, fake_subnets) + self.assertEqual(response, fake_exec_response) + mock_port_bind.assert_called_with(fake_endpoint_id, + fake_neutron_port, fake_subnets, + pf_ifname=fake_pf_ifname, vf_num='3', + driver='kuryr.lib.binding.drivers.hw_veb') + mock_listdir.assert_called_with( + '/sys/bus/pci/devices/0000:0a:00.1/physfn/net') + mock_iglob.assert_called_with( + '/sys/bus/pci/devices/0000:0a:00.1/physfn/virtfn*') + mock_readlink.assert_called_with( + '/sys/bus/pci/devices/0000:0a:00.1/physfn/virtfn3') + + @mock.patch.object(os, 'readlink') + @mock.patch.object(glob, 'iglob') + @mock.patch.object(os, 'listdir') + @mock.patch.object(binding, 'port_bind') + def test_create_host_iface_pf_not_found( + self, mock_port_bind, mock_listdir, mock_iglob, mock_readlink): + sriov_driver = sriov.SriovDriver() + fake_endpoint_id = lib_utils.get_hash() + fake_neutron_port_id = uuidutils.generate_uuid() + fake_neutron_port = self._get_fake_port( + fake_endpoint_id, uuidutils.generate_uuid(), + fake_neutron_port_id, + binding_profile={'pci_slot': '0000:0a:00.1'})['port'] + fake_subnets = mock.sentinel.binding_subnets + mock_listdir.side_effect = OSError('No such file or directory') + + self.assertRaises(exceptions.KuryrException, + sriov_driver.create_host_iface, + fake_endpoint_id, fake_neutron_port, fake_subnets) + mock_port_bind.assert_not_called() + mock_listdir.assert_called_with( + '/sys/bus/pci/devices/0000:0a:00.1/physfn/net') + mock_iglob.assert_not_called() + mock_readlink.assert_not_called() + + @mock.patch.object(os, 'readlink') + @mock.patch.object(glob, 'iglob') + @mock.patch.object(os, 'listdir') + @mock.patch.object(binding, 'port_bind') + def test_create_host_iface_vf_num_not_found( + self, mock_port_bind, mock_listdir, mock_iglob, mock_readlink): + sriov_driver = sriov.SriovDriver() + fake_endpoint_id = lib_utils.get_hash() + fake_neutron_port_id = uuidutils.generate_uuid() + fake_neutron_port = self._get_fake_port( + fake_endpoint_id, uuidutils.generate_uuid(), + fake_neutron_port_id, + binding_profile={'pci_slot': '0000:0a:00.1'})['port'] + fake_subnets = mock.sentinel.binding_subnets + fake_pf_ifname = 'eth3' + mock_listdir.return_value = [fake_pf_ifname] + mock_iglob.return_value = [ + '/sys/bus/pci/devices/0000:0a:00.1/physfn/virtfn3', + ] + mock_readlink.return_value = '../../0000:0a:00.2' + + self.assertRaises(exceptions.KuryrException, + sriov_driver.create_host_iface, + fake_endpoint_id, fake_neutron_port, fake_subnets) + mock_port_bind.assert_not_called() + mock_listdir.assert_called_with( + '/sys/bus/pci/devices/0000:0a:00.1/physfn/net') + mock_iglob.assert_called_with( + '/sys/bus/pci/devices/0000:0a:00.1/physfn/virtfn*') + mock_readlink.assert_called_with( + '/sys/bus/pci/devices/0000:0a:00.1/physfn/virtfn3') + + @mock.patch.object(os, 'readlink') + @mock.patch.object(glob, 'iglob') + @mock.patch.object(os, 'listdir') + @mock.patch.object(binding, 'port_unbind') + def test_delete_host_iface(self, mock_port_unbind, mock_listdir, + mock_iglob, mock_readlink): + sriov_driver = sriov.SriovDriver() + fake_endpoint_id = lib_utils.get_hash() + fake_neutron_port_id = uuidutils.generate_uuid() + fake_neutron_port = self._get_fake_port( + fake_endpoint_id, uuidutils.generate_uuid(), + fake_neutron_port_id, + binding_profile={'pci_slot': '0000:0a:00.1'})['port'] + fake_pf_ifname = 'eth3' + mock_listdir.return_value = [fake_pf_ifname] + mock_iglob.return_value = [ + '/sys/bus/pci/devices/0000:0a:00.1/physfn/virtfn3', + ] + mock_readlink.return_value = '../../0000:0a:00.1' + fake_unbind_response = ('fake_stdout', '') + mock_port_unbind.return_value = fake_unbind_response + + response = sriov_driver.delete_host_iface(fake_endpoint_id, + fake_neutron_port) + self.assertEqual(response, fake_unbind_response) + mock_port_unbind.assert_called_with(fake_endpoint_id, + fake_neutron_port, pf_ifname=fake_pf_ifname, + vf_num='3', driver='kuryr.lib.binding.drivers.hw_veb') + mock_listdir.assert_called_with( + '/sys/bus/pci/devices/0000:0a:00.1/physfn/net') + mock_iglob.assert_called_with( + '/sys/bus/pci/devices/0000:0a:00.1/physfn/virtfn*') + mock_readlink.assert_called_with( + '/sys/bus/pci/devices/0000:0a:00.1/physfn/virtfn3') + + @mock.patch.object(os, 'listdir') + def test_get_container_iface_name(self, mock_listdir): + sriov_driver = sriov.SriovDriver() + fake_endpoint_id = lib_utils.get_hash() + fake_neutron_port_id = uuidutils.generate_uuid() + fake_neutron_port = self._get_fake_port( + fake_endpoint_id, uuidutils.generate_uuid(), + fake_neutron_port_id, + binding_profile={'pci_slot': '0000:0a:00.1'})['port'] + fake_vf_ifname = 'vf01' + mock_listdir.return_value = [fake_vf_ifname] + response = sriov_driver.get_container_iface_name(fake_neutron_port) + self.assertEqual(response, fake_vf_ifname) + mock_listdir.assert_called_with( + '/sys/bus/pci/devices/0000:0a:00.1/net') diff --git a/kuryr_libnetwork/tests/unit/port_driver/drivers/test_veth.py b/kuryr_libnetwork/tests/unit/port_driver/drivers/test_veth.py index f2ffd160..30f809e8 100644 --- a/kuryr_libnetwork/tests/unit/port_driver/drivers/test_veth.py +++ b/kuryr_libnetwork/tests/unit/port_driver/drivers/test_veth.py @@ -77,7 +77,10 @@ class TestVethDriver(base.TestKuryrBase): def test_get_container_iface_name(self, mock_get_veth_pair_names): veth_driver = veth.VethDriver() fake_neutron_port_id = uuidutils.generate_uuid() - response = veth_driver.get_container_iface_name(fake_neutron_port_id) + fake_neutron_port = self._get_fake_port( + uuidutils.generate_uuid(), uuidutils.generate_uuid(), + fake_neutron_port_id)['port'] + response = veth_driver.get_container_iface_name(fake_neutron_port) mock_get_veth_pair_names.assert_called_with(fake_neutron_port_id) self.assertEqual(response, "fake_container_ifname") diff --git a/kuryr_libnetwork/tests/unit/port_driver/drivers/test_vlan.py b/kuryr_libnetwork/tests/unit/port_driver/drivers/test_vlan.py index a088b824..dcca2787 100644 --- a/kuryr_libnetwork/tests/unit/port_driver/drivers/test_vlan.py +++ b/kuryr_libnetwork/tests/unit/port_driver/drivers/test_vlan.py @@ -177,7 +177,10 @@ class TestVlanDriver(base.TestKuryrBase): mock_vlan_check.return_value = None vlan_driver = vlan.VlanDriver() fake_neutron_port_id = uuidutils.generate_uuid() - response = vlan_driver.get_container_iface_name(fake_neutron_port_id) + fake_neutron_port = self._get_fake_port( + uuidutils.generate_uuid(), uuidutils.generate_uuid(), + fake_neutron_port_id)['port'] + response = vlan_driver.get_container_iface_name(fake_neutron_port) mock_get_pair_names.assert_called_with(fake_neutron_port_id) self.assertEqual(response, "fake_container_name") diff --git a/kuryr_libnetwork/tests/unit/port_driver/test_base.py b/kuryr_libnetwork/tests/unit/port_driver/test_base.py index 90ca7d2d..07c0cfec 100644 --- a/kuryr_libnetwork/tests/unit/port_driver/test_base.py +++ b/kuryr_libnetwork/tests/unit/port_driver/test_base.py @@ -29,7 +29,7 @@ class TestBaseDriver(d_base.BaseNestedDriver): def delete_host_iface(self, endpoint_id, neutron_port): pass - def get_container_iface_name(self, neutron_port_id): + def get_container_iface_name(self, neutron_port): pass def get_supported_bindings(self): diff --git a/kuryr_libnetwork/tests/unit/port_driver/test_driver.py b/kuryr_libnetwork/tests/unit/port_driver/test_driver.py index bca87bd9..79905ead 100644 --- a/kuryr_libnetwork/tests/unit/port_driver/test_driver.py +++ b/kuryr_libnetwork/tests/unit/port_driver/test_driver.py @@ -46,7 +46,7 @@ class TestDriver(base.TestKuryrBase): @mock.patch('kuryr_libnetwork.config.CONF') @ddt.data('kuryr_libnetwork.port_driver.drivers.veth', 'veth') def test__parse_port_driver_config(self, port_driver_value, mock_conf): - mock_conf.port_driver = port_driver_value + mock_conf.default_port_driver = port_driver_value module, name, classname = driver._parse_port_driver_config() self.assertEqual(module, 'kuryr_libnetwork.port_driver.drivers.veth') @@ -60,7 +60,7 @@ class TestDriver(base.TestKuryrBase): @mock.patch('kuryr_libnetwork.config.CONF') def test__verify_binding_driver_compatibility(self, mock_conf): - mock_conf.binding.default_driver = 'veth' + mock_conf.binding.enabled_drivers = ['veth'] fake_driver = mock.Mock(spec=driver.Driver) fake_driver.get_supported_bindings.return_value = ('veth',) @@ -68,13 +68,24 @@ class TestDriver(base.TestKuryrBase): fake_driver.get_supported_bindings.assert_called_once() self.assertIsNone(ret) + @mock.patch('kuryr_libnetwork.config.CONF') + def test__verify_binding_driver_compatibility_multi_drivers( + self, mock_conf): + mock_conf.binding.enabled_drivers = ['veth', 'sriov'] + fake_driver = mock.Mock(spec=driver.Driver) + fake_driver.get_supported_bindings.return_value = ('sriov',) + + ret = driver._verify_binding_driver_compatibility(fake_driver, 'sriov') + fake_driver.get_supported_bindings.assert_called_once() + self.assertIsNone(ret) + class TestNestedDriverFailures(base.TestKuryrFailures): """Unit tests for driver loading failures""" @mock.patch('kuryr_libnetwork.config.CONF') def test__parse_port_driver_config_empty(self, mock_conf): - mock_conf.port_driver = '' + mock_conf.default_port_driver = '' self.assertRaisesRegex(exceptions.KuryrException, "No port driver provided", driver._parse_port_driver_config) @@ -93,9 +104,21 @@ class TestNestedDriverFailures(base.TestKuryrFailures): @mock.patch('kuryr_libnetwork.config.CONF') def test__verify_binding_driver_compatibility_not_compatible(self, m_conf): - m_conf.binding.default_driver = 'macvlan' + m_conf.binding.enabled_drivers = ['macvlan'] message = "Configuration file error: port driver 'veth' is not " \ - "compatible with binding driver 'macvlan'" + "compatible with binding driver '\['macvlan'\]'" + + fake_driver = mock.Mock(spec=driver.Driver) + fake_driver.get_supported_bindings.return_value = ('veth',) + self.assertRaisesRegex(exceptions.KuryrException, message, + driver._verify_binding_driver_compatibility, fake_driver, 'veth') + + @mock.patch('kuryr_libnetwork.config.CONF') + def test__verify_binding_driver_compatibility_not_compatible_multi_drivers( + self, m_conf): + m_conf.binding.enabled_drivers = ['macvlan', 'sriov'] + message = "Configuration file error: port driver 'veth' is not " \ + "compatible with binding driver '\['macvlan'\, 'sriov']'" fake_driver = mock.Mock(spec=driver.Driver) fake_driver.get_supported_bindings.return_value = ('veth',) @@ -104,8 +127,8 @@ class TestNestedDriverFailures(base.TestKuryrFailures): @mock.patch('kuryr_libnetwork.config.CONF') def test__verify_binding_driver_compatibility_not_supported(self, m_conf): - m_conf.binding.default_driver = 'ipvlan' - message = "Configuration file error: binding driver 'ipvlan' is " \ + m_conf.binding.enabled_drivers = ['ipvlan'] + message = "Configuration file error: binding driver '\['ipvlan'\]' is " \ "currently not supported with 'nested' port driver" fake_driver = mock.Mock(spec=driver.Driver) diff --git a/kuryr_libnetwork/tests/unit/test_kuryr.py b/kuryr_libnetwork/tests/unit/test_kuryr.py index 3dc98ead..e46c8ba8 100644 --- a/kuryr_libnetwork/tests/unit/test_kuryr.py +++ b/kuryr_libnetwork/tests/unit/test_kuryr.py @@ -1462,10 +1462,11 @@ class TestKuryr(base.TestKuryrBase): decoded_json = jsonutils.loads(response.data) self.assertEqual(constants.SCHEMA['SUCCESS'], decoded_json) - @mock.patch('kuryr_libnetwork.controllers.app.driver.create_host_iface') + @mock.patch('kuryr_libnetwork.controllers.DEFAULT_DRIVER' + '.create_host_iface') @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_networks') @mock.patch('kuryr_libnetwork.controllers.app.neutron.show_port') - @mock.patch('kuryr_libnetwork.controllers.app.driver.update_port') + @mock.patch('kuryr_libnetwork.controllers.DEFAULT_DRIVER.update_port') @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_ports') @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_subnets') @mock.patch('kuryr_libnetwork.controllers.app') @@ -1570,7 +1571,8 @@ class TestKuryr(base.TestKuryrBase): expected = {'Interface': {}} self.assertEqual(expected, decoded_json) - @mock.patch('kuryr_libnetwork.controllers.app.driver.create_host_iface') + @mock.patch('kuryr_libnetwork.controllers.DEFAULT_DRIVER' + '.create_host_iface') @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_networks') @mock.patch('kuryr_libnetwork.controllers.app.neutron.show_port') @mock.patch('kuryr_libnetwork.controllers.app.neutron.create_port') @@ -1721,10 +1723,11 @@ class TestKuryr(base.TestKuryrBase): expected = {'Interface': {}} self.assertEqual(expected, decoded_json) - @mock.patch('kuryr_libnetwork.controllers.app.driver.create_host_iface') + @mock.patch('kuryr_libnetwork.controllers.DEFAULT_DRIVER' + '.create_host_iface') @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_networks') @mock.patch('kuryr_libnetwork.controllers.app.neutron.show_port') - @mock.patch('kuryr_libnetwork.controllers.app.driver.update_port') + @mock.patch('kuryr_libnetwork.controllers.DEFAULT_DRIVER.update_port') @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_ports') @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_subnets') @mock.patch('kuryr_libnetwork.controllers.app') @@ -1905,7 +1908,8 @@ class TestKuryr(base.TestKuryrBase): self.assertEqual(fake_port_response['ports'][0]['status'], decoded_json['Value']['status']) - @mock.patch('kuryr_libnetwork.controllers.app.driver.delete_host_iface') + @mock.patch('kuryr_libnetwork.controllers.DEFAULT_DRIVER' + '.delete_host_iface') @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_ports') @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_networks') def test_network_driver_delete_endpoint(self, mock_list_networks, @@ -1954,7 +1958,7 @@ class TestKuryr(base.TestKuryrBase): @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_ports') @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_subnets') @mock.patch( - 'kuryr_libnetwork.controllers.app.driver.get_container_iface_name') + 'kuryr_libnetwork.controllers.DEFAULT_DRIVER.get_container_iface_name') def test_network_driver_join(self, mock_get_container_iface_name, mock_list_subnets, mock_list_ports, mock_list_networks, mock_get_veth_pair_names): @@ -2026,7 +2030,8 @@ class TestKuryr(base.TestKuryrBase): decoded_json = jsonutils.loads(response.data) mock_list_networks.assert_any_call(tags=t) - mock_get_container_iface_name.assert_called_with(fake_neutron_port_id) + mock_get_container_iface_name.assert_called_with( + fake_neutron_ports_response['ports'][0]) mock_list_ports.assert_called_with(name=neutron_port_name) mock_list_subnets.assert_called_with(network_id=fake_neutron_net_id) @@ -2037,7 +2042,7 @@ class TestKuryr(base.TestKuryrBase): @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_ports') @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_subnets') @mock.patch( - 'kuryr_libnetwork.controllers.app.driver.get_container_iface_name') + 'kuryr_libnetwork.controllers.DEFAULT_DRIVER.get_container_iface_name') def test_network_driver_join_multiple_subnets( self, mock_get_container_iface_name, mock_list_subnets, mock_list_ports, mock_list_networks, @@ -2143,7 +2148,8 @@ class TestKuryr(base.TestKuryrBase): decoded_json = jsonutils.loads(response.data) mock_list_networks.assert_any_call(tags=t) - mock_get_container_iface_name.assert_called_with(fake_neutron_port_id) + mock_get_container_iface_name.assert_called_with( + fake_neutron_ports_response['ports'][0]) mock_list_ports.assert_called_with(name=neutron_port_name) mock_list_subnets.assert_called_with(network_id=fake_neutron_net_id) @@ -2154,7 +2160,7 @@ class TestKuryr(base.TestKuryrBase): @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_ports') @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_subnets') @mock.patch( - 'kuryr_libnetwork.controllers.app.driver.get_container_iface_name') + 'kuryr_libnetwork.controllers.DEFAULT_DRIVER.get_container_iface_name') @mock.patch('kuryr_libnetwork.controllers.app') @ddt.data(True, False) def test_network_driver_join_with_static_route_return(self, @@ -2254,7 +2260,8 @@ class TestKuryr(base.TestKuryrBase): else: mock_list_networks.assert_any_call(name=fake_docker_net_id) - mock_get_container_iface_name.assert_called_with(fake_neutron_port_id) + mock_get_container_iface_name.assert_called_with( + fake_neutron_ports_response['ports'][0]) neutron_port_name = utils.get_neutron_port_name( fake_docker_endpoint_id) mock_list_ports.assert_called_with(name=neutron_port_name) diff --git a/kuryr_libnetwork/tests/unit/test_kuryr_endpoint.py b/kuryr_libnetwork/tests/unit/test_kuryr_endpoint.py index d9c7fbab..2bca3f57 100644 --- a/kuryr_libnetwork/tests/unit/test_kuryr_endpoint.py +++ b/kuryr_libnetwork/tests/unit/test_kuryr_endpoint.py @@ -121,8 +121,9 @@ class TestKuryrEndpointCreateFailures(base.TestKuryrFailures): self.assertIn('Err', decoded_json) self.assertEqual({'Err': GivenException.message}, decoded_json) - @mock.patch('kuryr_libnetwork.controllers.app.driver.create_host_iface') - @mock.patch('kuryr_libnetwork.controllers.app.driver.update_port') + @mock.patch('kuryr_libnetwork.controllers.DEFAULT_DRIVER' + '.create_host_iface') + @mock.patch('kuryr_libnetwork.controllers.DEFAULT_DRIVER.update_port') @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_subnets') @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_ports') @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_networks') @@ -249,7 +250,8 @@ class TestKuryrEndpointDeleteFailures(base.TestKuryrFailures): data=jsonutils.dumps(data)) return response - @mock.patch('kuryr_libnetwork.controllers.app.driver.delete_host_iface') + @mock.patch('kuryr_libnetwork.controllers.DEFAULT_DRIVER' + '.delete_host_iface') @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_ports') @mock.patch('kuryr_libnetwork.controllers.app.neutron.list_networks') @ddt.data(k_exceptions.VethDeletionFailure,