Add MACVLAN based interfaces for nested containers

Currently nested containers can only be run by using trunk support and
vlan based interfaces. This patch introduces the additional option of
MACVLAN slave interfaces for pods running in VMs.

This patch includes both a new VIF driver on the controller side and the
binding driver for the CNI plugin.

Implements: blueprint macvlan-pod-in-vm
Depends-On: Ib71204d2d14d3d4f15beada701094e37d89d7801
Co-Authored-By: Marco Chiappero <marco.chiappero@intel.com>
Change-Id: I03c536bb0057bba0a5eb4d1c135baa8ab625e400

README.rst

@@ -57,8 +57,8 @@ vif binding executables. For example, if you installed it on Debian or Ubuntu::
     bindir = /usr/local/libexec/kuryr
 
 
-How to try out nested-pods locally
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+How to try out nested-pods locally (VLAN + trunk)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Following are the instructions for an all-in-one setup where K8s will also be
 running inside the same Nova VM in which Kuryr-controller and Kuryr-cni will be
@@ -110,6 +110,52 @@ running. 4GB memory and 2 vCPUs, is the minimum resource requirement for the VM:
 
 Now launch pods using kubectl, Undercloud Neutron will serve the networking.
 
+How to try out nested-pods locally (MACVLAN)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Following are the instructions for an all-in-one setup, as above, but using the
+nested MACVLAN driver rather than VLAN and trunk ports.
+
+1. To install OpenStack services run devstack with ``devstack/local.conf.pod-in-vm.undercloud.sample``.
+2. Launch a Nova VM with MACVLAN support
+3. Log into the VM and set up Kubernetes along with Kuryr using devstack:
+    - Since undercloud Neutron will be used by pods, Neutron services should be
+      disabled in localrc.
+    - Run devstack with ``devstack/local.conf.pod-in-vm.overcloud.sample``.
+      With this config devstack will not configure Neutron resources for the
+      local cloud. These variables have to be added manually
+      to ``/etc/kuryr/kuryr.conf``.
+
+4. Once devstack is done and all services are up inside VM:
+    - Configure ``/etc/kuryr/kuryr.conf`` with the following content, replacing
+      the values with correct UUIDs of Neutron resources from the undercloud::
+
+       [neutron_defaults]
+       pod_security_groups = <UNDERCLOUD_DEFAULT_SG_UUID>
+       pod_subnet = <UNDERCLOUD_SUBNET_FOR_PODS_UUID>
+       project = <UNDERCLOUD_DEFAULT_PROJECT_UUID>
+       service_subnet = <UNDERCLOUD_SUBNET_FOR_SERVICES_UUID>
+
+    - Configure worker VMs subnet::
+
+       [pod_vif_nested]
+       worker_nodes_subnet = <UNDERCLOUD_SUBNET_WORKER_NODES_UUID>
+
+    - Configure “pod_vif_driver” as “nested-macvlan”::
+
+       [kubernetes]
+       pod_vif_driver = nested-macvlan
+
+    - Configure binding section::
+
+       [binding]
+       link_iface = <VM interface name eg. eth0>
+
+    - Restart kuryr-k8s-controller::
+
+       sudo systemctl restart devstack@kuryr-kubernetes.service
+
+Now launch pods using kubectl, Undercloud Neutron will serve the networking.
 
 How to watch K8S api-server over HTTPS
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

kuryr_kubernetes/cni/binding/nested.py

@@ -12,13 +12,24 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-# from kuryr.lib import constants
-# from kuryr.lib import utils
+import abc
+import six
+
 from kuryr_kubernetes.cni.binding import base as b_base
 from kuryr_kubernetes import config
 
+VLAN_KIND = 'vlan'
+MACVLAN_KIND = 'macvlan'
+MACVLAN_MODE_BRIDGE = 'bridge'
+
+
+@six.add_metaclass(abc.ABCMeta)
+class NestedDriver(object):
+
+    @abc.abstractmethod
+    def _get_iface_create_args(self, vif):
+        raise NotImplementedError()
 
-class VlanDriver(object):
     def connect(self, vif, ifname, netns):
         h_ipdb = b_base.get_ipdb()
         c_ipdb = b_base.get_ipdb(netns)
@@ -33,11 +44,11 @@ class VlanDriver(object):
         # TODO(vikasc): evaluate whether we should have stevedore
         #               driver for getting the link device.
         vm_iface_name = config.CONF.binding.link_iface
-        vlan_id = vif.vlan_id
 
+        args = self._get_iface_create_args(vif)
         with h_ipdb.create(ifname=temp_name,
                            link=h_ipdb.interfaces[vm_iface_name],
-                           kind='vlan', vlan_id=vlan_id) as iface:
+                           **args) as iface:
             iface.net_ns_fd = netns
 
         with c_ipdb.interfaces[temp_name] as iface:
@@ -50,3 +61,13 @@ class VlanDriver(object):
         # NOTE(vikasc): device will get deleted with container namespace, so
         # nothing to be done here.
         pass
+
+
+class VlanDriver(NestedDriver):
+    def _get_iface_create_args(self, vif):
+        return {'kind': VLAN_KIND, 'vlan_id': vif.vlan_id}
+
+
+class MacvlanDriver(NestedDriver):
+    def _get_iface_create_args(self, vif):
+        return {'kind': MACVLAN_KIND, 'macvlan_mode': MACVLAN_MODE_BRIDGE}

kuryr_kubernetes/controller/drivers/nested_macvlan_vif.py

@@ -0,0 +1,145 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import threading
+
+from neutronclient.common import exceptions as n_exc
+from oslo_log import log as logging
+
+from kuryr_kubernetes import clients
+from kuryr_kubernetes.controller.drivers import nested_vif
+from kuryr_kubernetes import exceptions as k_exc
+from kuryr_kubernetes import os_vif_util as ovu
+
+LOG = logging.getLogger(__name__)
+
+
+class NestedMacvlanPodVIFDriver(nested_vif.NestedPodVIFDriver):
+    """Manages ports for nested-containers using MACVLAN to provide VIFs."""
+
+    def __init__(self):
+        self.lock = threading.Lock()
+
+    def request_vif(self, pod, project_id, subnets, security_groups):
+        neutron = clients.get_neutron_client()
+        req = self._get_port_request(pod, project_id, subnets,
+                                     security_groups)
+        container_port = neutron.create_port(req).get('port')
+
+        container_mac = container_port['mac_address']
+        container_ips = frozenset(entry['ip_address'] for entry in
+                                  container_port['fixed_ips'])
+
+        with self.lock:
+            self.lock.acquire()
+            vm_port = self._get_parent_port(neutron, pod)
+            self._add_to_allowed_address_pairs(neutron, vm_port,
+                container_ips, container_mac)
+
+        return ovu.neutron_to_osvif_vif_nested_macvlan(container_port, subnets)
+
+    def request_vifs(self, pod, project_id, subnets, security_groups,
+                     num_ports):
+        # TODO(mchiappe): provide an implementation
+        raise NotImplementedError()
+
+    def release_vif(self, pod, vif):
+        neutron = clients.get_neutron_client()
+        container_port = neutron.show_port(vif.id).get('port')
+
+        container_mac = container_port['mac_address']
+        container_ips = frozenset(entry['ip_address'] for entry in
+                                  container_port['fixed_ips'])
+
+        with self.lock:
+            self.lock.acquire()
+            vm_port = self._get_parent_port(neutron, pod)
+            self._remove_from_allowed_address_pairs(neutron, vm_port,
+                container_ips, container_mac)
+
+        try:
+            neutron.delete_port(vif.id)
+        except n_exc.PortNotFoundClient:
+            LOG.warning("Unable to release port %s as it no longer exists.",
+                        vif.id)
+
+    def activate_vif(self, pod, vif):
+        # NOTE(mchiappe): there is no way to get feedback on the actual
+        # interface creation or activation as no plugging can happen for this
+        # interface type. However the status of the port is not relevant as
+        # it is used for IPAM purposes only, thus just set 'active'
+        # immediately to let the CNI driver make progress.
+        vif.active = True
+
+    def _add_to_allowed_address_pairs(self, neutron, port, ip_addresses,
+                                      mac_address=None):
+        if not ip_addresses:
+            raise k_exc.IntegrityError("Cannot add pair from the "
+                "allowed_address_pairs of port %s: missing IP address",
+                port['id'])
+
+        mac = mac_address if mac_address else port['mac_address']
+        address_pairs = port['allowed_address_pairs']
+
+        # look for duplicates or near-matches
+        for pair in address_pairs:
+            if pair['ip_address'] in ip_addresses:
+                if pair['mac_address'] is mac:
+                    raise k_exc.AllowedAddressAlreadyPresent("Pair %s already "
+                        "present in the 'allowed_address_pair' list. This is "
+                        "due to a misconfiguration or a bug", pair)
+                else:
+                    LOG.warning("A pair with IP %s but different MAC address "
+                        "is already present in the 'allowed_address_pair'. "
+                        "This could indicate a misconfiguration or a "
+                        "bug", pair['ip_address'])
+
+        for ip in ip_addresses:
+            address_pairs.append({'ip_address': ip, 'mac_address': mac})
+
+        self._update_port_address_pairs(neutron, port['id'], address_pairs)
+
+    def _remove_from_allowed_address_pairs(self, neutron, port, ip_addresses,
+                                           mac_address=None):
+        if not ip_addresses:
+            raise k_exc.IntegrityError("Cannot remove pair from the "
+                "allowed_address_pairs of port %s: missing IP address",
+                port['id'])
+
+        mac = mac_address if mac_address else port['mac_address']
+        address_pairs = port['allowed_address_pairs']
+        updated = False
+
+        for ip in ip_addresses:
+            try:
+                address_pairs.remove({'ip_address': ip, 'mac_address': mac})
+                updated = True
+            except ValueError:
+                LOG.error("No {'ip_address': %s, 'mac_address': %s} pair "
+                          "found in the 'allowed_address_pair' list while "
+                          "trying to remove it.", ip, mac)
+
+        if updated:
+            self._update_port_address_pairs(neutron, port['id'], address_pairs)
+
+    def _update_port_address_pairs(self, neutron, port_id, address_pairs):
+        try:
+            neutron.update_port(
+                port_id,
+                {'port': {'allowed_address_pairs': address_pairs}}
+            )
+        except n_exc.NeutronClientException as ex:
+            LOG.error("Error happened during updating Neutron "
+                      "port %s: %s", port_id, ex)
+            raise ex

kuryr_kubernetes/controller/drivers/nested_vlan_vif.py

@@ -20,7 +20,6 @@ from neutronclient.common import exceptions as n_exc
 from oslo_log import log as logging
 
 from kuryr_kubernetes import clients
-from kuryr_kubernetes import constants as const
 from kuryr_kubernetes.controller.drivers import nested_vif
 from kuryr_kubernetes import exceptions as k_exc
 from kuryr_kubernetes import os_vif_util as ovu
@@ -35,8 +34,6 @@ DEFAULT_RETRY_INTERVAL = 1
 class NestedVlanPodVIFDriver(nested_vif.NestedPodVIFDriver):
     """Manages ports for nested-containers using VLANs to provide VIFs."""
 
-    _vif_plugin = const.K8S_OS_VIF_NOOP_PLUGIN
-
     def request_vif(self, pod, project_id, subnets, security_groups):
         neutron = clients.get_neutron_client()
         parent_port = self._get_parent_port(neutron, pod)
@@ -44,12 +41,9 @@ class NestedVlanPodVIFDriver(nested_vif.NestedPodVIFDriver):
 
         rq = self._get_port_request(pod, project_id, subnets, security_groups)
         port = neutron.create_port(rq).get('port')
-
         vlan_id = self._add_subport(neutron, trunk_id, port['id'])
 
-        vif = ovu.neutron_to_osvif_vif(self._vif_plugin, port, subnets)
-        vif.vlan_id = vlan_id
-        return vif
+        return ovu.neutron_to_osvif_vif_nested_vlan(port, subnets, vlan_id)
 
     def request_vifs(self, pod, project_id, subnets, security_groups,
                      num_ports):
@@ -101,8 +95,8 @@ class NestedVlanPodVIFDriver(nested_vif.NestedPodVIFDriver):
 
         vifs = []
         for index, port in enumerate(ports):
-            vif = ovu.neutron_to_osvif_vif(self._vif_plugin, port, subnets)
-            vif.vlan_id = subports_info[index]['segmentation_id']
+            vlan_id = subports_info[index]['segmentation_id']
+            vif = ovu.neutron_to_osvif_vif_nested_vlan(port, subnets, vlan_id)
             vifs.append(vif)
         return vifs
 

kuryr_kubernetes/exceptions.py

@@ -42,3 +42,12 @@ class K8sNodeTrunkPortFailure(Exception):
     This exception is thrown when Neutron port is not associated to a Neutron
     vlan trunk.
     """
+
+
+class AllowedAddressAlreadyPresent(Exception):
+    """Exception indicates an already present 'allowed address pair' on port
+
+    This exception is raised when an attempt to add an already inserted
+    'allowed address pair' on a port is made. Such a condition likely indicates
+    a bad program state or a programming bug.
+    """

kuryr_kubernetes/objects/vif.py

@@ -28,3 +28,15 @@ class VIFVlanNested(obj_osvif.VIFBase):
         # vlan ID allocated to this vif
         'vlan_id': obj_fields.IntegerField()
     }
+
+
+@obj_base.VersionedObjectRegistry.register
+class VIFMacvlanNested(obj_osvif.VIFBase):
+    # This is OVO based macvlan vif.
+
+    VERSION = '1.0'
+
+    fields = {
+        # Name of the device to create
+        'vif_name': obj_fields.StringField(),
+    }

kuryr_kubernetes/os_vif_plug_noop.py

@@ -27,6 +27,10 @@ class NoOpPlugin(PluginBase):
                     vif_object_name=k_vif.VIFVlanNested.__name__,
                     min_version="1.0",
                     max_version="1.0"),
+                objects.host_info.HostVIFInfo(
+                    vif_object_name=k_vif.VIFMacvlanNested.__name__,
+                    min_version="1.0",
+                    max_version="1.0"),
             ])
 
     def plug(self, vif, instance_info):

kuryr_kubernetes/os_vif_util.py

@@ -27,6 +27,7 @@ from oslo_config import cfg as oslo_cfg
 from stevedore import driver as stv_driver
 
 from kuryr_kubernetes import config
+from kuryr_kubernetes import constants as const
 from kuryr_kubernetes import exceptions as k_exc
 from kuryr_kubernetes.objects import vif as k_vif
 
@@ -251,36 +252,54 @@ def neutron_to_osvif_vif_ovs(vif_plugin, neutron_port, subnets):
     return vif
 
 
-def neutron_to_osvif_vif_nested(vif_plugin, neutron_port, subnets):
-    """Converts Neutron port to VIF object for nested containers.
+def neutron_to_osvif_vif_nested_vlan(neutron_port, subnets, vlan_id):
+    """Converts Neutron port to VIF object for VLAN nested containers.
 
-    :param vif_plugin: name of the os-vif plugin to use (i.e. 'noop')
     :param neutron_port: dict containing port information as returned by
                          neutron client's 'show_port'
     :param subnets: subnet mapping as returned by PodSubnetsDriver.get_subnets
-    :return: kuryr-k8s native VIF object (eg. VIFVlanNested)
+    :param vlan_id: VLAN id associated to the VIF object for the pod
+    :return: kuryr-k8s native VIF object for VLAN nested
     """
-
     details = neutron_port.get('binding:vif_details', {})
-    network = _make_vif_network(neutron_port, subnets)
 
-    vif = k_vif.VIFVlanNested(
+    return k_vif.VIFVlanNested(
         id=neutron_port['id'],
         address=neutron_port['mac_address'],
-        network=network,
+        network=_make_vif_network(neutron_port, subnets),
         has_traffic_filtering=details.get('port_filter', False),
         preserve_on_delete=False,
         active=_is_port_active(neutron_port),
-        plugin=vif_plugin,
+        plugin=const.K8S_OS_VIF_NOOP_PLUGIN,
+        vif_name=_get_vif_name(neutron_port),
+        vlan_id=vlan_id)
+
+
+def neutron_to_osvif_vif_nested_macvlan(neutron_port, subnets):
+    """Converts Neutron port to VIF object for MACVLAN nested containers.
+
+    :param neutron_port: dict containing port information as returned by
+                         neutron client's 'show_port'
+    :param subnets: subnet mapping as returned by PodSubnetsDriver.get_subnets
+    :return: kuryr-k8s native VIF object for MACVLAN nested
+    """
+    details = neutron_port.get('binding:vif_details', {})
+
+    return k_vif.VIFMacvlanNested(
+        id=neutron_port['id'],
+        address=neutron_port['mac_address'],
+        network=_make_vif_network(neutron_port, subnets),
+        has_traffic_filtering=details.get('port_filter', False),
+        preserve_on_delete=False,
+        active=_is_port_active(neutron_port),
+        plugin=const.K8S_OS_VIF_NOOP_PLUGIN,
         vif_name=_get_vif_name(neutron_port))
 
-    return vif
 
-
-def neutron_to_osvif_vif(vif_plugin, neutron_port, subnets):
+def neutron_to_osvif_vif(vif_translator, neutron_port, subnets):
     """Converts Neutron port to os-vif VIF object.
 
-    :param vif_plugin: name of the os-vif plugin to use
+    :param vif_translator: name of the traslator for the os-vif plugin to use
     :param neutron_port: dict containing port information as returned by
                          neutron client
     :param subnets: subnet mapping as returned by PodSubnetsDriver.get_subnets
@@ -288,13 +307,13 @@ def neutron_to_osvif_vif(vif_plugin, neutron_port, subnets):
     """
 
     try:
-        mgr = _VIF_MANAGERS[vif_plugin]
+        mgr = _VIF_MANAGERS[vif_translator]
     except KeyError:
         mgr = stv_driver.DriverManager(namespace=_VIF_TRANSLATOR_NAMESPACE,
-                                       name=vif_plugin, invoke_on_load=False)
-        _VIF_MANAGERS[vif_plugin] = mgr
+                                    name=vif_translator, invoke_on_load=False)
+        _VIF_MANAGERS[vif_translator] = mgr
 
-    return mgr.driver(vif_plugin, neutron_port, subnets)
+    return mgr.driver(vif_translator, neutron_port, subnets)
 
 
 def osvif_to_neutron_fixed_ips(subnets):

kuryr_kubernetes/tests/unit/controller/drivers/test_nested_macvlan_vif.py

@@ -0,0 +1,479 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import ddt
+import mock
+import threading
+
+from kuryr.lib import utils as lib_utils
+from neutronclient.common import exceptions as n_exc
+
+from kuryr_kubernetes.controller.drivers import nested_macvlan_vif
+from kuryr_kubernetes import exceptions as k_exc
+from kuryr_kubernetes.tests import base as test_base
+from kuryr_kubernetes.tests.unit import kuryr_fixtures as k_fix
+
+
+@ddt.ddt
+class TestNestedMacvlanPodVIFDriver(test_base.TestCase):
+
+    @mock.patch(
+        'kuryr_kubernetes.os_vif_util.neutron_to_osvif_vif_nested_macvlan')
+    def test_request_vif(self, m_to_vif):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+
+        pod = mock.sentinel.pod
+        project_id = mock.sentinel.project_id
+        subnets = mock.sentinel.subnets
+        security_groups = mock.sentinel.security_groups
+        container_mac = mock.sentinel.mac_address
+        container_ip = mock.sentinel.ip_address
+        container_port = self._get_fake_port(mac_address=container_mac,
+                                       ip_address=container_ip)
+
+        vif = mock.Mock()
+        port_request = mock.sentinel.port_request
+        vm_port = self._get_fake_port()
+
+        m_to_vif.return_value = vif
+        m_driver._get_port_request.return_value = port_request
+        m_driver._get_parent_port.return_value = vm_port
+        m_driver.lock = mock.MagicMock(spec=threading.Lock())
+        neutron.create_port.return_value = container_port
+
+        self.assertEqual(vif, cls.request_vif(m_driver, pod, project_id,
+                                              subnets, security_groups))
+
+        m_driver._get_port_request.assert_called_once_with(pod, project_id,
+                                                    subnets, security_groups)
+        neutron.create_port.assert_called_once_with(port_request)
+        m_driver._get_parent_port.assert_called_once_with(neutron, pod)
+        m_driver._add_to_allowed_address_pairs.assert_called_once_with(
+            neutron, vm_port, frozenset([container_ip]), container_mac)
+        m_to_vif.assert_called_once_with(container_port['port'], subnets)
+
+    @mock.patch(
+        'kuryr_kubernetes.os_vif_util.neutron_to_osvif_vif_nested_macvlan')
+    def test_request_vif_port_create_failed(self, m_to_vif):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+
+        pod = mock.sentinel.pod
+        project_id = mock.sentinel.project_id
+        subnets = mock.sentinel.subnets
+        security_groups = mock.sentinel.security_groups
+
+        port_request = mock.sentinel.port_request
+        m_driver._get_port_request.return_value = port_request
+        neutron.create_port.side_effect = n_exc.NeutronClientException
+
+        self.assertRaises(n_exc.NeutronClientException, cls.request_vif,
+                          m_driver, pod, project_id, subnets, security_groups)
+        m_driver._get_port_request.assert_called_once_with(pod, project_id,
+                                                      subnets, security_groups)
+        neutron.create_port.assert_called_once_with(port_request)
+        m_driver._add_to_allowed_address_pairs.assert_not_called()
+        m_to_vif.assert_not_called()
+
+    @mock.patch(
+        'kuryr_kubernetes.os_vif_util.neutron_to_osvif_vif_nested_macvlan')
+    def test_request_vif_parent_not_found(self, m_to_vif):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+
+        pod = mock.sentinel.pod
+        project_id = mock.sentinel.project_id
+        subnets = mock.sentinel.subnets
+        security_groups = mock.sentinel.security_groups
+        container_mac = mock.sentinel.mac_address
+        container_ip = mock.sentinel.ip_address
+        container_port = self._get_fake_port(mac_address=container_mac,
+                                       ip_address=container_ip)
+
+        port_request = mock.sentinel.port_request
+        m_driver._get_port_request.return_value = port_request
+        m_driver.lock = mock.MagicMock(spec=threading.Lock())
+        neutron.create_port.return_value = container_port
+        m_driver._get_parent_port.side_effect = n_exc.NeutronClientException
+
+        self.assertRaises(n_exc.NeutronClientException, cls.request_vif,
+                          m_driver, pod, project_id, subnets, security_groups)
+        m_driver._get_port_request.assert_called_once_with(pod, project_id,
+                                                      subnets, security_groups)
+        neutron.create_port.assert_called_once_with(port_request)
+        m_driver._get_parent_port.assert_called_once_with(neutron, pod)
+        m_driver._add_to_allowed_address_pairs.assert_not_called()
+        m_to_vif.assert_not_called()
+
+    def test_release_vif(self):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+
+        port_id = lib_utils.get_hash()
+        pod = mock.sentinel.pod
+        vif = mock.Mock()
+        vif.id = port_id
+
+        container_mac = mock.sentinel.mac_address
+        container_ip = mock.sentinel.ip_address
+        container_port = self._get_fake_port(port_id, container_ip,
+                                             container_mac)
+        neutron.show_port.return_value = container_port
+
+        vm_port = self._get_fake_port()
+        m_driver._get_parent_port.return_value = vm_port
+        m_driver.lock = mock.MagicMock(spec=threading.Lock())
+
+        cls.release_vif(m_driver, pod, vif)
+
+        neutron.show_port.assert_called_once_with(port_id)
+        m_driver._get_parent_port.assert_called_once_with(neutron, pod)
+        m_driver._remove_from_allowed_address_pairs.assert_called_once_with(
+            neutron, vm_port, frozenset([container_ip]), container_mac)
+        neutron.delete_port.assert_called_once_with(vif.id)
+
+    def test_release_vif_not_found(self):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+
+        pod = mock.sentinel.pod
+        vif = mock.Mock()
+        vif.id = lib_utils.get_hash()
+
+        neutron.show_port.side_effect = n_exc.PortNotFoundClient
+
+        self.assertRaises(n_exc.PortNotFoundClient, cls.release_vif,
+                          m_driver, pod, vif)
+        m_driver._remove_from_allowed_address_pairs.assert_not_called()
+        neutron.delete_port.assert_not_called()
+
+    def test_release_vif_parent_not_found(self):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+
+        port_id = lib_utils.get_hash()
+        pod = mock.sentinel.pod
+        vif = mock.Mock()
+        vif.id = port_id
+
+        container_mac = mock.sentinel.mac_address
+        container_ip = mock.sentinel.ip_address
+        container_port = self._get_fake_port(port_id, container_ip,
+                                             container_mac)
+        neutron.show_port.return_value = container_port
+
+        m_driver.lock = mock.MagicMock(spec=threading.Lock())
+        m_driver._get_parent_port.side_effect = n_exc.NeutronClientException
+
+        self.assertRaises(n_exc.NeutronClientException, cls.release_vif,
+                          m_driver, pod, vif)
+        neutron.show_port.assert_called_once_with(port_id)
+        m_driver._get_parent_port.assert_called_once_with(neutron, pod)
+        m_driver._remove_from_allowed_address_pairs.assert_not_called()
+        neutron.delete_port.assert_not_called()
+
+    def test_release_vif_delete_failed(self):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+
+        port_id = lib_utils.get_hash()
+        pod = mock.sentinel.pod
+        vif = mock.Mock()
+        vif.id = port_id
+
+        container_mac = mock.sentinel.mac_address
+        container_ip = mock.sentinel.ip_addresses
+        container_port = self._get_fake_port(port_id, container_ip,
+                                             container_mac)
+        neutron.show_port.return_value = container_port
+        neutron.delete_port.side_effect = n_exc.PortNotFoundClient
+
+        vm_port = self._get_fake_port()
+        m_driver._get_parent_port.return_value = vm_port
+        m_driver.lock = mock.MagicMock(spec=threading.Lock())
+
+        cls.release_vif(m_driver, pod, vif)
+
+        neutron.show_port.assert_called_once_with(port_id)
+        m_driver._get_parent_port.assert_called_once_with(neutron, pod)
+        m_driver._remove_from_allowed_address_pairs.assert_called_once_with(
+            neutron, vm_port, frozenset([container_ip]), container_mac)
+        neutron.delete_port.assert_called_once_with(vif.id)
+
+    @ddt.data((False), (True))
+    def test_activate_vif(self, active_value):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        pod = mock.sentinel.pod
+        vif = mock.Mock()
+        vif.active = active_value
+
+        cls.activate_vif(m_driver, pod, vif)
+
+        self.assertEqual(vif.active, True)
+
+    @ddt.data((None), ('fa:16:3e:71:cb:80'))
+    def test_add_to_allowed_address_pairs(self, m_mac):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+
+        port_id = lib_utils.get_hash()
+        vm_port = self._get_fake_port(port_id)['port']
+
+        mac_addr = 'fa:16:3e:1b:30:00' if m_mac else vm_port['mac_address']
+        address_pairs = [
+            {'ip_address': '10.0.0.30',
+             'mac_address': mac_addr},
+            {'ip_address': 'fe80::f816:3eff:fe1c:36a9',
+             'mac_address': mac_addr},
+        ]
+        vm_port['allowed_address_pairs'].extend(address_pairs)
+
+        ip_addr = '10.0.0.29'
+        address_pairs.append(
+            {'ip_address': ip_addr,
+             'mac_address': m_mac if m_mac else vm_port['mac_address']}
+        )
+
+        cls._add_to_allowed_address_pairs(m_driver, neutron, vm_port,
+                                          frozenset([ip_addr]), m_mac)
+
+        m_driver._update_port_address_pairs.assert_called_once_with(neutron,
+                                          port_id, address_pairs)
+
+    def test_add_to_allowed_address_pairs_no_ip_addresses(self):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+
+        port_id = lib_utils.get_hash()
+        vm_port = self._get_fake_port(port_id)['port']
+
+        self.assertRaises(k_exc.IntegrityError,
+                          cls._add_to_allowed_address_pairs, m_driver,
+                          neutron, vm_port, frozenset())
+
+    def test_add_to_allowed_address_pairs_same_ip(self):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+
+        port_id = lib_utils.get_hash()
+        vm_port = self._get_fake_port(port_id)['port']
+        address_pairs = [
+            {'ip_address': '10.0.0.30',
+             'mac_address': 'fa:16:3e:1b:30:00'},
+            {'ip_address': 'fe80::f816:3eff:fe1c:36a9',
+             'mac_address': 'fa:16:3e:1b:30:00'},
+        ]
+        vm_port['allowed_address_pairs'].extend(address_pairs)
+
+        mac_addr = 'fa:16:3e:71:cb:80'
+        ip_addr = '10.0.0.30'
+        address_pairs.append({'ip_address': ip_addr, 'mac_address': mac_addr})
+
+        cls._add_to_allowed_address_pairs(m_driver, neutron, vm_port,
+                                          frozenset([ip_addr]), mac_addr)
+
+        m_driver._update_port_address_pairs.assert_called_once_with(neutron,
+                                          port_id, address_pairs)
+
+    def test_add_to_allowed_address_pairs_already_present(self):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+
+        port_id = lib_utils.get_hash()
+        vm_port = self._get_fake_port(port_id)['port']
+        address_pairs = [
+            {'ip_address': '10.0.0.30',
+             'mac_address': 'fa:16:3e:1b:30:00'},
+            {'ip_address': 'fe80::f816:3eff:fe1c:36a9',
+             'mac_address': 'fa:16:3e:1b:30:00'},
+        ]
+        vm_port['allowed_address_pairs'].extend(address_pairs)
+
+        mac_addr = 'fa:16:3e:1b:30:00'
+        ip_addr = '10.0.0.30'
+
+        self.assertRaises(k_exc.AllowedAddressAlreadyPresent,
+                          cls._add_to_allowed_address_pairs, m_driver, neutron,
+                          vm_port, frozenset([ip_addr]), mac_addr)
+
+    @ddt.data((None), ('fa:16:3e:71:cb:80'))
+    def test_remove_from_allowed_address_pairs(self, m_mac):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+
+        port_id = lib_utils.get_hash()
+        vm_port = self._get_fake_port(port_id)['port']
+
+        mac_addr = 'fa:16:3e:1b:30:00' if m_mac else vm_port['mac_address']
+        address_pairs = [
+            {'ip_address': '10.0.0.30',
+             'mac_address': mac_addr},
+            {'ip_address': 'fe80::f816:3eff:fe1c:36a9',
+             'mac_address': mac_addr},
+        ]
+        vm_port['allowed_address_pairs'].extend(address_pairs)
+
+        ip_addr = '10.0.0.29'
+        vm_port['allowed_address_pairs'].append(
+            {'ip_address': ip_addr,
+             'mac_address': m_mac if m_mac else vm_port['mac_address']}
+        )
+
+        cls._remove_from_allowed_address_pairs(m_driver, neutron,
+                            vm_port, frozenset([ip_addr]), m_mac)
+
+        m_driver._update_port_address_pairs.assert_called_once_with(neutron,
+                                          port_id, address_pairs)
+
+    def test_remove_from_allowed_address_pairs_no_ip_addresses(self):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+
+        port_id = lib_utils.get_hash()
+        vm_port = self._get_fake_port(port_id)['port']
+
+        self.assertRaises(k_exc.IntegrityError,
+                          cls._remove_from_allowed_address_pairs, m_driver,
+                          neutron, vm_port, frozenset())
+
+    @ddt.data((None), ('fa:16:3e:71:cb:80'))
+    def test_remove_from_allowed_address_pairs_missing(self, m_mac):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+
+        port_id = lib_utils.get_hash()
+        vm_port = self._get_fake_port(port_id)['port']
+
+        mac_addr = 'fa:16:3e:1b:30:00' if m_mac else vm_port['mac_address']
+        address_pairs = [
+            {'ip_address': '10.0.0.30',
+             'mac_address': mac_addr},
+            {'ip_address': 'fe80::f816:3eff:fe1c:36a9',
+             'mac_address': mac_addr},
+        ]
+        mac_addr = m_mac if m_mac else vm_port['mac_address']
+        vm_port['allowed_address_pairs'].extend(address_pairs)
+        vm_port['allowed_address_pairs'].append({'ip_address': '10.0.0.28',
+                                                 'mac_address': mac_addr})
+        ip_addr = ['10.0.0.29', '10.0.0.28']
+
+        cls._remove_from_allowed_address_pairs(m_driver, neutron,
+                            vm_port, frozenset(ip_addr), m_mac)
+
+        m_driver._update_port_address_pairs.assert_called_once_with(neutron,
+                                          port_id, address_pairs)
+
+    @ddt.data((None), ('fa:16:3e:71:cb:80'))
+    def test_remove_from_allowed_address_pairs_no_update(self, m_mac):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+
+        port_id = lib_utils.get_hash()
+        vm_port = self._get_fake_port(port_id)['port']
+
+        mac_addr = 'fa:16:3e:1b:30:00' if m_mac else vm_port['mac_address']
+        address_pairs = [
+            {'ip_address': '10.0.0.30',
+             'mac_address': mac_addr},
+            {'ip_address': 'fe80::f816:3eff:fe1c:36a9',
+             'mac_address': mac_addr},
+        ]
+        vm_port['allowed_address_pairs'].extend(address_pairs)
+
+        ip_addr = ['10.0.0.29']
+
+        cls._remove_from_allowed_address_pairs(m_driver, neutron,
+                            vm_port, frozenset(ip_addr), m_mac)
+
+        m_driver._update_port_address_pairs.assert_not_called()
+
+    def test_update_port_address_pairs(self):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+
+        port_id = lib_utils.get_hash()
+        pairs = mock.sentinel.allowed_address_pairs
+
+        cls._update_port_address_pairs(m_driver, neutron, port_id, pairs)
+
+        neutron.update_port.assert_called_with(port_id,
+            {'port': {'allowed_address_pairs': pairs}})
+
+    def test_update_port_address_pairs_failure(self):
+        cls = nested_macvlan_vif.NestedMacvlanPodVIFDriver
+        m_driver = mock.Mock(spec=cls)
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+
+        port_id = lib_utils.get_hash()
+        pairs = mock.sentinel.allowed_address_pairs
+        neutron.update_port.side_effect = n_exc.NeutronClientException
+
+        self.assertRaises(n_exc.NeutronClientException,
+                          cls._update_port_address_pairs, m_driver, neutron,
+                          port_id, pairs)
+
+        neutron.update_port.assert_called_with(port_id,
+            {'port': {'allowed_address_pairs': pairs}})
+
+    # TODO(garyloug) consider exending and moving to a parent class
+    def _get_fake_port(self, port_id=None, ip_address=None, mac_address=None):
+        fake_port = {
+            'port': {
+                "mac_address": "fa:16:3e:20:57:c4",
+                "fixed_ips": [],
+                "id": "07b21ebf-b105-4720-9f2e-95670c4032e4",
+                "allowed_address_pairs": []
+            }
+        }
+
+        if port_id:
+            fake_port['port']['id'] = port_id
+
+        if ip_address:
+            fake_port['port']['fixed_ips'].append({
+                "subnet_id": lib_utils.get_hash(),
+                "ip_address": ip_address
+            })
+
+        if mac_address:
+            fake_port['port']['mac_address'] = mac_address
+
+        return fake_port
+
+    def _get_fake_ports(self, ip_address, mac_address):
+        fake_port = self._get_fake_port(ip_address=ip_address,
+                                        mac_address=mac_address)
+        fake_port = fake_port['port']
+        fake_ports = {
+            'ports': [
+                fake_port
+            ]
+        }
+        return fake_ports

kuryr_kubernetes/tests/unit/controller/drivers/test_nested_vlan_vif.py

@@ -16,7 +16,6 @@ from kuryr.lib import constants as kl_const
 from kuryr.lib import exceptions as kl_exc
 from neutronclient.common import exceptions as n_exc
 
-from kuryr_kubernetes import constants as const
 from kuryr_kubernetes.controller.drivers import nested_vlan_vif
 from kuryr_kubernetes import exceptions as k_exc
 from kuryr_kubernetes.tests import base as test_base
@@ -25,7 +24,8 @@ from kuryr_kubernetes.tests.unit import kuryr_fixtures as k_fix
 
 class TestNestedVlanPodVIFDriver(test_base.TestCase):
 
-    @mock.patch('kuryr_kubernetes.os_vif_util.neutron_to_osvif_vif')
+    @mock.patch(
+        'kuryr_kubernetes.os_vif_util.neutron_to_osvif_vif_nested_vlan')
     def test_request_vif(self, m_to_vif):
         cls = nested_vlan_vif.NestedVlanPodVIFDriver
         m_driver = mock.Mock(spec=cls)
@@ -47,7 +47,6 @@ class TestNestedVlanPodVIFDriver(test_base.TestCase):
         vif = mock.Mock()
 
         m_to_vif.return_value = vif
-        m_driver._vif_plugin = const.K8S_OS_VIF_NOOP_PLUGIN
         m_driver._get_parent_port.return_value = parent_port
         m_driver._get_trunk_id.return_value = trunk_id
         m_driver._get_port_request.return_value = port_request
@@ -66,11 +65,10 @@ class TestNestedVlanPodVIFDriver(test_base.TestCase):
         m_driver._add_subport.assert_called_once_with(neutron,
                                                       trunk_id,
                                                       port_id)
-        m_to_vif.assert_called_once_with(const.K8S_OS_VIF_NOOP_PLUGIN, port,
-                                         subnets)
-        self.assertEqual(vif.vlan_id, vlan_id)
+        m_to_vif.assert_called_once_with(port, subnets, vlan_id)
 
-    @mock.patch('kuryr_kubernetes.os_vif_util.neutron_to_osvif_vif')
+    @mock.patch(
+        'kuryr_kubernetes.os_vif_util.neutron_to_osvif_vif_nested_vlan')
     def test_request_vifs(self, m_to_vif):
         cls = nested_vlan_vif.NestedVlanPodVIFDriver
         m_driver = mock.Mock(spec=cls)
@@ -102,7 +100,6 @@ class TestNestedVlanPodVIFDriver(test_base.TestCase):
                                                        subports_info)
         neutron.create_port.return_value = {'ports': [port, port]}
         m_to_vif.return_value = vif
-        m_driver._vif_plugin = const.K8S_OS_VIF_NOOP_PLUGIN
 
         self.assertEqual([vif, vif], cls.request_vifs(
             m_driver, pod, project_id, subnets, security_groups, num_ports))
@@ -117,8 +114,8 @@ class TestNestedVlanPodVIFDriver(test_base.TestCase):
             {'sub_ports': subports_info})
         neutron.delete_port.assert_not_called()
 
-        calls = [mock.call(const.K8S_OS_VIF_NOOP_PLUGIN, port, subnets)
-                 for _ in range(len(subports_info))]
+        calls = [mock.call(port, subnets, info['segmentation_id'])
+                 for info in subports_info]
         m_to_vif.assert_has_calls(calls)
 
     def test_request_vifs_no_vlans(self):

kuryr_kubernetes/tests/unit/test_os_vif_plug_noop.py

@@ -81,5 +81,9 @@ class TestNoOpPlugin(base.TestCase):
                     vif_object_name=k_vif.VIFVlanNested.__name__,
                     min_version="1.0",
                     max_version="1.0"),
+                objects.host_info.HostVIFInfo(
+                    vif_object_name=k_vif.VIFMacvlanNested.__name__,
+                    min_version="1.0",
+                    max_version="1.0"),
             ])
         self.assertEqual(expected, result)

kuryr_kubernetes/tests/unit/test_os_vif_util.py

@@ -22,6 +22,7 @@ from os_vif.objects import subnet as osv_subnet
 from oslo_config import cfg as o_cfg
 from oslo_utils import uuidutils
 
+from kuryr_kubernetes import constants as const
 from kuryr_kubernetes import exceptions as k_exc
 from kuryr_kubernetes import os_vif_util as ovu
 from kuryr_kubernetes.tests import base as test_base
@@ -262,9 +263,54 @@ class TestOSVIFUtils(test_base.TestCase):
     @mock.patch('kuryr_kubernetes.os_vif_util._is_port_active')
     @mock.patch('kuryr_kubernetes.os_vif_util._make_vif_network')
     @mock.patch('kuryr_kubernetes.objects.vif.VIFVlanNested')
-    def test_neutron_to_osvif_nested(self, m_mk_vif, m_make_vif_network,
-                                     m_is_port_active, m_get_vif_name):
-        vif_plugin = 'noop'
+    def test_neutron_to_osvif_nested_vlan(self, m_mk_vif, m_make_vif_network,
+                               m_is_port_active, m_get_vif_name):
+        vif_plugin = const.K8S_OS_VIF_NOOP_PLUGIN
+        port_id = mock.sentinel.port_id
+        mac_address = mock.sentinel.mac_address
+        port_filter = mock.sentinel.port_filter
+        subnets = mock.sentinel.subnets
+        network = mock.sentinel.network
+        port_active = mock.sentinel.port_active
+        vif_name = mock.sentinel.vif_name
+        vif = mock.sentinel.vif
+        vlan_id = mock.sentinel.vlan_id
+
+        m_make_vif_network.return_value = network
+        m_is_port_active.return_value = port_active
+        m_get_vif_name.return_value = vif_name
+        m_mk_vif.return_value = vif
+
+        port = {'id': port_id,
+                'mac_address': mac_address,
+                'binding:vif_details': {
+                    'port_filter': port_filter},
+                }
+
+        self.assertEqual(vif, ovu.neutron_to_osvif_vif_nested_vlan(port,
+                         subnets, vlan_id))
+
+        m_make_vif_network.assert_called_once_with(port, subnets)
+        m_is_port_active.assert_called_once_with(port)
+        m_get_vif_name.assert_called_once_with(port)
+        m_mk_vif.assert_called_once_with(
+            id=port_id,
+            address=mac_address,
+            network=network,
+            has_traffic_filtering=port_filter,
+            preserve_on_delete=False,
+            active=port_active,
+            plugin=vif_plugin,
+            vif_name=vif_name,
+            vlan_id=vlan_id)
+
+    @mock.patch('kuryr_kubernetes.os_vif_util._get_vif_name')
+    @mock.patch('kuryr_kubernetes.os_vif_util._is_port_active')
+    @mock.patch('kuryr_kubernetes.os_vif_util._make_vif_network')
+    @mock.patch('kuryr_kubernetes.objects.vif.VIFMacvlanNested')
+    def test_neutron_to_osvif_nested_macvlan(self, m_mk_vif,
+                        m_make_vif_network, m_is_port_active, m_get_vif_name):
+        vif_plugin = const.K8S_OS_VIF_NOOP_PLUGIN
         port_id = mock.sentinel.port_id
         mac_address = mock.sentinel.mac_address
         port_filter = mock.sentinel.port_filter
@@ -285,8 +331,8 @@ class TestOSVIFUtils(test_base.TestCase):
                     'port_filter': port_filter},
                 }
 
-        self.assertEqual(vif, ovu.neutron_to_osvif_vif_nested(vif_plugin, port,
-                                                           subnets))
+        self.assertEqual(vif, ovu.neutron_to_osvif_vif_nested_macvlan(port,
+                                                                      subnets))
 
         m_make_vif_network.assert_called_once_with(port, subnets)
         m_is_port_active.assert_called_once_with(port)

setup.cfg

@@ -32,12 +32,12 @@ console_scripts =
 
 kuryr_kubernetes.vif_translators =
     ovs = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_ovs
-    noop = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_nested
 
 kuryr_kubernetes.cni.binding =
     VIFBridge = kuryr_kubernetes.cni.binding.bridge:BridgeDriver
     VIFOpenVSwitch = kuryr_kubernetes.cni.binding.bridge:VIFOpenVSwitchDriver
     VIFVlanNested = kuryr_kubernetes.cni.binding.nested:VlanDriver
+    VIFMacvlanNested = kuryr_kubernetes.cni.binding.nested:MacvlanDriver
 
 kuryr_kubernetes.controller.drivers.pod_project =
     default = kuryr_kubernetes.controller.drivers.default_project:DefaultPodProjectDriver
@@ -60,6 +60,7 @@ kuryr_kubernetes.controller.drivers.service_security_groups =
 kuryr_kubernetes.controller.drivers.pod_vif =
     neutron-vif = kuryr_kubernetes.controller.drivers.neutron_vif:NeutronPodVIFDriver
     nested-vlan = kuryr_kubernetes.controller.drivers.nested_vlan_vif:NestedVlanPodVIFDriver
+    nested-macvlan = kuryr_kubernetes.controller.drivers.nested_macvlan_vif:NestedMacvlanPodVIFDriver
 
 kuryr_kubernetes.controller.drivers.endpoints_lbaas =
     lbaasv2 = kuryr_kubernetes.controller.drivers.lbaasv2:LBaaSv2Driver