Add DPDK support for nested pods
Add DPDK support for nested K8s pods. Patch includes a new VIF driver on the controller and a new CNI binding driver. This patch introduces dependency from os-vif v.1.12.0, since there a new vif type. Change-Id: I6be9110192f524325e24fb97d905faff86d0cfef Implements: blueprint nested-dpdk-support Co-Authored-By: Kural Ramakrishnan <kuralamudhan.ramakrishnan@intel.com> Co-Authored-By: Marco Chiappero <marco.chiappero@intel.com> Signed-off-by: Alexey Perevalov <a.perevalov@samsung.com> Signed-off-by: Danil Golov <d.golov@samsung.com>
This commit is contained in:
parent
09b07992b0
commit
edc6597fe2
|
@ -11,7 +11,7 @@ ARG UPPER_CONSTRAINTS_FILE="https://releases.openstack.org/constraints/upper/mas
|
||||||
ARG OSLO_LOCK_PATH=/var/kuryr-lock
|
ARG OSLO_LOCK_PATH=/var/kuryr-lock
|
||||||
|
|
||||||
RUN yum install -y epel-release https://rdoproject.org/repos/rdo-release.rpm \
|
RUN yum install -y epel-release https://rdoproject.org/repos/rdo-release.rpm \
|
||||||
&& yum install -y --setopt=tsflags=nodocs python3-pip openvswitch sudo \
|
&& yum install -y --setopt=tsflags=nodocs python3-pip openvswitch sudo iproute libstdc++ pciutils kmod-libs \
|
||||||
&& yum install -y --setopt=tsflags=nodocs gcc python3-devel git
|
&& yum install -y --setopt=tsflags=nodocs gcc python3-devel git
|
||||||
|
|
||||||
COPY . /opt/kuryr-kubernetes
|
COPY . /opt/kuryr-kubernetes
|
||||||
|
|
|
@ -636,6 +636,8 @@ spec:
|
||||||
mountPath: /etc/kuryr
|
mountPath: /etc/kuryr
|
||||||
- name: proc
|
- name: proc
|
||||||
mountPath: /host_proc
|
mountPath: /host_proc
|
||||||
|
- name: var-pci
|
||||||
|
mountPath: /var/pci_address
|
||||||
EOF
|
EOF
|
||||||
if [[ -n "$VAR_RUN_PATH" ]]; then
|
if [[ -n "$VAR_RUN_PATH" ]]; then
|
||||||
cat >> "${output_dir}/cni_ds.yml" << EOF
|
cat >> "${output_dir}/cni_ds.yml" << EOF
|
||||||
|
@ -669,6 +671,9 @@ EOF
|
||||||
- name: proc
|
- name: proc
|
||||||
hostPath:
|
hostPath:
|
||||||
path: /proc
|
path: /proc
|
||||||
|
- name: var-pci
|
||||||
|
hostPath:
|
||||||
|
path: /var/pci_address
|
||||||
EOF
|
EOF
|
||||||
if [[ -n "$VAR_RUN_PATH" ]]; then
|
if [[ -n "$VAR_RUN_PATH" ]]; then
|
||||||
cat >> "${output_dir}/cni_ds.yml" << EOF
|
cat >> "${output_dir}/cni_ds.yml" << EOF
|
||||||
|
|
|
@ -58,6 +58,10 @@ def get_pod_resources_client():
|
||||||
return _clients[_POD_RESOURCES_CLIENT]
|
return _clients[_POD_RESOURCES_CLIENT]
|
||||||
|
|
||||||
|
|
||||||
|
def get_compute_client():
|
||||||
|
return _clients[_OPENSTACKSDK].compute
|
||||||
|
|
||||||
|
|
||||||
def setup_clients():
|
def setup_clients():
|
||||||
setup_neutron_client()
|
setup_neutron_client()
|
||||||
setup_kubernetes_client()
|
setup_kubernetes_client()
|
||||||
|
|
|
@ -121,7 +121,13 @@ def _configure_l3(vif, ifname, netns, is_default_gateway):
|
||||||
|
|
||||||
def _need_configure_l3(vif):
|
def _need_configure_l3(vif):
|
||||||
if not hasattr(vif, 'physnet'):
|
if not hasattr(vif, 'physnet'):
|
||||||
|
# NOTE(danil): non-sriov vif. Figure out if it is nested-dpdk
|
||||||
|
if vif.obj_attr_is_set('port_profile') and hasattr(vif.port_profile,
|
||||||
|
'l3_setup'):
|
||||||
|
return vif.port_profile.l3_setup
|
||||||
|
# NOTE(danil): by default kuryr-kubernetes has to setup l3
|
||||||
return True
|
return True
|
||||||
|
# NOTE(danil): sriov vif. Figure out what driver should compute it
|
||||||
physnet = vif.physnet
|
physnet = vif.physnet
|
||||||
mapping_res = config.CONF.sriov.physnet_resource_mappings
|
mapping_res = config.CONF.sriov.physnet_resource_mappings
|
||||||
try:
|
try:
|
||||||
|
@ -139,6 +145,7 @@ def _need_configure_l3(vif):
|
||||||
LOG.info("_configure_l3 will not be called for vif %s "
|
LOG.info("_configure_l3 will not be called for vif %s "
|
||||||
"because of it's driver", vif)
|
"because of it's driver", vif)
|
||||||
return False
|
return False
|
||||||
|
# NOTE(danil): sriov vif computed by kernel driver
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,194 @@
|
||||||
|
# Copyright (C) 2020 Intel Corporation
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
from oslo_log import log as logging
|
||||||
|
from oslo_serialization import jsonutils
|
||||||
|
|
||||||
|
from kuryr_kubernetes import clients
|
||||||
|
from kuryr_kubernetes.cni.binding import base as b_base
|
||||||
|
from kuryr_kubernetes import constants
|
||||||
|
from kuryr_kubernetes.handlers import health
|
||||||
|
from kuryr_kubernetes import utils
|
||||||
|
|
||||||
|
from kuryr.lib._i18n import _
|
||||||
|
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
CONF = cfg.CONF
|
||||||
|
|
||||||
|
NET_DEV_PATH = "/sys/class/net/{}/device"
|
||||||
|
VIRTIO_DEVS_PATH = "/sys/bus/virtio/devices"
|
||||||
|
PCI_PATH = "/sys/bus/pci/devices"
|
||||||
|
PCI_DRVS_PATH = "/sys/bus/pci/drivers"
|
||||||
|
|
||||||
|
|
||||||
|
# TODO(garyloug) These should probably eventually move to config.py
|
||||||
|
# TODO(garyloug) Would be nice if dpdk_driver is set as CNI arg
|
||||||
|
nested_dpdk_opts = [
|
||||||
|
cfg.StrOpt('dpdk_driver',
|
||||||
|
help=_('The DPDK driver that the device will be bound to after '
|
||||||
|
'it is unbound from the kernel driver'),
|
||||||
|
default='uio_pci_generic'),
|
||||||
|
cfg.StrOpt('pci_mount_point',
|
||||||
|
help=_('Absolute path to directory containing pci address of '
|
||||||
|
'devices to be used by DPDK application'),
|
||||||
|
default='/var/pci_address'),
|
||||||
|
]
|
||||||
|
|
||||||
|
CONF.register_opts(nested_dpdk_opts, "nested_dpdk")
|
||||||
|
|
||||||
|
|
||||||
|
class DpdkDriver(health.HealthHandler):
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super(DpdkDriver, self).__init__()
|
||||||
|
|
||||||
|
def connect(self, vif, ifname, netns, container_id):
|
||||||
|
name = self._get_iface_name_by_mac(vif.address)
|
||||||
|
driver, pci_addr = self._get_device_info(name)
|
||||||
|
|
||||||
|
vif.dev_driver = driver
|
||||||
|
vif.pci_address = pci_addr
|
||||||
|
dpdk_driver = CONF.nested_dpdk.dpdk_driver
|
||||||
|
self._change_driver_binding(pci_addr, dpdk_driver)
|
||||||
|
self._create_pci_file(pci_addr, container_id, ifname)
|
||||||
|
self._set_vif(vif)
|
||||||
|
|
||||||
|
def disconnect(self, vif, ifname, netns, container_id):
|
||||||
|
self._remove_pci_file(container_id, ifname)
|
||||||
|
|
||||||
|
def _get_iface_name_by_mac(self, mac_address):
|
||||||
|
with b_base.get_ipdb() as h_ipdb:
|
||||||
|
for name, data in h_ipdb.interfaces.items():
|
||||||
|
if data['address'] == mac_address:
|
||||||
|
return data['ifname']
|
||||||
|
|
||||||
|
def _get_device_info(self, ifname):
|
||||||
|
"""Get driver and PCI addr by using sysfs"""
|
||||||
|
|
||||||
|
# TODO(garyloug): check the type (virtio)
|
||||||
|
dev = os.path.basename(os.readlink(NET_DEV_PATH.format(ifname)))
|
||||||
|
pci_link = os.readlink(os.path.join(VIRTIO_DEVS_PATH, dev))
|
||||||
|
pci_addr = os.path.basename(os.path.dirname(pci_link))
|
||||||
|
pci_driver_link = os.readlink(os.path.join(PCI_PATH, pci_addr,
|
||||||
|
'driver'))
|
||||||
|
pci_driver = os.path.basename(pci_driver_link)
|
||||||
|
|
||||||
|
return pci_driver, pci_addr
|
||||||
|
|
||||||
|
def _change_driver_binding(self, pci, driver):
|
||||||
|
old_driver_path = os.path.join(PCI_PATH, pci, 'driver')
|
||||||
|
old_driver_link = os.readlink(old_driver_path)
|
||||||
|
old_driver = os.path.basename(old_driver_link)
|
||||||
|
|
||||||
|
unbind_path = os.path.join(PCI_DRVS_PATH, old_driver, 'unbind')
|
||||||
|
bind_path = os.path.join(PCI_DRVS_PATH, driver, 'bind')
|
||||||
|
|
||||||
|
with open(unbind_path, 'w') as unbind_fd:
|
||||||
|
unbind_fd.write(pci)
|
||||||
|
|
||||||
|
override = os.path.join(PCI_PATH, pci, 'driver_override')
|
||||||
|
# NOTE(danil): to change driver for device it is necessary to
|
||||||
|
# write the name of this driver into override_fd. Before that
|
||||||
|
# Null should be written there. This process is described properly
|
||||||
|
# in dpdk-devbind.py script by DPDK
|
||||||
|
with open(override, 'w') as override_fd:
|
||||||
|
override_fd.write("\00")
|
||||||
|
|
||||||
|
with open(override, 'w') as override_fd:
|
||||||
|
override_fd.write(driver)
|
||||||
|
|
||||||
|
with open(bind_path, 'w') as bind_fd:
|
||||||
|
bind_fd.write(pci)
|
||||||
|
|
||||||
|
LOG.info("Device %s was binded on driver %s. Old driver is %s", pci,
|
||||||
|
driver, old_driver)
|
||||||
|
|
||||||
|
def _create_pci_file(self, pci_addr, container_id, ifname):
|
||||||
|
# NOTE(danil): writing used pci addresses is necessary to know what
|
||||||
|
# device to use by dpdk applications inside containers
|
||||||
|
try:
|
||||||
|
os.makedirs(CONF.nested_dpdk.pci_mount_point, exists_ok=True)
|
||||||
|
file_path = os.path.join(CONF.nested_dpdk.pci_mount_point,
|
||||||
|
container_id + '-' + ifname)
|
||||||
|
with open(file_path, 'w') as fd:
|
||||||
|
fd.write(pci_addr)
|
||||||
|
except OSError as err:
|
||||||
|
LOG.exception('Cannot create file %s. Error message: (%d) %s',
|
||||||
|
file_path, err.errno, err.strerror)
|
||||||
|
|
||||||
|
def _remove_pci_file(self, container_id, ifname):
|
||||||
|
file_path = os.path.join(CONF.nested_dpdk.pci_mount_point,
|
||||||
|
container_id + '-' + ifname)
|
||||||
|
try:
|
||||||
|
os.remove(file_path)
|
||||||
|
except OSError as err:
|
||||||
|
LOG.warning('Cannot remove file %s. Error message: (%d) %s',
|
||||||
|
file_path, err.errno, err.strerror)
|
||||||
|
|
||||||
|
def _set_vif(self, vif):
|
||||||
|
# TODO(ivc): extract annotation interactions
|
||||||
|
state, labels, resource_version = self._get_pod_details(
|
||||||
|
vif.port_profile.selflink)
|
||||||
|
for ifname, vif_ex in state.vifs.items():
|
||||||
|
if vif.id == vif_ex.id:
|
||||||
|
state.vifs[ifname] = vif
|
||||||
|
break
|
||||||
|
self._set_pod_details(state, vif.port_profile.selflink, labels,
|
||||||
|
resource_version)
|
||||||
|
|
||||||
|
def _get_pod_details(self, selflink):
|
||||||
|
k8s = clients.get_kubernetes_client()
|
||||||
|
pod = k8s.get(selflink)
|
||||||
|
annotations = pod['metadata']['annotations']
|
||||||
|
resource_version = pod['metadata']['resourceVersion']
|
||||||
|
labels = pod['metadata'].get('labels')
|
||||||
|
try:
|
||||||
|
annotations = annotations[constants.K8S_ANNOTATION_VIF]
|
||||||
|
state_annotation = jsonutils.loads(annotations)
|
||||||
|
state = utils.extract_pod_annotation(state_annotation)
|
||||||
|
except KeyError:
|
||||||
|
LOG.exception("No annotations %s", constants.K8S_ANNOTATION_VIF)
|
||||||
|
raise
|
||||||
|
except ValueError:
|
||||||
|
LOG.exception("Unable encode annotations")
|
||||||
|
raise
|
||||||
|
LOG.info("Got VIFs from annotation: %s", state.vifs)
|
||||||
|
return state, labels, resource_version
|
||||||
|
|
||||||
|
def _set_pod_details(self, state, selflink, labels, resource_version):
|
||||||
|
if not state:
|
||||||
|
LOG.info("Removing VIFs annotation: %r", state)
|
||||||
|
annotation = None
|
||||||
|
else:
|
||||||
|
state_dict = state.obj_to_primitive()
|
||||||
|
annotation = jsonutils.dumps(state_dict, sort_keys=True)
|
||||||
|
LOG.info("Setting VIFs annotation: %r", annotation)
|
||||||
|
|
||||||
|
if not labels:
|
||||||
|
LOG.info("Removing Label annotation: %r", labels)
|
||||||
|
labels_annotation = None
|
||||||
|
else:
|
||||||
|
labels_annotation = jsonutils.dumps(labels, sort_keys=True)
|
||||||
|
LOG.info("Setting Labels annotation: %r", labels_annotation)
|
||||||
|
|
||||||
|
k8s = clients.get_kubernetes_client()
|
||||||
|
k8s.annotate(selflink,
|
||||||
|
{constants.K8S_ANNOTATION_VIF: annotation,
|
||||||
|
constants.K8S_ANNOTATION_LABEL: labels_annotation},
|
||||||
|
resource_version=resource_version)
|
|
@ -0,0 +1,76 @@
|
||||||
|
# Copyright (C) 2020 Intel Corporation
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from openstack import exceptions as o_exc
|
||||||
|
from oslo_log import log as logging
|
||||||
|
|
||||||
|
from kuryr_kubernetes import clients
|
||||||
|
from kuryr_kubernetes.controller.drivers import nested_vif
|
||||||
|
from kuryr_kubernetes.controller.drivers import utils
|
||||||
|
from kuryr_kubernetes import os_vif_util as ovu
|
||||||
|
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class NestedDpdkPodVIFDriver(nested_vif.NestedPodVIFDriver):
|
||||||
|
"""Manages ports for DPDK based nested-containers to provide VIFs."""
|
||||||
|
|
||||||
|
# TODO(garyloug): maybe log a warning if the vswitch is not ovs-dpdk?
|
||||||
|
|
||||||
|
def request_vif(self, pod, project_id, subnets, security_groups):
|
||||||
|
neutron = clients.get_neutron_client()
|
||||||
|
compute = clients.get_compute_client()
|
||||||
|
|
||||||
|
vm_id = self._get_parent_port(neutron, pod)['device_id']
|
||||||
|
net_id = utils.get_network_id(subnets)
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = compute.create_server_interface(vm_id, net_id=net_id)
|
||||||
|
except o_exc.SDKException:
|
||||||
|
LOG.warning("Unable to create interface for server %s.",
|
||||||
|
vm_id)
|
||||||
|
raise
|
||||||
|
port = neutron.show_port(result.port_id).get('port')
|
||||||
|
return ovu.neutron_to_osvif_vif_dpdk(port, subnets, pod)
|
||||||
|
|
||||||
|
def request_vifs(self, pod, project_id, subnets, security_groups,
|
||||||
|
num_ports):
|
||||||
|
# TODO(garyloug): provide an implementation
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def release_vif(self, pod, vif, project_id=None, security_groups=None):
|
||||||
|
neutron = clients.get_neutron_client()
|
||||||
|
compute = clients.get_compute_client()
|
||||||
|
|
||||||
|
vm_id = self._get_parent_port(neutron, pod)['device_id']
|
||||||
|
LOG.debug("release_vif for vm_id %s %s", vm_id, vif.id)
|
||||||
|
|
||||||
|
try:
|
||||||
|
compute.delete_server_interface(vif.id, server=vm_id)
|
||||||
|
except o_exc.SDKException:
|
||||||
|
LOG.warning("Unable to delete interface %s for server %s.",
|
||||||
|
vif.id, vm_id)
|
||||||
|
raise
|
||||||
|
|
||||||
|
def activate_vif(self, pod, vif):
|
||||||
|
# NOTE(danil): new virtual interface was created in nova instance
|
||||||
|
# during request_vif call, thus if it was not created successfully
|
||||||
|
# an exception o_exc.SDKException would be throwed. During binding
|
||||||
|
# process only rebinding of interface on userspace driver was done.
|
||||||
|
# There is no any chance to check the state of rebinded interface.
|
||||||
|
# Thus just set 'active' immediately to let the CNI driver make
|
||||||
|
# progress.
|
||||||
|
vif.active = True
|
|
@ -99,7 +99,8 @@ VIF_TYPE_TO_DRIVER_MAPPING = {
|
||||||
'VIFBridge': 'neutron-vif',
|
'VIFBridge': 'neutron-vif',
|
||||||
'VIFVlanNested': 'nested-vlan',
|
'VIFVlanNested': 'nested-vlan',
|
||||||
'VIFMacvlanNested': 'nested-macvlan',
|
'VIFMacvlanNested': 'nested-macvlan',
|
||||||
'VIFSriov': 'sriov'
|
'VIFSriov': 'sriov',
|
||||||
|
'VIFDPDKNested': 'nested-dpdk',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -83,3 +83,15 @@ class VIFSriov(obj_osvif.VIFDirect):
|
||||||
'pod_name': obj_fields.StringField(),
|
'pod_name': obj_fields.StringField(),
|
||||||
'pod_link': obj_fields.StringField(),
|
'pod_link': obj_fields.StringField(),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@obj_base.VersionedObjectRegistry.register
|
||||||
|
class VIFDPDKNested(obj_osvif.VIFNestedDPDK):
|
||||||
|
# This is OVO based DPDK Nested vif.
|
||||||
|
|
||||||
|
VERSION = '1.0'
|
||||||
|
|
||||||
|
fields = {
|
||||||
|
# name of the VIF
|
||||||
|
'vif_name': obj_fields.StringField(),
|
||||||
|
}
|
||||||
|
|
|
@ -31,6 +31,10 @@ class NoOpPlugin(PluginBase):
|
||||||
vif_object_name=k_vif.VIFMacvlanNested.__name__,
|
vif_object_name=k_vif.VIFMacvlanNested.__name__,
|
||||||
min_version="1.0",
|
min_version="1.0",
|
||||||
max_version="1.0"),
|
max_version="1.0"),
|
||||||
|
objects.host_info.HostVIFInfo(
|
||||||
|
vif_object_name=k_vif.VIFDPDKNested.__name__,
|
||||||
|
min_version="1.0",
|
||||||
|
max_version="1.0"),
|
||||||
])
|
])
|
||||||
|
|
||||||
def plug(self, vif, instance_info):
|
def plug(self, vif, instance_info):
|
||||||
|
|
|
@ -349,6 +349,35 @@ def neutron_to_osvif_vif_sriov(vif_plugin, os_port, subnets):
|
||||||
return vif
|
return vif
|
||||||
|
|
||||||
|
|
||||||
|
def neutron_to_osvif_vif_dpdk(os_port, subnets, pod):
|
||||||
|
"""Converts Neutron port to VIF object for nested dpdk containers.
|
||||||
|
|
||||||
|
:param os_port: dict containing port information as returned by
|
||||||
|
neutron client's 'show_port'
|
||||||
|
:param subnets: subnet mapping as returned by PodSubnetsDriver.get_subnets
|
||||||
|
:param pod: pod object received by k8s and containing profile details
|
||||||
|
:return: os-vif VIF object
|
||||||
|
"""
|
||||||
|
|
||||||
|
details = os_port.get('binding:vif_details', {})
|
||||||
|
profile = osv_vif.VIFPortProfileK8sDPDK(
|
||||||
|
l3_setup=False,
|
||||||
|
selflink=pod['metadata']['selfLink'])
|
||||||
|
|
||||||
|
return k_vif.VIFDPDKNested(
|
||||||
|
id=os_port['id'],
|
||||||
|
port_profile=profile,
|
||||||
|
address=os_port['mac_address'],
|
||||||
|
network=_make_vif_network(os_port, subnets),
|
||||||
|
has_traffic_filtering=details.get('port_filter', False),
|
||||||
|
preserve_on_delete=False,
|
||||||
|
active=_is_port_active(os_port),
|
||||||
|
plugin=const.K8S_OS_VIF_NOOP_PLUGIN,
|
||||||
|
pci_address="",
|
||||||
|
dev_driver="",
|
||||||
|
vif_name=_get_vif_name(os_port))
|
||||||
|
|
||||||
|
|
||||||
def neutron_to_osvif_vif(vif_translator, os_port, subnets):
|
def neutron_to_osvif_vif(vif_translator, os_port, subnets):
|
||||||
"""Converts Neutron port to os-vif VIF object.
|
"""Converts Neutron port to os-vif VIF object.
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,231 @@
|
||||||
|
# Copyright (C) 2020 Intel Corporation
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import ddt
|
||||||
|
import mock
|
||||||
|
|
||||||
|
from kuryr_kubernetes.controller.drivers import nested_dpdk_vif
|
||||||
|
from kuryr_kubernetes.tests import base as test_base
|
||||||
|
from kuryr_kubernetes.tests.unit import kuryr_fixtures as k_fix
|
||||||
|
|
||||||
|
from neutronclient.common import exceptions as ntron_exc
|
||||||
|
from openstack import exceptions as o_exc
|
||||||
|
|
||||||
|
|
||||||
|
@ddt.ddt
|
||||||
|
class TestNestedDpdkVIFDriver(test_base.TestCase):
|
||||||
|
|
||||||
|
@mock.patch(
|
||||||
|
'kuryr_kubernetes.os_vif_util.neutron_to_osvif_vif_dpdk')
|
||||||
|
@mock.patch('kuryr_kubernetes.controller.drivers.utils.get_network_id')
|
||||||
|
def test_request_vif(self, m_get_network_id, m_to_vif):
|
||||||
|
cls = nested_dpdk_vif.NestedDpdkPodVIFDriver
|
||||||
|
m_driver = mock.Mock(spec=cls)
|
||||||
|
neutron = self.useFixture(k_fix.MockNeutronClient()).client
|
||||||
|
compute = self.useFixture(k_fix.MockComputeClient()).client
|
||||||
|
|
||||||
|
pod = mock.sentinel.pod
|
||||||
|
project_id = mock.sentinel.project_id
|
||||||
|
subnets = mock.sentinel.subnets
|
||||||
|
security_groups = mock.sentinel.security_groups
|
||||||
|
vm_id = mock.sentinel.parent_port_id
|
||||||
|
net_id = mock.sentinel.net_id
|
||||||
|
port_id = mock.sentinel.port_id
|
||||||
|
port = mock.sentinel.port
|
||||||
|
|
||||||
|
parent_port = mock.MagicMock()
|
||||||
|
vif = mock.Mock()
|
||||||
|
result = mock.Mock()
|
||||||
|
|
||||||
|
parent_port.__getitem__.return_value = vm_id
|
||||||
|
result.port_id = port_id
|
||||||
|
compute.create_server_interface.return_value = result
|
||||||
|
m_to_vif.return_value = vif
|
||||||
|
m_driver._get_parent_port.return_value = parent_port
|
||||||
|
m_get_network_id.return_value = net_id
|
||||||
|
neutron.show_port.return_value.get.return_value = port
|
||||||
|
|
||||||
|
self.assertEqual(vif, cls.request_vif(m_driver, pod, project_id,
|
||||||
|
subnets, security_groups))
|
||||||
|
|
||||||
|
m_driver._get_parent_port.assert_called_once_with(neutron, pod)
|
||||||
|
m_get_network_id.assert_called_once_with(subnets)
|
||||||
|
compute.create_server_interface.assert_called_once_with(
|
||||||
|
vm_id, net_id=net_id)
|
||||||
|
neutron.show_port.assert_called_once_with(result.port_id)
|
||||||
|
m_to_vif.assert_called_once_with(port, subnets, pod)
|
||||||
|
|
||||||
|
@mock.patch(
|
||||||
|
'kuryr_kubernetes.os_vif_util.neutron_to_osvif_vif_dpdk')
|
||||||
|
@mock.patch('kuryr_kubernetes.controller.drivers.utils.get_network_id')
|
||||||
|
def test_request_vif_parent_not_found(self, m_get_network_id, m_to_vif):
|
||||||
|
cls = nested_dpdk_vif.NestedDpdkPodVIFDriver
|
||||||
|
m_driver = mock.Mock(spec=cls)
|
||||||
|
neutron = self.useFixture(k_fix.MockNeutronClient()).client
|
||||||
|
compute = self.useFixture(k_fix.MockComputeClient()).client
|
||||||
|
|
||||||
|
pod = mock.sentinel.pod
|
||||||
|
project_id = mock.sentinel.project_id
|
||||||
|
subnets = mock.sentinel.subnets
|
||||||
|
security_groups = mock.sentinel.security_groups
|
||||||
|
vm_id = mock.sentinel.parent_port_id
|
||||||
|
net_id = mock.sentinel.net_id
|
||||||
|
port_id = mock.sentinel.port_id
|
||||||
|
port = mock.sentinel.port
|
||||||
|
|
||||||
|
parent_port = mock.MagicMock()
|
||||||
|
vif = mock.Mock()
|
||||||
|
result = mock.Mock()
|
||||||
|
|
||||||
|
parent_port.__getitem__.return_value = vm_id
|
||||||
|
result.port_id = port_id
|
||||||
|
compute.create_server_interface.return_value = result
|
||||||
|
m_to_vif.return_value = vif
|
||||||
|
m_driver._get_parent_port.side_effect = \
|
||||||
|
ntron_exc.NeutronClientException
|
||||||
|
m_get_network_id.return_value = net_id
|
||||||
|
neutron.show_port.return_value.get.return_value = port
|
||||||
|
|
||||||
|
self.assertRaises(ntron_exc.NeutronClientException, cls.request_vif,
|
||||||
|
m_driver, pod, project_id, subnets, security_groups)
|
||||||
|
|
||||||
|
m_driver._get_parent_port.assert_called_once_with(neutron, pod)
|
||||||
|
m_get_network_id.assert_not_called()
|
||||||
|
compute.create_server_interface.assert_not_called()
|
||||||
|
neutron.show_port.assert_not_called()
|
||||||
|
m_to_vif.assert_not_called()
|
||||||
|
|
||||||
|
@mock.patch(
|
||||||
|
'kuryr_kubernetes.os_vif_util.neutron_to_osvif_vif_dpdk')
|
||||||
|
@mock.patch('kuryr_kubernetes.controller.drivers.utils.get_network_id')
|
||||||
|
def test_request_vif_attach_failed(self, m_get_network_id, m_to_vif):
|
||||||
|
cls = nested_dpdk_vif.NestedDpdkPodVIFDriver
|
||||||
|
m_driver = mock.Mock(spec=cls)
|
||||||
|
neutron = self.useFixture(k_fix.MockNeutronClient()).client
|
||||||
|
compute = self.useFixture(k_fix.MockComputeClient()).client
|
||||||
|
|
||||||
|
pod = mock.sentinel.pod
|
||||||
|
project_id = mock.sentinel.project_id
|
||||||
|
subnets = mock.sentinel.subnets
|
||||||
|
security_groups = mock.sentinel.security_groups
|
||||||
|
vm_id = mock.sentinel.parent_port_id
|
||||||
|
net_id = mock.sentinel.net_id
|
||||||
|
port_id = mock.sentinel.port_id
|
||||||
|
port = mock.sentinel.port
|
||||||
|
|
||||||
|
parent_port = mock.MagicMock()
|
||||||
|
vif = mock.Mock()
|
||||||
|
result = mock.Mock()
|
||||||
|
|
||||||
|
parent_port.__getitem__.return_value = vm_id
|
||||||
|
result.port_id = port_id
|
||||||
|
m_to_vif.return_value = vif
|
||||||
|
m_driver._get_parent_port.return_value = parent_port
|
||||||
|
m_get_network_id.return_value = net_id
|
||||||
|
neutron.show_port.return_value.get.return_value = port
|
||||||
|
compute.create_server_interface.side_effect = o_exc.SDKException
|
||||||
|
|
||||||
|
self.assertRaises(o_exc.SDKException, cls.request_vif,
|
||||||
|
m_driver, pod, project_id, subnets, security_groups)
|
||||||
|
|
||||||
|
m_driver._get_parent_port.assert_called_once_with(neutron, pod)
|
||||||
|
m_get_network_id.assert_called_once_with(subnets)
|
||||||
|
compute.create_server_interface.assert_called_once_with(
|
||||||
|
vm_id, net_id=net_id)
|
||||||
|
neutron.show_port.assert_not_called()
|
||||||
|
m_to_vif.assert_not_called()
|
||||||
|
|
||||||
|
def test_release_vif(self):
|
||||||
|
cls = nested_dpdk_vif.NestedDpdkPodVIFDriver
|
||||||
|
m_driver = mock.Mock(spec=cls)
|
||||||
|
neutron = self.useFixture(k_fix.MockNeutronClient()).client
|
||||||
|
compute = self.useFixture(k_fix.MockComputeClient()).client
|
||||||
|
|
||||||
|
port_id = mock.sentinel.port_id
|
||||||
|
pod = mock.sentinel.pod
|
||||||
|
vif = mock.Mock()
|
||||||
|
vif.id = port_id
|
||||||
|
|
||||||
|
vm_id = mock.sentinel.vm_id
|
||||||
|
vm_port = mock.MagicMock()
|
||||||
|
vm_port.__getitem__.return_value = vm_id
|
||||||
|
|
||||||
|
m_driver._get_parent_port.return_value = vm_port
|
||||||
|
|
||||||
|
cls.release_vif(m_driver, pod, vif)
|
||||||
|
|
||||||
|
m_driver._get_parent_port.assert_called_once_with(neutron, pod)
|
||||||
|
compute.delete_server_interface.assert_called_once_with(
|
||||||
|
vif.id, server=vm_id)
|
||||||
|
|
||||||
|
def test_release_parent_not_found(self):
|
||||||
|
cls = nested_dpdk_vif.NestedDpdkPodVIFDriver
|
||||||
|
m_driver = mock.Mock(spec=cls)
|
||||||
|
neutron = self.useFixture(k_fix.MockNeutronClient()).client
|
||||||
|
compute = self.useFixture(k_fix.MockComputeClient()).client
|
||||||
|
|
||||||
|
pod = mock.sentinel.pod
|
||||||
|
vif = mock.Mock()
|
||||||
|
vif.id = mock.sentinel.vif_id
|
||||||
|
|
||||||
|
vm_id = mock.sentinel.parent_port_id
|
||||||
|
parent_port = mock.MagicMock()
|
||||||
|
parent_port.__getitem__.return_value = vm_id
|
||||||
|
|
||||||
|
m_driver._get_parent_port.side_effect = \
|
||||||
|
ntron_exc.NeutronClientException
|
||||||
|
|
||||||
|
self.assertRaises(ntron_exc.NeutronClientException, cls.release_vif,
|
||||||
|
m_driver, pod, vif)
|
||||||
|
|
||||||
|
m_driver._get_parent_port.assert_called_once_with(neutron, pod)
|
||||||
|
compute.delete_server_interface.assert_not_called()
|
||||||
|
|
||||||
|
def test_release_detach_failed(self):
|
||||||
|
cls = nested_dpdk_vif.NestedDpdkPodVIFDriver
|
||||||
|
m_driver = mock.Mock(spec=cls)
|
||||||
|
neutron = self.useFixture(k_fix.MockNeutronClient()).client
|
||||||
|
compute = self.useFixture(k_fix.MockComputeClient()).client
|
||||||
|
|
||||||
|
pod = mock.sentinel.pod
|
||||||
|
vif = mock.Mock()
|
||||||
|
vif.id = mock.sentinel.vif_id
|
||||||
|
|
||||||
|
vm_id = mock.sentinel.parent_port_id
|
||||||
|
parent_port = mock.MagicMock()
|
||||||
|
parent_port.__getitem__.return_value = vm_id
|
||||||
|
|
||||||
|
compute.delete_server_interface.side_effect = o_exc.SDKException
|
||||||
|
|
||||||
|
m_driver._get_parent_port.return_value = parent_port
|
||||||
|
|
||||||
|
self.assertRaises(o_exc.SDKException, cls.release_vif,
|
||||||
|
m_driver, pod, vif)
|
||||||
|
|
||||||
|
m_driver._get_parent_port.assert_called_once_with(neutron, pod)
|
||||||
|
compute.delete_server_interface.assert_called_once_with(
|
||||||
|
vif.id, server=vm_id)
|
||||||
|
|
||||||
|
@ddt.data((False), (True))
|
||||||
|
def test_activate_vif(self, active_value):
|
||||||
|
cls = nested_dpdk_vif.NestedDpdkPodVIFDriver
|
||||||
|
m_driver = mock.Mock(spec=cls)
|
||||||
|
pod = mock.sentinel.pod
|
||||||
|
vif = mock.Mock()
|
||||||
|
vif.active = active_value
|
||||||
|
|
||||||
|
cls.activate_vif(m_driver, pod, vif)
|
||||||
|
|
||||||
|
self.assertEqual(vif.active, True)
|
|
@ -49,3 +49,11 @@ class MockNetworkClient(fixtures.Fixture):
|
||||||
self.useFixture(fixtures.MockPatch(
|
self.useFixture(fixtures.MockPatch(
|
||||||
'kuryr_kubernetes.clients.get_network_client',
|
'kuryr_kubernetes.clients.get_network_client',
|
||||||
lambda: self.client))
|
lambda: self.client))
|
||||||
|
|
||||||
|
|
||||||
|
class MockComputeClient(fixtures.Fixture):
|
||||||
|
def _setUp(self):
|
||||||
|
self.client = mock.Mock()
|
||||||
|
self.useFixture(fixtures.MockPatch(
|
||||||
|
'kuryr_kubernetes.clients.get_compute_client',
|
||||||
|
lambda: self.client))
|
||||||
|
|
|
@ -32,6 +32,7 @@ class TestK8sClient(test_base.TestCase):
|
||||||
openstacksdk_mock = mock.Mock()
|
openstacksdk_mock = mock.Mock()
|
||||||
openstacksdk_mock.load_balancer = mock.Mock()
|
openstacksdk_mock.load_balancer = mock.Mock()
|
||||||
openstacksdk_mock.network = mock.Mock()
|
openstacksdk_mock.network = mock.Mock()
|
||||||
|
openstacksdk_mock.compute = mock.Mock()
|
||||||
k8s_dummy = object()
|
k8s_dummy = object()
|
||||||
|
|
||||||
m_cfg.kubernetes.api_root = k8s_api_root
|
m_cfg.kubernetes.api_root = k8s_api_root
|
||||||
|
@ -49,3 +50,5 @@ class TestK8sClient(test_base.TestCase):
|
||||||
clients.get_loadbalancer_client())
|
clients.get_loadbalancer_client())
|
||||||
self.assertIs(openstacksdk_mock.network,
|
self.assertIs(openstacksdk_mock.network,
|
||||||
clients.get_network_client())
|
clients.get_network_client())
|
||||||
|
self.assertIs(openstacksdk_mock.compute,
|
||||||
|
clients.get_compute_client())
|
||||||
|
|
|
@ -85,5 +85,9 @@ class TestNoOpPlugin(base.TestCase):
|
||||||
vif_object_name=k_vif.VIFMacvlanNested.__name__,
|
vif_object_name=k_vif.VIFMacvlanNested.__name__,
|
||||||
min_version="1.0",
|
min_version="1.0",
|
||||||
max_version="1.0"),
|
max_version="1.0"),
|
||||||
|
objects.host_info.HostVIFInfo(
|
||||||
|
vif_object_name=k_vif.VIFDPDKNested.__name__,
|
||||||
|
min_version="1.0",
|
||||||
|
max_version="1.0"),
|
||||||
])
|
])
|
||||||
self.assertEqual(expected, result)
|
self.assertEqual(expected, result)
|
||||||
|
|
|
@ -344,6 +344,62 @@ class TestOSVIFUtils(test_base.TestCase):
|
||||||
plugin=vif_plugin,
|
plugin=vif_plugin,
|
||||||
vif_name=vif_name)
|
vif_name=vif_name)
|
||||||
|
|
||||||
|
@mock.patch('kuryr_kubernetes.os_vif_util._get_vif_name')
|
||||||
|
@mock.patch('kuryr_kubernetes.os_vif_util._is_port_active')
|
||||||
|
@mock.patch('kuryr_kubernetes.os_vif_util._make_vif_network')
|
||||||
|
@mock.patch('kuryr_kubernetes.objects.vif.VIFDPDKNested')
|
||||||
|
@mock.patch('os_vif.objects.vif.VIFPortProfileK8sDPDK')
|
||||||
|
def test_neutron_to_osvif_nested_dpdk(self, m_mk_port_profile, m_mk_vif,
|
||||||
|
m_make_vif_network,
|
||||||
|
m_is_port_active, m_get_vif_name):
|
||||||
|
vif_plugin = const.K8S_OS_VIF_NOOP_PLUGIN
|
||||||
|
port_id = mock.sentinel.port_id
|
||||||
|
mac_address = mock.sentinel.mac_address
|
||||||
|
port_filter = mock.sentinel.port_filter
|
||||||
|
subnets = mock.sentinel.subnets
|
||||||
|
network = mock.sentinel.network
|
||||||
|
port_active = mock.sentinel.port_active
|
||||||
|
vif_name = mock.sentinel.vif_name
|
||||||
|
vif = mock.sentinel.vif
|
||||||
|
port_profile = mock.sentinel.port_profile
|
||||||
|
|
||||||
|
m_make_vif_network.return_value = network
|
||||||
|
m_is_port_active.return_value = port_active
|
||||||
|
m_get_vif_name.return_value = vif_name
|
||||||
|
m_mk_vif.return_value = vif
|
||||||
|
m_mk_port_profile.return_value = port_profile
|
||||||
|
|
||||||
|
pod = mock.MagicMock()
|
||||||
|
|
||||||
|
port = {'id': port_id,
|
||||||
|
'mac_address': mac_address,
|
||||||
|
'binding:vif_details': {
|
||||||
|
'port_filter': port_filter},
|
||||||
|
}
|
||||||
|
|
||||||
|
self.assertEqual(vif, ovu.neutron_to_osvif_vif_dpdk(port,
|
||||||
|
subnets, pod))
|
||||||
|
|
||||||
|
m_make_vif_network.assert_called_once_with(port, subnets)
|
||||||
|
m_is_port_active.assert_called_once_with(port)
|
||||||
|
m_get_vif_name.assert_called_once_with(port)
|
||||||
|
m_mk_port_profile.assert_called_once_with(
|
||||||
|
l3_setup=False,
|
||||||
|
selflink=pod['metadata']['selfLink'])
|
||||||
|
|
||||||
|
m_mk_vif.assert_called_once_with(
|
||||||
|
id=port_id,
|
||||||
|
port_profile=port_profile,
|
||||||
|
address=mac_address,
|
||||||
|
network=network,
|
||||||
|
has_traffic_filtering=port_filter,
|
||||||
|
preserve_on_delete=False,
|
||||||
|
active=port_active,
|
||||||
|
plugin=vif_plugin,
|
||||||
|
pci_address="",
|
||||||
|
dev_driver="",
|
||||||
|
vif_name=vif_name)
|
||||||
|
|
||||||
def test_neutron_to_osvif_vif_ovs_no_bridge(self):
|
def test_neutron_to_osvif_vif_ovs_no_bridge(self):
|
||||||
vif_plugin = 'ovs'
|
vif_plugin = 'ovs'
|
||||||
port = fake.get_port_obj(port_id=str(uuid.uuid4()))
|
port = fake.get_port_obj(port_id=str(uuid.uuid4()))
|
||||||
|
|
|
@ -36,7 +36,8 @@ LOG = log.getLogger(__name__)
|
||||||
VALID_MULTI_POD_POOLS_OPTS = {'noop': ['neutron-vif',
|
VALID_MULTI_POD_POOLS_OPTS = {'noop': ['neutron-vif',
|
||||||
'nested-vlan',
|
'nested-vlan',
|
||||||
'nested-macvlan',
|
'nested-macvlan',
|
||||||
'sriov'],
|
'sriov',
|
||||||
|
'nested-dpdk'],
|
||||||
'neutron': ['neutron-vif'],
|
'neutron': ['neutron-vif'],
|
||||||
'nested': ['nested-vlan'],
|
'nested': ['nested-vlan'],
|
||||||
}
|
}
|
||||||
|
|
|
@ -60,7 +60,7 @@ openstacksdk==0.36.0
|
||||||
os-client-config==1.29.0
|
os-client-config==1.29.0
|
||||||
os-service-types==1.7.0
|
os-service-types==1.7.0
|
||||||
os-testr==1.0.0
|
os-testr==1.0.0
|
||||||
os-vif==1.7.0
|
os-vif==1.12.0
|
||||||
osc-lib==1.10.0
|
osc-lib==1.10.0
|
||||||
oslo.cache==1.26.0
|
oslo.cache==1.26.0
|
||||||
oslo.concurrency==3.26.0
|
oslo.concurrency==3.26.0
|
||||||
|
|
|
@ -17,7 +17,7 @@ oslo.reports>=1.18.0 # Apache-2.0
|
||||||
oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0
|
oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0
|
||||||
oslo.service!=1.28.1,>=1.24.0 # Apache-2.0
|
oslo.service!=1.28.1,>=1.24.0 # Apache-2.0
|
||||||
oslo.utils>=3.33.0 # Apache-2.0
|
oslo.utils>=3.33.0 # Apache-2.0
|
||||||
os-vif!=1.8.0,>=1.7.0 # Apache-2.0
|
os-vif>=1.12.0 # Apache-2.0
|
||||||
PrettyTable<0.8,>=0.7.2 # BSD
|
PrettyTable<0.8,>=0.7.2 # BSD
|
||||||
pyroute2>=0.5.7;sys_platform!='win32' # Apache-2.0 (+ dual licensed GPL2)
|
pyroute2>=0.5.7;sys_platform!='win32' # Apache-2.0 (+ dual licensed GPL2)
|
||||||
retrying!=1.3.0,>=1.2.3 # Apache-2.0
|
retrying!=1.3.0,>=1.2.3 # Apache-2.0
|
||||||
|
|
|
@ -39,6 +39,7 @@ kuryr_kubernetes.vif_translators =
|
||||||
kuryr_kubernetes.cni.binding =
|
kuryr_kubernetes.cni.binding =
|
||||||
VIFBridge = kuryr_kubernetes.cni.binding.bridge:BridgeDriver
|
VIFBridge = kuryr_kubernetes.cni.binding.bridge:BridgeDriver
|
||||||
VIFOpenVSwitch = kuryr_kubernetes.cni.binding.bridge:VIFOpenVSwitchDriver
|
VIFOpenVSwitch = kuryr_kubernetes.cni.binding.bridge:VIFOpenVSwitchDriver
|
||||||
|
VIFDPDKNested = kuryr_kubernetes.cni.binding.dpdk:DpdkDriver
|
||||||
VIFVlanNested = kuryr_kubernetes.cni.binding.nested:VlanDriver
|
VIFVlanNested = kuryr_kubernetes.cni.binding.nested:VlanDriver
|
||||||
VIFMacvlanNested = kuryr_kubernetes.cni.binding.nested:MacvlanDriver
|
VIFMacvlanNested = kuryr_kubernetes.cni.binding.nested:MacvlanDriver
|
||||||
VIFSriov = kuryr_kubernetes.cni.binding.sriov:VIFSriovDriver
|
VIFSriov = kuryr_kubernetes.cni.binding.sriov:VIFSriovDriver
|
||||||
|
@ -80,6 +81,7 @@ kuryr_kubernetes.controller.drivers.pod_vif =
|
||||||
nested-vlan = kuryr_kubernetes.controller.drivers.nested_vlan_vif:NestedVlanPodVIFDriver
|
nested-vlan = kuryr_kubernetes.controller.drivers.nested_vlan_vif:NestedVlanPodVIFDriver
|
||||||
nested-macvlan = kuryr_kubernetes.controller.drivers.nested_macvlan_vif:NestedMacvlanPodVIFDriver
|
nested-macvlan = kuryr_kubernetes.controller.drivers.nested_macvlan_vif:NestedMacvlanPodVIFDriver
|
||||||
sriov = kuryr_kubernetes.controller.drivers.sriov:SriovVIFDriver
|
sriov = kuryr_kubernetes.controller.drivers.sriov:SriovVIFDriver
|
||||||
|
nested-dpdk = kuryr_kubernetes.controller.drivers.nested_dpdk_vif:NestedDpdkPodVIFDriver
|
||||||
|
|
||||||
kuryr_kubernetes.controller.drivers.endpoints_lbaas =
|
kuryr_kubernetes.controller.drivers.endpoints_lbaas =
|
||||||
lbaasv2 = kuryr_kubernetes.controller.drivers.lbaasv2:LBaaSv2Driver
|
lbaasv2 = kuryr_kubernetes.controller.drivers.lbaasv2:LBaaSv2Driver
|
||||||
|
|
Loading…
Reference in New Issue