diff --git a/README.rst b/README.rst index 38ccfbdbb..ff82944e9 100644 --- a/README.rst +++ b/README.rst @@ -55,6 +55,45 @@ vif binding executables. For example, if you installed it on Debian or Ubuntu:: bindir = /usr/local/libexec/kuryr +How to try out nested-pods locally: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +1. To install OpenStack services run devstack with ``devstack/local.conf.pod-in-vm.undercloud.sample``. + Ensure that "trunk" service plugin is enabled in ``/etc/neutron/neutron.conf``:: + + [DEFAULT] + service_plugins = neutron.services.l3_router.l3_router_plugin.L3RouterPlugin,neutron.services.trunk.plugin.TrunkPlugin + +2. Launch a VM with `Neutron trunk port. `_ +3. Inside VM, install and setup Kubernetes along with Kuryr using devstack: + - Since undercloud Neutron will be used by pods, neutron services should be + disabled in localrc. + - git clone kuryr-kubernetes at ``/opt/stack/``. + - In the ``devstack/plugin.sh``, comment out `configure_neutron_defaults `_. + This method is getting UUID of default Neutron resources project, pod_subnet etc. using local neutron client + and setting those values in ``/etc/kuryr/kuryr.conf``. + This will not work at the moment because Neutron is running remotely. Thats why this is being commented out + and manually these variables will be configured in ``/etc/kuryr/kuryr.conf`` + - Run devstack with ``devstack/local.conf.pod-in-vm.overcloud.sample``. +4. Once devstack is done and all services are up inside VM: + - Configure ``/etc/kuryr/kuryr.conf`` to set UUID of Neutron resources from undercloud Neutron:: + + [neutron_defaults] + ovs_bridge = br-int + pod_security_groups = + pod_subnet = + project = + worker_nodes_subnet = + + - Configure “pod_vif_driver” as “nested-vlan”:: + + [kubernetes] + pod_vif_driver = nested-vlan + + - Restart kuryr-k8s-controller from within devstack screen. + +Now launch pods using kubectl, Undercloud Neutron will serve the networking. + Features -------- diff --git a/devstack/local.conf.pod-in-vm.overcloud.sample b/devstack/local.conf.pod-in-vm.overcloud.sample new file mode 100644 index 000000000..0a4266527 --- /dev/null +++ b/devstack/local.conf.pod-in-vm.overcloud.sample @@ -0,0 +1,28 @@ +[[local|localrc]] + +RECLONE="no" + +enable_plugin kuryr-kubernetes \ + https://git.openstack.org/openstack/kuryr-kubernetes + +OFFLINE="no" +LOGFILE=devstack.log +LOG_COLOR=False +ADMIN_PASSWORD=pass +DATABASE_PASSWORD=pass +RABBIT_PASSWORD=pass +SERVICE_PASSWORD=pass +SERVICE_TOKEN=pass +IDENTITY_API_VERSION=3 +ENABLED_SERVICES="" + +enable_service key +enable_service mysql + +enable_service docker +enable_service etcd +enable_service kubernetes-api +enable_service kubernetes-controller-manager +enable_service kubernetes-scheduler +enable_service kubelet +enable_service kuryr-kubernetes diff --git a/devstack/local.conf.pod-in-vm.undercloud.sample b/devstack/local.conf.pod-in-vm.undercloud.sample new file mode 100644 index 000000000..33beb9691 --- /dev/null +++ b/devstack/local.conf.pod-in-vm.undercloud.sample @@ -0,0 +1,32 @@ +[[local|localrc]] + +# If you do not want stacking to clone new versions of the enabled services, +# like for example when you did local modifications and need to ./unstack.sh +# and ./stack.sh again, uncomment the following +# RECLONE="no" + +# Log settings for better readability +LOGFILE=devstack.log +LOG_COLOR=False +# If you want the screen tabs logged in a specific location, you can use: +# SCREEN_LOGDIR="${HOME}/devstack_logs" + +# Credentials +ADMIN_PASSWORD=pass +DATABASE_PASSWORD=pass +RABBIT_PASSWORD=pass +SERVICE_PASSWORD=pass +SERVICE_TOKEN=pass +TUNNEL_TYPE=vxlan +# Enable Keystone v3 +IDENTITY_API_VERSION=3 + +# LBaaSv2 service and Haproxy agent +enable_plugin neutron-lbaas \ + git://git.openstack.org/openstack/neutron-lbaas +enable_service q-lbaasv2 +NEUTRON_LBAAS_SERVICE_PROVIDERV2="LOADBALANCERV2:Haproxy:neutron_lbaas.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default" + +[[post-config|/$Q_PLUGIN_CONF_FILE]] +[securitygroup] +firewall_driver = openvswitch diff --git a/kuryr_kubernetes/cni/binding/nested.py b/kuryr_kubernetes/cni/binding/nested.py new file mode 100644 index 000000000..e27517520 --- /dev/null +++ b/kuryr_kubernetes/cni/binding/nested.py @@ -0,0 +1,52 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# from kuryr.lib import constants +# from kuryr.lib import utils +from kuryr_kubernetes.cni.binding import base as b_base +from kuryr_kubernetes import config + + +class VlanDriver(object): + def connect(self, vif, ifname, netns): + h_ipdb = b_base.get_ipdb() + c_ipdb = b_base.get_ipdb(netns) + + # NOTE(vikasc): Ideally 'ifname' should be used here but instead a + # temporary name is being used while creating the device for container + # in host network namespace. This is because cni expects only 'eth0' + # as interface name and if host already has an interface named 'eth0', + # device creation will fail with 'already exists' error. + temp_name = vif.vif_name + + # TODO(vikasc): evaluate whether we should have stevedore + # driver for getting the link device. + vm_iface_name = config.CONF.binding.link_iface + vlan_id = vif.vlan_id + + with h_ipdb.create(ifname=temp_name, + link=h_ipdb.interfaces[vm_iface_name], + kind='vlan', vlan_id=vlan_id) as iface: + iface.net_ns_fd = netns + + with c_ipdb.interfaces[temp_name] as iface: + iface.ifname = ifname + iface.mtu = vif.network.mtu + iface.address = str(vif.address) + iface.up() + + def disconnect(self, vif, ifname, netns): + # NOTE(vikasc): device will get deleted with container namespace, so + # nothing to be done here. + pass diff --git a/kuryr_kubernetes/cni/main.py b/kuryr_kubernetes/cni/main.py index c8ee362db..3188600d4 100644 --- a/kuryr_kubernetes/cni/main.py +++ b/kuryr_kubernetes/cni/main.py @@ -25,6 +25,7 @@ from kuryr_kubernetes.cni import api as cni_api from kuryr_kubernetes.cni import handlers as h_cni from kuryr_kubernetes import config from kuryr_kubernetes import constants as k_const +from kuryr_kubernetes import objects from kuryr_kubernetes import watcher as k_watcher LOG = logging.getLogger(__name__) @@ -75,6 +76,10 @@ def run(): # REVISIT(ivc): current CNI implementation provided by this package is # experimental and its primary purpose is to enable development of other # components (e.g. functional tests, service/LBaaSv2 support) + + # TODO(vikasc): Should be done using dynamically loadable OVO types plugin. + objects.register_locally_defined_vifs() + runner = cni_api.CNIRunner(K8sCNIPlugin()) def _timeout(signum, frame): diff --git a/kuryr_kubernetes/config.py b/kuryr_kubernetes/config.py index 95ca823e8..92de72435 100644 --- a/kuryr_kubernetes/config.py +++ b/kuryr_kubernetes/config.py @@ -56,9 +56,12 @@ neutron_defaults = [ help=_("Default Neutron security groups' IDs for Kubernetes pods")), cfg.StrOpt('ovs_bridge', help=_("Default OpenVSwitch integration bridge"), - sample_default="br-int") + sample_default="br-int"), + cfg.StrOpt('worker_nodes_subnet', + help=_("Neutron subnet ID for k8s worker node vms.")), ] + CONF = cfg.CONF CONF.register_opts(kuryr_k8s_opts) CONF.register_opts(k8s_opts, group='kubernetes') diff --git a/kuryr_kubernetes/constants.py b/kuryr_kubernetes/constants.py index 8d6548dff..75e82387c 100644 --- a/kuryr_kubernetes/constants.py +++ b/kuryr_kubernetes/constants.py @@ -26,5 +26,7 @@ K8S_POD_STATUS_PENDING = 'Pending' K8S_ANNOTATION_PREFIX = 'openstack.org/kuryr' K8S_ANNOTATION_VIF = K8S_ANNOTATION_PREFIX + '-vif' +K8S_OS_VIF_NOOP_PLUGIN = "noop" + CNI_EXCEPTION_CODE = 100 CNI_TIMEOUT_CODE = 200 diff --git a/kuryr_kubernetes/controller/drivers/nested_vlan_vif.py b/kuryr_kubernetes/controller/drivers/nested_vlan_vif.py new file mode 100644 index 000000000..c4b18ffda --- /dev/null +++ b/kuryr_kubernetes/controller/drivers/nested_vlan_vif.py @@ -0,0 +1,190 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from time import sleep + +from kuryr.lib._i18n import _LE +from kuryr.lib import constants as kl_const +from kuryr.lib import segmentation_type_drivers as seg_driver +from neutronclient.common import exceptions as n_exc +from oslo_config import cfg as oslo_cfg +from oslo_log import log as logging + +from kuryr_kubernetes import clients +from kuryr_kubernetes import config +from kuryr_kubernetes import constants as const +from kuryr_kubernetes.controller.drivers import generic_vif +from kuryr_kubernetes import exceptions as k_exc +from kuryr_kubernetes import os_vif_util as ovu + + +LOG = logging.getLogger(__name__) + +DEFAULT_MAX_RETRY_COUNT = 3 +DEFAULT_RETRY_INTERVAL = 1 + + +class NestedVlanPodVIFDriver(generic_vif.GenericPodVIFDriver): + """Manages ports for nested-containers to provide VIFs.""" + + def request_vif(self, pod, project_id, subnets, security_groups): + neutron = clients.get_neutron_client() + parent_port = self._get_parent_port(neutron, pod) + trunk_id = self._get_trunk_id(parent_port) + + rq = self._get_port_request(pod, project_id, subnets, security_groups) + port = neutron.create_port(rq).get('port') + + vlan_id = self._add_subport(neutron, trunk_id, port['id']) + + vif_plugin = const.K8S_OS_VIF_NOOP_PLUGIN + vif = ovu.neutron_to_osvif_vif(vif_plugin, port, subnets) + vif.vlan_id = vlan_id + return vif + + def release_vif(self, pod, vif): + neutron = clients.get_neutron_client() + parent_port = self._get_parent_port(neutron, pod) + trunk_id = self._get_trunk_id(parent_port) + self._remove_subport(neutron, trunk_id, vif.id) + self._release_vlan_id(vif.vlan_id) + try: + neutron.delete_port(vif.id) + except n_exc.PortNotFoundClient: + LOG.debug('Unable to release port %s as it no longer exists.', + vif.id) + + def _get_port_request(self, pod, project_id, subnets, security_groups): + port_req_body = {'project_id': project_id, + 'name': self._get_port_name(pod), + 'network_id': self._get_network_id(subnets), + 'fixed_ips': ovu.osvif_to_neutron_fixed_ips(subnets), + 'device_owner': kl_const.DEVICE_OWNER, + 'admin_state_up': True} + + if security_groups: + port_req_body['security_groups'] = security_groups + + return {'port': port_req_body} + + def _get_trunk_id(self, port): + try: + return port['trunk_details']['trunk_id'] + except KeyError: + LOG.error(_LE("Neutron port is missing trunk details. " + "Please ensure that k8s node port is associated " + "with a Neutron vlan trunk")) + raise k_exc.K8sNodeTrunkPortFailure + + def _get_parent_port(self, neutron, pod): + node_subnet_id = config.CONF.neutron_defaults.worker_nodes_subnet + if not node_subnet_id: + raise oslo_cfg.RequiredOptError('worker_nodes_subnet', + 'neutron_defaults') + + try: + # REVISIT(vikasc): Assumption is being made that hostIP is the IP + # of trunk interface on the node(vm). + node_fixed_ip = pod['status']['hostIP'] + except KeyError: + if pod['status']['conditions'][0]['type'] != "Initialized": + LOG.debug("Pod condition type is not 'Initialized'") + + LOG.error(_LE("Failed to get parent vm port ip")) + raise + + try: + fixed_ips = ['subnet_id=%s' % str(node_subnet_id), + 'ip_address=%s' % str(node_fixed_ip)] + ports = neutron.list_ports(fixed_ips=fixed_ips) + except n_exc.NeutronClientException as ex: + LOG.error(_LE("Parent vm port with fixed ips %s not found!"), + fixed_ips) + raise ex + + if ports['ports']: + return ports['ports'][0] + else: + LOG.error(_LE("Neutron port for vm port with fixed ips %s" + " not found!"), fixed_ips) + raise k_exc.K8sNodeTrunkPortFailure + + def _add_subport(self, neutron, trunk_id, subport): + """Adds subport port to Neutron trunk + + This method gets vlanid allocated from kuryr segmentation driver. + In active/active HA type deployment, possibility of vlanid conflict + is there. In such a case, vlanid will be requested again and subport + addition is re-tried. This is tried DEFAULT_MAX_RETRY_COUNT times in + case of vlanid conflict. + """ + # TODO(vikasc): Better approach for retrying in case of + # vlan-id conflict. + retry_count = 1 + while True: + try: + vlan_id = self._get_vlan_id(trunk_id) + except n_exc.NeutronClientException as ex: + LOG.error(_LE("Getting VlanID for subport on " + "trunk %s failed!!"), trunk_id) + raise ex + subport = [{'segmentation_id': vlan_id, + 'port_id': subport, + 'segmentation_type': 'vlan'}] + try: + neutron.trunk_add_subports(trunk_id, + {'sub_ports': subport}) + except n_exc.Conflict as ex: + if retry_count < DEFAULT_MAX_RETRY_COUNT: + LOG.error(_LE("vlanid already in use on trunk, " + "%s. Retrying..."), trunk_id) + retry_count += 1 + sleep(DEFAULT_RETRY_INTERVAL) + continue + else: + LOG.error(_LE( + "MAX retry count reached. Failed to add subport")) + raise ex + + except n_exc.NeutronClientException as ex: + LOG.error(_LE("Error happened during subport" + "addition to trunk, %s"), trunk_id) + raise ex + return vlan_id + + def _remove_subport(self, neutron, trunk_id, subport_id): + subport_id = [{'port_id': subport_id}] + try: + neutron.trunk_remove_subports(trunk_id, + {'sub_ports': subport_id}) + except n_exc.NeutronClientException as ex: + LOG.error(_LE( + "Error happened during subport removal from trunk," + "%s"), trunk_id) + raise ex + + def _get_vlan_id(self, trunk_id): + vlan_ids = self._get_in_use_vlan_ids_set(trunk_id) + return seg_driver.allocate_segmentation_id(vlan_ids) + + def _release_vlan_id(self, id): + return seg_driver.release_segmentation_id(id) + + def _get_in_use_vlan_ids_set(self, trunk_id): + vlan_ids = set() + neutron = clients.get_neutron_client() + trunk = neutron.show_trunk(trunk_id) + for port in trunk['trunk']['sub_ports']: + vlan_ids.add(port['segmentation_id']) + + return vlan_ids diff --git a/kuryr_kubernetes/controller/service.py b/kuryr_kubernetes/controller/service.py index 6632009ad..4ec5c047d 100644 --- a/kuryr_kubernetes/controller/service.py +++ b/kuryr_kubernetes/controller/service.py @@ -25,6 +25,7 @@ from kuryr_kubernetes import config from kuryr_kubernetes import constants from kuryr_kubernetes.controller.handlers import pipeline as h_pipeline from kuryr_kubernetes.controller.handlers import vif as h_vif +from kuryr_kubernetes import objects from kuryr_kubernetes import watcher LOG = logging.getLogger(__name__) @@ -36,6 +37,7 @@ class KuryrK8sService(service.Service): def __init__(self): super(KuryrK8sService, self).__init__() + objects.register_locally_defined_vifs() pipeline = h_pipeline.ControllerPipeline(self.tg) self.watcher = watcher.Watcher(pipeline, self.tg) # TODO(ivc): pluggable resource/handler registration diff --git a/kuryr_kubernetes/exceptions.py b/kuryr_kubernetes/exceptions.py index aadb85bdf..bb3494000 100644 --- a/kuryr_kubernetes/exceptions.py +++ b/kuryr_kubernetes/exceptions.py @@ -36,3 +36,12 @@ class CNIError(Exception): def format_msg(exception): return "%s: %s" % (exception.__class__.__name__, exception) + + +class K8sNodeTrunkPortFailure(Exception): + """Exception represents that error is related to K8s node trunk port + + This exception is thrown when Neutron port for k8s node could + not be found using subnet ID and IP address OR neutron port is + not associated to a Neutron vlan trunk. + """ diff --git a/kuryr_kubernetes/objects/__init__.py b/kuryr_kubernetes/objects/__init__.py new file mode 100644 index 000000000..6d97e444d --- /dev/null +++ b/kuryr_kubernetes/objects/__init__.py @@ -0,0 +1,2 @@ +def register_locally_defined_vifs(): + __import__('kuryr_kubernetes.objects.vif') diff --git a/kuryr_kubernetes/objects/vif.py b/kuryr_kubernetes/objects/vif.py new file mode 100644 index 000000000..fbbf4a4f1 --- /dev/null +++ b/kuryr_kubernetes/objects/vif.py @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_versionedobjects import base as obj_base +from oslo_versionedobjects import fields as obj_fields + +from os_vif.objects import vif as obj_osvif + + +@obj_base.VersionedObjectRegistry.register +class VIFVlanNested(obj_osvif.VIFBase): + # This is OVO based vlan vif. + + VERSION = '1.0' + + fields = { + # Name of the device to create + 'vif_name': obj_fields.StringField(), + # vlan ID allocated to this vif + 'vlan_id': obj_fields.IntegerField() + } diff --git a/kuryr_kubernetes/os_vif_plug_noop.py b/kuryr_kubernetes/os_vif_plug_noop.py new file mode 100644 index 000000000..aa902107b --- /dev/null +++ b/kuryr_kubernetes/os_vif_plug_noop.py @@ -0,0 +1,36 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from os_vif import objects +from os_vif.plugin import PluginBase + +from kuryr_kubernetes.objects import vif as k_vif + + +class NoOpPlugin(PluginBase): + """No Op Plugin to be used with VIF types that dont need plugging""" + + def describe(self): + return objects.host_info.HostPluginInfo( + plugin_name='noop', + vif_info=[ + objects.host_info.HostVIFInfo( + vif_object_name=k_vif.VIFVlanNested.__name__, + min_version="1.0", + max_version="1.0"), + ]) + + def plug(self, vif, instance_info): + pass + + def unplug(self, vif, instance_info): + pass diff --git a/kuryr_kubernetes/os_vif_util.py b/kuryr_kubernetes/os_vif_util.py index a6e4bc678..a1d7cf8e8 100644 --- a/kuryr_kubernetes/os_vif_util.py +++ b/kuryr_kubernetes/os_vif_util.py @@ -29,6 +29,7 @@ from stevedore import driver as stv_driver from kuryr_kubernetes import config from kuryr_kubernetes import exceptions as k_exc +from kuryr_kubernetes.objects import vif as k_vif LOG = logging.getLogger(__name__) @@ -254,6 +255,32 @@ def neutron_to_osvif_vif_ovs(vif_plugin, neutron_port, subnets): return vif +def neutron_to_osvif_vif_nested(vif_plugin, neutron_port, subnets): + """Converts Neutron port to VIF object for nested containers. + + :param vif_plugin: name of the os-vif plugin to use (i.e. 'noop') + :param neutron_port: dict containing port information as returned by + neutron client's 'show_port' + :param subnets: subnet mapping as returned by PodSubnetsDriver.get_subnets + :return: kuryr-k8s native VIF object (eg. VIFVlanNested) + """ + + details = neutron_port.get('binding:vif_details', {}) + network = _make_vif_network(neutron_port, subnets) + + vif = k_vif.VIFVlanNested( + id=neutron_port['id'], + address=neutron_port['mac_address'], + network=network, + has_traffic_filtering=details.get('port_filter', False), + preserve_on_delete=False, + active=_is_port_active(neutron_port), + plugin=vif_plugin, + vif_name=_get_vif_name(neutron_port)) + + return vif + + def neutron_to_osvif_vif(vif_plugin, neutron_port, subnets): """Converts Neutron port to os-vif VIF object. diff --git a/kuryr_kubernetes/tests/base.py b/kuryr_kubernetes/tests/base.py index bc2d9c879..21a23a33d 100644 --- a/kuryr_kubernetes/tests/base.py +++ b/kuryr_kubernetes/tests/base.py @@ -9,10 +9,14 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - +from kuryr_kubernetes import config from oslotest import base class TestCase(base.BaseTestCase): """Test case base class for all unit tests.""" + def setUp(self): + super(TestCase, self).setUp() + args = [] + config.init(args=args) diff --git a/kuryr_kubernetes/tests/unit/controller/drivers/test_nested_vlan_vif.py b/kuryr_kubernetes/tests/unit/controller/drivers/test_nested_vlan_vif.py new file mode 100644 index 000000000..b3037689b --- /dev/null +++ b/kuryr_kubernetes/tests/unit/controller/drivers/test_nested_vlan_vif.py @@ -0,0 +1,347 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from kuryr.lib import constants as kl_const +from kuryr.lib import exceptions as kl_exc +from neutronclient.common import exceptions as n_exc +from oslo_config import cfg as oslo_cfg + +from kuryr_kubernetes import constants as const +from kuryr_kubernetes.controller.drivers import nested_vlan_vif +from kuryr_kubernetes import exceptions as k_exc +from kuryr_kubernetes.tests import base as test_base +from kuryr_kubernetes.tests.unit import kuryr_fixtures as k_fix + + +class TestNestedVlanPodVIFDriver(test_base.TestCase): + + @mock.patch('kuryr_kubernetes.os_vif_util.neutron_to_osvif_vif') + def test_request_vif(self, m_to_vif): + cls = nested_vlan_vif.NestedVlanPodVIFDriver + m_driver = mock.Mock(spec=cls) + neutron = self.useFixture(k_fix.MockNeutronClient()).client + + pod = mock.sentinel.pod + project_id = mock.sentinel.project_id + subnets = mock.sentinel.subnets + security_groups = mock.sentinel.security_groups + + parent_port = mock.sentinel.parent_port + trunk_id = mock.sentinel.trunk_id + port_id = mock.sentinel.port_id + port = mock.MagicMock() + port.__getitem__.return_value = port_id + port_request = mock.sentinel.port_request + vlan_id = mock.sentinel.vlan_id + + vif = mock.Mock() + vif_plugin = const.K8S_OS_VIF_NOOP_PLUGIN + + m_to_vif.return_value = vif + m_driver._get_parent_port.return_value = parent_port + m_driver._get_trunk_id.return_value = trunk_id + m_driver._get_port_request.return_value = port_request + m_driver._add_subport.return_value = vlan_id + neutron.list_ports.return_value = {'ports': [parent_port]} + neutron.create_port.return_value = {'port': port} + + self.assertEqual(vif, cls.request_vif(m_driver, pod, project_id, + subnets, security_groups)) + + m_driver._get_parent_port.assert_called_once_with(neutron, pod) + m_driver._get_trunk_id.assert_called_once_with(parent_port) + m_driver._get_port_request.assert_called_once_with( + pod, project_id, subnets, security_groups) + neutron.create_port.assert_called_once_with(port_request) + m_driver._add_subport.assert_called_once_with(neutron, + trunk_id, + port_id) + m_to_vif.assert_called_once_with(vif_plugin, port, subnets) + self.assertEqual(vif.vlan_id, vlan_id) + + def test_release_vif(self): + cls = nested_vlan_vif.NestedVlanPodVIFDriver + m_driver = mock.Mock(spec=cls) + neutron = self.useFixture(k_fix.MockNeutronClient()).client + parent_port = mock.sentinel.parent_port + trunk_id = mock.sentinel.trunk_id + + m_driver._get_parent_port.return_value = parent_port + m_driver._get_trunk_id.return_value = trunk_id + pod = mock.sentinel.pod + vif = mock.Mock() + + cls.release_vif(m_driver, pod, vif) + + m_driver._get_parent_port.assert_called_once_with(neutron, pod) + m_driver._get_trunk_id.assert_called_once_with(parent_port) + m_driver._remove_subport.assert_called_once_with( + neutron, trunk_id, vif.id) + neutron.delete_port.assert_called_once_with(vif.id) + + def test_release_vif_not_found(self): + cls = nested_vlan_vif.NestedVlanPodVIFDriver + m_driver = mock.Mock(spec=cls) + neutron = self.useFixture(k_fix.MockNeutronClient()).client + parent_port = mock.sentinel.parent_port + trunk_id = mock.sentinel.trunk_id + + m_driver._get_parent_port.return_value = parent_port + m_driver._get_trunk_id.return_value = trunk_id + pod = mock.sentinel.pod + vlan_id = mock.sentinel.vlan_id + vif = mock.Mock() + m_driver._port_vlan_mapping = {vif.id: vlan_id} + self.assertTrue(vif.id in m_driver._port_vlan_mapping.keys()) + neutron.delete_port.side_effect = n_exc.PortNotFoundClient + + cls.release_vif(m_driver, pod, vif) + + m_driver._get_parent_port.assert_called_once_with(neutron, pod) + m_driver._get_trunk_id.assert_called_once_with(parent_port) + m_driver._remove_subport.assert_called_once_with( + neutron, trunk_id, vif.id) + neutron.delete_port.assert_called_once_with(vif.id) + + def _test_get_port_request(self, m_to_fips, security_groups): + cls = nested_vlan_vif.NestedVlanPodVIFDriver + m_driver = mock.Mock(spec=cls) + + pod = mock.sentinel.pod + project_id = mock.sentinel.project_id + subnets = mock.sentinel.subnets + port_name = mock.sentinel.port_name + network_id = mock.sentinel.project_id + fixed_ips = mock.sentinel.fixed_ips + + m_driver._get_port_name.return_value = port_name + m_driver._get_network_id.return_value = network_id + m_to_fips.return_value = fixed_ips + + expected = {'port': {'project_id': project_id, + 'name': port_name, + 'network_id': network_id, + 'fixed_ips': fixed_ips, + 'device_owner': kl_const.DEVICE_OWNER, + 'admin_state_up': True}} + + if security_groups: + expected['port']['security_groups'] = security_groups + + ret = cls._get_port_request(m_driver, pod, project_id, subnets, + security_groups) + + self.assertEqual(expected, ret) + m_driver._get_port_name.assert_called_once_with(pod) + m_driver._get_network_id.assert_called_once_with(subnets) + m_to_fips.assert_called_once_with(subnets) + + @mock.patch('kuryr_kubernetes.os_vif_util.osvif_to_neutron_fixed_ips') + def test_get_port_request(self, m_to_fips): + security_groups = mock.sentinel.security_groups + self._test_get_port_request(m_to_fips, security_groups) + + @mock.patch('kuryr_kubernetes.os_vif_util.osvif_to_neutron_fixed_ips') + def test_get_port_request_no_sg(self, m_to_fips): + security_groups = [] + self._test_get_port_request(m_to_fips, security_groups) + + def test_get_trunk_id(self): + cls = nested_vlan_vif.NestedVlanPodVIFDriver + m_driver = mock.Mock(spec=cls) + trunk_id = mock.sentinel.trunk_id + port = {'trunk_details': {'trunk_id': trunk_id}} + + self.assertEqual(trunk_id, cls._get_trunk_id(m_driver, port)) + + def test_get_trunk_id_details_missing(self): + cls = nested_vlan_vif.NestedVlanPodVIFDriver + m_driver = mock.Mock(spec=cls) + trunk_id = mock.sentinel.trunk_id + port = {'trunk_details_missing': {'trunk_id_missing': trunk_id}} + self.assertRaises(k_exc.K8sNodeTrunkPortFailure, + cls._get_trunk_id, m_driver, port) + + def test_get_parent_port(self): + cls = nested_vlan_vif.NestedVlanPodVIFDriver + m_driver = mock.Mock(spec=cls) + neutron = self.useFixture(k_fix.MockNeutronClient()).client + + node_subnet_id = mock.sentinel.node_subnet_id + nested_vlan_vif.config.CONF.neutron_defaults.worker_nodes_subnet =\ + node_subnet_id + + node_fixed_ip = mock.sentinel.node_fixed_ip + pod_status = mock.MagicMock() + pod_status.__getitem__.return_value = node_fixed_ip + + pod = mock.MagicMock() + pod.__getitem__.return_value = pod_status + + port = mock.sentinel.port + ports = {'ports': [port]} + neutron.list_ports.return_value = ports + + self.assertEqual(port, cls._get_parent_port(m_driver, neutron, pod)) + fixed_ips = ['subnet_id=%s' % str(node_subnet_id), + 'ip_address=%s' % str(node_fixed_ip)] + neutron.list_ports.assert_called_once_with(fixed_ips=fixed_ips) + + def test_get_parent_port_subnet_id_not_configured(self): + cls = nested_vlan_vif.NestedVlanPodVIFDriver + m_driver = mock.Mock(spec=cls) + neutron = self.useFixture(k_fix.MockNeutronClient()).client + nested_vlan_vif.config.CONF.neutron_defaults.worker_nodes_subnet = '' + pod = mock.MagicMock() + self.assertRaises(oslo_cfg.RequiredOptError, + cls._get_parent_port, m_driver, neutron, pod) + + def test_get_parent_port_trunk_not_found(self): + cls = nested_vlan_vif.NestedVlanPodVIFDriver + m_driver = mock.Mock(spec=cls) + neutron = self.useFixture(k_fix.MockNeutronClient()).client + + node_subnet_id = mock.sentinel.node_subnet_id + nested_vlan_vif.config.CONF.neutron_defaults.worker_nodes_subnet =\ + node_subnet_id + + node_fixed_ip = mock.sentinel.node_fixed_ip + pod_status = mock.MagicMock() + pod_status.__getitem__.return_value = node_fixed_ip + + pod = mock.MagicMock() + pod.__getitem__.return_value = pod_status + + ports = {'ports': []} + neutron.list_ports.return_value = ports + + self.assertRaises(k_exc.K8sNodeTrunkPortFailure, + cls._get_parent_port, m_driver, neutron, pod) + fixed_ips = ['subnet_id=%s' % str(node_subnet_id), + 'ip_address=%s' % str(node_fixed_ip)] + neutron.list_ports.assert_called_once_with(fixed_ips=fixed_ips) + + def test_add_subport(self): + cls = nested_vlan_vif.NestedVlanPodVIFDriver + m_driver = mock.Mock(spec=cls) + neutron = self.useFixture(k_fix.MockNeutronClient()).client + trunk_id = mock.sentinel.trunk_id + subport = mock.sentinel.subport + vlan_id = mock.sentinel.vlan_id + m_driver._get_vlan_id.return_value = vlan_id + subport_dict = [{'segmentation_id': vlan_id, + 'port_id': subport, + 'segmentation_type': 'vlan'}] + nested_vlan_vif.DEFAULT_MAX_RETRY_COUNT = 1 + self.assertEqual(vlan_id, cls._add_subport(m_driver, + neutron, trunk_id, subport)) + m_driver._get_vlan_id.assert_called_once_with(trunk_id) + neutron.trunk_add_subports.assert_called_once_with(trunk_id, + {'sub_ports': subport_dict}) + + def test_add_subport_get_vlanid_failure(self): + cls = nested_vlan_vif.NestedVlanPodVIFDriver + m_driver = mock.Mock(spec=cls) + neutron = self.useFixture(k_fix.MockNeutronClient()).client + trunk_id = mock.sentinel.trunk_id + subport = mock.sentinel.subport + m_driver._get_vlan_id.side_effect = n_exc.NeutronClientException + nested_vlan_vif.DEFAULT_MAX_RETRY_COUNT = 1 + self.assertRaises(n_exc.NeutronClientException, + cls._add_subport, m_driver, neutron, trunk_id, subport) + + m_driver._get_vlan_id.assert_called_once_with(trunk_id) + + def test_add_subport_with_vlan_id_conflict(self): + cls = nested_vlan_vif.NestedVlanPodVIFDriver + m_driver = mock.Mock(spec=cls) + neutron = self.useFixture(k_fix.MockNeutronClient()).client + trunk_id = mock.sentinel.trunk_id + subport = mock.sentinel.subport + vlan_id = mock.sentinel.vlan_id + m_driver._get_vlan_id.return_value = vlan_id + subport_dict = [{'segmentation_id': vlan_id, + 'port_id': subport, + 'segmentation_type': 'vlan'}] + neutron.trunk_add_subports.side_effect = n_exc.Conflict + nested_vlan_vif.DEFAULT_MAX_RETRY_COUNT = 1 + self.assertRaises(n_exc.Conflict, cls._add_subport, m_driver, + neutron, trunk_id, subport) + + neutron.trunk_add_subports.assert_called_once_with(trunk_id, + {'sub_ports': subport_dict}) + + def test_remove_subport(self): + cls = nested_vlan_vif.NestedVlanPodVIFDriver + m_driver = mock.Mock(spec=cls) + neutron = self.useFixture(k_fix.MockNeutronClient()).client + trunk_id = mock.sentinel.trunk_id + subport_id = mock.sentinel.subport_id + subportid_dict = [{'port_id': subport_id}] + cls._remove_subport(m_driver, neutron, trunk_id, subport_id) + + neutron.trunk_remove_subports.assert_called_once_with(trunk_id, + {'sub_ports': subportid_dict}) + + @mock.patch('kuryr.lib.segmentation_type_drivers.allocate_segmentation_id') + def test_get_vlan_id(self, mock_alloc_seg_id): + cls = nested_vlan_vif.NestedVlanPodVIFDriver + m_driver = mock.Mock(spec=cls) + vlanid_set = mock.sentinel.vlanid_set + trunk_id = mock.sentinel.trunk_id + m_driver._get_in_use_vlan_ids_set.return_value = vlanid_set + cls._get_vlan_id(m_driver, trunk_id) + + mock_alloc_seg_id.assert_called_once_with(vlanid_set) + + @mock.patch('kuryr.lib.segmentation_type_drivers.allocate_segmentation_id') + def test_get_vlan_id_exhausted(self, mock_alloc_seg_id): + cls = nested_vlan_vif.NestedVlanPodVIFDriver + m_driver = mock.Mock(spec=cls) + vlanid_set = mock.sentinel.vlanid_set + trunk_id = mock.sentinel.trunk_id + m_driver._get_in_use_vlan_ids_set.return_value = vlanid_set + mock_alloc_seg_id.side_effect = kl_exc.SegmentationIdAllocationFailure + self.assertRaises(kl_exc.SegmentationIdAllocationFailure, + cls._get_vlan_id, m_driver, trunk_id) + + mock_alloc_seg_id.assert_called_once_with(vlanid_set) + + @mock.patch('kuryr.lib.segmentation_type_drivers.release_segmentation_id') + def test_release_vlan_id(self, mock_rel_seg_id): + cls = nested_vlan_vif.NestedVlanPodVIFDriver + m_driver = mock.Mock(spec=cls) + vlanid = mock.sentinel.vlanid + cls._release_vlan_id(m_driver, vlanid) + + mock_rel_seg_id.assert_called_once_with(vlanid) + + def test_get_in_use_vlan_ids_set(self): + cls = nested_vlan_vif.NestedVlanPodVIFDriver + m_driver = mock.Mock(spec=cls) + neutron = self.useFixture(k_fix.MockNeutronClient()).client + + vlan_ids = set() + trunk_id = mock.sentinel.trunk_id + vlan_ids.add('100') + + port = mock.MagicMock() + port.__getitem__.return_value = '100' + trunk_obj = mock.MagicMock() + trunk_obj.__getitem__.return_value = [port] + trunk = mock.MagicMock() + trunk.__getitem__.return_value = trunk_obj + neutron.show_trunk.return_value = trunk + self.assertEqual(vlan_ids, + cls._get_in_use_vlan_ids_set(m_driver, trunk_id)) diff --git a/kuryr_kubernetes/tests/unit/test_os_vif_plug_noop.py b/kuryr_kubernetes/tests/unit/test_os_vif_plug_noop.py new file mode 100644 index 000000000..9080762d6 --- /dev/null +++ b/kuryr_kubernetes/tests/unit/test_os_vif_plug_noop.py @@ -0,0 +1,85 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +from stevedore import extension + +import os_vif +from os_vif import objects + +from kuryr_kubernetes.objects import vif as k_vif +from kuryr_kubernetes.os_vif_plug_noop import NoOpPlugin +from kuryr_kubernetes.tests import base + + +class TestNoOpPlugin(base.TestCase): + + def setUp(self): + super(TestNoOpPlugin, self).setUp() + os_vif._EXT_MANAGER = None + + @mock.patch('stevedore.extension.ExtensionManager') + def test_initialize(self, mock_EM): + self.assertIsNone(os_vif._EXT_MANAGER) + os_vif.initialize() + mock_EM.assert_called_once_with( + invoke_on_load=False, namespace='os_vif') + self.assertIsNotNone(os_vif._EXT_MANAGER) + + @mock.patch.object(NoOpPlugin, "plug") + def test_plug(self, mock_plug): + plg = extension.Extension(name="noop", + entry_point="os-vif", + plugin=NoOpPlugin, + obj=None) + with mock.patch('stevedore.extension.ExtensionManager.names', + return_value=['foobar']),\ + mock.patch('stevedore.extension.ExtensionManager.__getitem__', + return_value=plg): + os_vif.initialize() + info = mock.sentinel.info + vif = mock.MagicMock() + vif.plugin_name = 'noop' + os_vif.plug(vif, info) + mock_plug.assert_called_once_with(vif, info) + + @mock.patch.object(NoOpPlugin, "unplug") + def test_unplug(self, mock_unplug): + plg = extension.Extension(name="demo", + entry_point="os-vif", + plugin=NoOpPlugin, + obj=None) + with mock.patch('stevedore.extension.ExtensionManager.names', + return_value=['foobar']),\ + mock.patch('stevedore.extension.ExtensionManager.__getitem__', + return_value=plg): + os_vif.initialize() + info = mock.sentinel.info + vif = mock.MagicMock() + vif.plugin_name = 'noop' + os_vif.unplug(vif, info) + mock_unplug.assert_called_once_with(vif, info) + + def test_describe_noop_plugin(self): + os_vif.initialize() + noop_plugin = NoOpPlugin.load('noop') + result = noop_plugin.describe() + + expected = objects.host_info.HostPluginInfo( + plugin_name='noop', + vif_info=[ + objects.host_info.HostVIFInfo( + vif_object_name=k_vif.VIFVlanNested.__name__, + min_version="1.0", + max_version="1.0"), + ]) + self.assertEqual(expected, result) diff --git a/kuryr_kubernetes/tests/unit/test_os_vif_util.py b/kuryr_kubernetes/tests/unit/test_os_vif_util.py index 96ecc7926..d685b408f 100644 --- a/kuryr_kubernetes/tests/unit/test_os_vif_util.py +++ b/kuryr_kubernetes/tests/unit/test_os_vif_util.py @@ -258,6 +258,49 @@ class TestOSVIFUtils(test_base.TestCase): m_get_vif_name.assert_called_once_with(port) self.assertEqual(ovs_bridge, network.bridge) + @mock.patch('kuryr_kubernetes.os_vif_util._get_vif_name') + @mock.patch('kuryr_kubernetes.os_vif_util._is_port_active') + @mock.patch('kuryr_kubernetes.os_vif_util._make_vif_network') + @mock.patch('kuryr_kubernetes.objects.vif.VIFVlanNested') + def test_neutron_to_osvif_nested(self, m_mk_vif, m_make_vif_network, + m_is_port_active, m_get_vif_name): + vif_plugin = 'noop' + port_id = mock.sentinel.port_id + mac_address = mock.sentinel.mac_address + port_filter = mock.sentinel.port_filter + subnets = mock.sentinel.subnets + network = mock.sentinel.network + port_active = mock.sentinel.port_active + vif_name = mock.sentinel.vif_name + vif = mock.sentinel.vif + + m_make_vif_network.return_value = network + m_is_port_active.return_value = port_active + m_get_vif_name.return_value = vif_name + m_mk_vif.return_value = vif + + port = {'id': port_id, + 'mac_address': mac_address, + 'binding:vif_details': { + 'port_filter': port_filter}, + } + + self.assertEqual(vif, ovu.neutron_to_osvif_vif_nested(vif_plugin, port, + subnets)) + + m_make_vif_network.assert_called_once_with(port, subnets) + m_is_port_active.assert_called_once_with(port) + m_get_vif_name.assert_called_once_with(port) + m_mk_vif.assert_called_once_with( + id=port_id, + address=mac_address, + network=network, + has_traffic_filtering=port_filter, + preserve_on_delete=False, + active=port_active, + plugin=vif_plugin, + vif_name=vif_name) + def test_neutron_to_osvif_vif_ovs_no_bridge(self): vif_plugin = 'ovs' port = {'id': uuidutils.generate_uuid()} diff --git a/setup.cfg b/setup.cfg index 0dd4ef3f5..dd1d251ee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,16 +23,21 @@ oslo.config.opts = kuryr_kubernetes = kuryr_kubernetes.opts:list_kuryr_opts kuryr_lib = kuryr.lib.opts:list_kuryr_opts +os_vif = + noop = kuryr_kubernetes.os_vif_plug_noop:NoOpPlugin + console_scripts = kuryr-k8s-controller = kuryr_kubernetes.cmd.eventlet.controller:start kuryr-cni = kuryr_kubernetes.cmd.cni:run kuryr_kubernetes.vif_translators = ovs = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_ovs + noop = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_nested kuryr_kubernetes.cni.binding = VIFBridge = kuryr_kubernetes.cni.binding.bridge:BridgeDriver VIFOpenVSwitch = kuryr_kubernetes.cni.binding.bridge:VIFOpenVSwitchDriver + VIFVlanNested = kuryr_kubernetes.cni.binding.nested:VlanDriver kuryr_kubernetes.controller.drivers.pod_project = default = kuryr_kubernetes.controller.drivers.default_project:DefaultPodProjectDriver @@ -45,6 +50,7 @@ kuryr_kubernetes.controller.drivers.pod_security_groups = kuryr_kubernetes.controller.drivers.pod_vif = generic = kuryr_kubernetes.controller.drivers.generic_vif:GenericPodVIFDriver + nested-vlan = kuryr_kubernetes.controller.drivers.nested_vlan_vif:NestedVlanPodVIFDriver [files] packages =