diff --git a/cni.Dockerfile b/cni.Dockerfile index 5f6aeb8a3..a01722b69 100644 --- a/cni.Dockerfile +++ b/cni.Dockerfile @@ -3,6 +3,14 @@ LABEL authors="Antoni Segura Puimedon, Vikas Choudhary] [] [] + +* ``cni_bin_dir`` - host directory where CNI binaries are located, defaults to + ``/opt/cni/bin``. +* ``cni_conf_dir`` - host directory where CNI configuration is located, + defaults to ``/etc/cni/net.d``. +* ``enable_cni_daemon`` - Set to ``True`` if you want CNI Docker image to run + CNI daemon by default. Defaults to ``False``. + +.. note:: + You can override those build variables by passing env variables when running + the Docker image. Supported variables are ``CNI_CONFIG_DIR_PATH``, + ``CNI_BIN_DIR_PATH`` and ``CNI_DAEMON``. Alternatively, you can remove ``imagePullPolicy: Never`` from kuryr-controller Deployment and kuryr-cni DaemonSet definitions to use pre-built diff --git a/kuryr_kubernetes/cni/binding/base.py b/kuryr_kubernetes/cni/binding/base.py index 5cd3451fa..e4c468bac 100644 --- a/kuryr_kubernetes/cni/binding/base.py +++ b/kuryr_kubernetes/cni/binding/base.py @@ -17,6 +17,8 @@ import os_vif import pyroute2 from stevedore import driver as stv_driver +from kuryr_kubernetes import utils + _BINDING_NAMESPACE = 'kuryr_kubernetes.cni.binding' @@ -29,6 +31,7 @@ def _get_binding_driver(vif): def get_ipdb(netns=None): if netns: + netns = utils.convert_netns(netns) ipdb = pyroute2.IPDB(nl=pyroute2.NetNS(netns)) else: ipdb = pyroute2.IPDB() @@ -39,10 +42,12 @@ def _enable_ipv6(netns): # Docker disables IPv6 for --net=none containers # TODO(apuimedo) remove when it is no longer the case try: - self_ns_fd = open('/proc/self/ns/net') + netns = utils.convert_netns(netns) + path = utils.convert_netns('/proc/self/ns/net') + self_ns_fd = open(path) pyroute2.netns.setns(netns) - with open('/proc/sys/net/ipv6/conf/all/disable_ipv6', - 'w') as disable_ipv6: + path = utils.convert_netns('/proc/sys/net/ipv6/conf/all/disable_ipv6') + with open(path, 'w') as disable_ipv6: disable_ipv6.write('0') except Exception: raise diff --git a/kuryr_kubernetes/cni/binding/nested.py b/kuryr_kubernetes/cni/binding/nested.py index f58a6bd70..5670a9cfa 100644 --- a/kuryr_kubernetes/cni/binding/nested.py +++ b/kuryr_kubernetes/cni/binding/nested.py @@ -17,6 +17,7 @@ import six from kuryr_kubernetes.cni.binding import base as b_base from kuryr_kubernetes import config +from kuryr_kubernetes import utils VLAN_KIND = 'vlan' MACVLAN_KIND = 'macvlan' @@ -48,7 +49,7 @@ class NestedDriver(object): with h_ipdb.create(ifname=temp_name, link=h_ipdb.interfaces[vm_iface_name], **args) as iface: - iface.net_ns_fd = netns + iface.net_ns_fd = utils.convert_netns(netns) with b_base.get_ipdb(netns) as c_ipdb: with c_ipdb.interfaces[temp_name] as iface: diff --git a/kuryr_kubernetes/cni/daemon/service.py b/kuryr_kubernetes/cni/daemon/service.py index e019786d0..60df0853d 100644 --- a/kuryr_kubernetes/cni/daemon/service.py +++ b/kuryr_kubernetes/cni/daemon/service.py @@ -13,6 +13,7 @@ # limitations under the License. import multiprocessing +import os from six.moves import http_client as httplib import socket import sys @@ -128,11 +129,34 @@ class DaemonServer(object): self.headers = {'ContentType': 'application/json', 'Connection': 'close'} + def _prepare_request(self): + if CONF.cni_daemon.docker_mode: + # FIXME(dulek): This is an awful hack to make os_vif's privsep + # daemon to run in FORK mode. This is required, + # as it's assumed kuryr-daemon is run as root, but + # it's not assumed that system it's running on has + # sudo command. Once os_vif allows to configure the + # mode, switch this to nicer method. It's placed + # here, because we need to repeat it for each process + # spawned by HTTP server. + ovs = os_vif._EXT_MANAGER['ovs'].obj + ovs_mod = sys.modules[ovs.__module__] + ovs_mod.linux_net.privsep.vif_plug.start( + ovs_mod.linux_net.privsep.priv_context.Method.FORK) + + params = utils.CNIParameters(flask.request.get_json()) + LOG.debug('Received %s request. CNI Params: %s', + params.CNI_COMMAND, params) + return params + def add(self): - params = None try: - params = utils.CNIParameters(flask.request.get_json()) - LOG.debug('Received addNetwork request. CNI Params: %s', params) + params = self._prepare_request() + except Exception: + LOG.exception('Exception when reading CNI params.') + return '', httplib.BAD_REQUEST, self.headers + + try: vif = self.plugin.add(params) data = jsonutils.dumps(vif.obj_to_primitive()) except exceptions.ResourceNotReady as e: @@ -143,13 +167,17 @@ class DaemonServer(object): LOG.exception('Error when processing addNetwork request. CNI ' 'Params: %s', params) return '', httplib.INTERNAL_SERVER_ERROR, self.headers + return data, httplib.ACCEPTED, self.headers def delete(self): - params = None try: - params = utils.CNIParameters(flask.request.get_json()) - LOG.debug('Received delNetwork request. CNI Params: %s', params) + params = self._prepare_request() + except Exception: + LOG.exception('Exception when reading CNI params.') + return '', httplib.BAD_REQUEST, self.headers + + try: self.plugin.delete(params) except exceptions.ResourceNotReady as e: # NOTE(dulek): It's better to ignore this error - most of the time @@ -212,6 +240,15 @@ class CNIDaemonWatcherService(cotyledon.Service): self.watcher = None self.registry = registry + def _get_nodename(self): + # NOTE(dulek): At first try to get it using environment variable, + # otherwise assume hostname is the nodename. + try: + nodename = os.environ['KUBERNETES_NODE_NAME'] + except KeyError: + nodename = socket.gethostname() + return nodename + def run(self): self.pipeline = h_cni.CNIPipeline() self.pipeline.register(h_cni.CallbackHandler(self.on_done)) @@ -219,7 +256,7 @@ class CNIDaemonWatcherService(cotyledon.Service): self.watcher.add( "%(base)s/pods?fieldSelector=spec.nodeName=%(node_name)s" % { 'base': k_const.K8S_API_BASE, - 'node_name': socket.gethostname()}) + 'node_name': self._get_nodename()}) self.watcher.start() def on_done(self, pod, vif): diff --git a/kuryr_kubernetes/config.py b/kuryr_kubernetes/config.py index bc18d546f..463e03b0c 100644 --- a/kuryr_kubernetes/config.py +++ b/kuryr_kubernetes/config.py @@ -54,6 +54,22 @@ daemon_opts = [ 'process all networking stack changes. This option ' 'allows to tune internal pyroute2 timeout.'), default=10), + cfg.BoolOpt('docker_mode', + help=_('Set to True when you are running kuryr-daemon inside ' + 'a Docker container on Kubernetes host. E.g. as ' + 'DaemonSet on Kubernetes cluster Kuryr is supposed to ' + 'provide networking for. This mainly means that' + 'kuryr-daemon will look for network namespaces in ' + '$netns_proc_dir instead of /proc.'), + default=False), + cfg.StrOpt('netns_proc_dir', + help=_("When docker_mode is set to True, this config option " + "should be set to where host's /proc directory is " + "mounted. Please note that mounting it is necessary to " + "allow Kuryr-Kubernetes to move host interfaces between " + "host network namespaces, which is essential for Kuryr " + "to work."), + default=None), ] k8s_opts = [ diff --git a/kuryr_kubernetes/tests/unit/cni/test_service.py b/kuryr_kubernetes/tests/unit/cni/test_service.py index 286187adb..5a69867e6 100644 --- a/kuryr_kubernetes/tests/unit/cni/test_service.py +++ b/kuryr_kubernetes/tests/unit/cni/test_service.py @@ -95,7 +95,8 @@ class TestDaemonServer(base.TestCase): self.srv.application.testing = True self.test_client = self.srv.application.test_client() - params = {'config_kuryr': {}, 'CNI_ARGS': 'foo=bar'} + params = {'config_kuryr': {}, 'CNI_ARGS': 'foo=bar', + 'CNI_CONTAINERID': 'baz', 'CNI_COMMAND': 'ADD'} self.params_str = jsonutils.dumps(params) @mock.patch('kuryr_kubernetes.cni.daemon.service.K8sCNIRegistryPlugin.add') diff --git a/kuryr_kubernetes/utils.py b/kuryr_kubernetes/utils.py index 461043b09..bf87c7ab7 100644 --- a/kuryr_kubernetes/utils.py +++ b/kuryr_kubernetes/utils.py @@ -10,8 +10,11 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo_config import cfg from oslo_serialization import jsonutils +CONF = cfg.CONF + def utf8_json_decoder(byte_data): """Deserializes the bytes into UTF-8 encoded JSON. @@ -20,3 +23,19 @@ def utf8_json_decoder(byte_data): :returns: The UTF-8 encoded JSON represented by Python dictionary format. """ return jsonutils.loads(byte_data.decode('utf8')) + + +def convert_netns(netns): + """Convert /proc based netns path to Docker-friendly path. + + When CONF.docker_mode is set this method will change /proc to + /CONF.netns_proc_dir. This allows netns manipulations to work when running + in Docker container on Kubernetes host. + + :param netns: netns path to convert. + :return: Converted netns path. + """ + if CONF.cni_daemon.docker_mode: + return netns.replace('/proc', CONF.cni_daemon.netns_proc_dir) + else: + return netns diff --git a/tools/build_cni_daemonset_image b/tools/build_cni_daemonset_image index 29b535a64..ec0863adf 100755 --- a/tools/build_cni_daemonset_image +++ b/tools/build_cni_daemonset_image @@ -2,6 +2,7 @@ CNI_BIN_DIR=$1 CNI_CONF_DIR=$2 +CNI_DAEMON=${3:-"False"} BUILDER_TAG="kuryr/cni-builder" CNI_TAG="kuryr/cni" @@ -12,6 +13,7 @@ else docker build -t "$BUILDER_TAG" \ --build-arg "CNI_BIN_DIR_PATH=$CNI_BIN_DIR" \ --build-arg "CNI_CONFIG_DIR_PATH=$CNI_CONF_DIR" \ + --build-arg "CNI_DAEMON=$CNI_DAEMON" \ -f cni_builder.Dockerfile . fi docker run \ @@ -20,4 +22,7 @@ docker run \ "$BUILDER_TAG":latest # create cni daemonset image -docker build -t "$CNI_TAG" -f cni.Dockerfile . +docker build -t "$CNI_TAG" \ + --build-arg "CNI_DAEMON=$CNI_DAEMON" \ + -f cni.Dockerfile . +