diff --git a/devstack/lib/kuryr_kubernetes b/devstack/lib/kuryr_kubernetes index 981684717..7146014b7 100644 --- a/devstack/lib/kuryr_kubernetes +++ b/devstack/lib/kuryr_kubernetes @@ -404,6 +404,7 @@ EOF function generate_controller_deployment() { output_dir=$1 + readiness_probe=${2:-False} mkdir -p "$output_dir" rm -f ${output_dir}/controller_deployment.yml cat >> "${output_dir}/controller_deployment.yml" << EOF @@ -434,6 +435,21 @@ spec: - name: config-volume mountPath: "/etc/kuryr/kuryr.conf" subPath: kuryr.conf +EOF + # Add readiness probe if ports pool functionality is enabled. The rationale + # behind is to make the controller not ready until the precreated ports are + # loaded into the pools + if [ "$readiness_probe" ]; then + cat >> "${output_dir}/controller_deployment.yml" << EOF + readinessProbe: + exec: + command: + - cat + - /tmp/pools_loaded +EOF + fi + + cat >> "${output_dir}/controller_deployment.yml" << EOF volumes: - name: config-volume configMap: diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 4bec324cd..13aa48fd3 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -107,7 +107,7 @@ function generate_containerized_kuryr_resources { local output_dir="${DATA_DIR}/kuryr-kubernetes" generate_kuryr_configmap $output_dir $KURYR_CONFIG $KURYR_CNI_CONFIG generate_kuryr_service_account $output_dir - generate_controller_deployment $output_dir + generate_controller_deployment $output_dir $KURYR_USE_PORTS_POOLS generate_cni_daemon_set $output_dir $CNI_BIN_DIR $CNI_CONF_DIR } diff --git a/doc/source/installation/containerized.rst b/doc/source/installation/containerized.rst index 2cadbd0f9..657eb1ec5 100644 --- a/doc/source/installation/containerized.rst +++ b/doc/source/installation/containerized.rst @@ -55,6 +55,15 @@ Below is the list of available variables: * ``$KURYR_K8S_BINDING_DRIVER`` - ``[binding]driver`` (default: ``kuryr.lib.binding.drivers.vlan``) * ``$KURYR_K8S_BINDING_IFACE`` - ``[binding]link_iface`` (default: eth0) +In case of using ports pool functionality, we may want to make the +kuryr-controller not ready until the pools are populated with the existing +ports. To achive this a readiness probe must be added to the kuryr-controller +deployment. To add the readiness probe, in addition to the above environment +variables or the kuryr-controller configuration file, and extra environmental +variable must be set: + +* ``$KURYR_USE_PORTS_POOLS`` - ``True`` (default: False) + Example run: :: $ KURYR_K8S_API_ROOT="192.168.0.1:6443" ./tools/generate_k8s_resource_definitions /tmp @@ -73,13 +82,13 @@ To deploy the files on your Kubernetes cluster run: :: $ kubectl apply -f config_map.yml -n kube-system $ kubectl apply -f service_account.yml -n kube-system - $ kubectl apply -f conteoller_deployment.yml -n kube-system + $ kubectl apply -f controller_deployment.yml -n kube-system $ kubectl apply -f cni_ds.yml -n kube-system After successful completion: * kuryr-controller Deployment object, with single replica count, will get - created in default namespace. + created in kube-system namespace. * kuryr-cni gets installed as a daemonset object on all the nodes in kube-system namespace diff --git a/kuryr_kubernetes/controller/drivers/vif_pool.py b/kuryr_kubernetes/controller/drivers/vif_pool.py index 26bd1b6be..cfadb8c9b 100644 --- a/kuryr_kubernetes/controller/drivers/vif_pool.py +++ b/kuryr_kubernetes/controller/drivers/vif_pool.py @@ -211,6 +211,18 @@ class BaseVIFPool(base.VIFPoolDriver): def show_pool(self, pool_key): return self._available_ports_pools.get(pool_key) + def _create_healthcheck_file(self): + # Note(ltomasbo): Create a health check file when the pre-created + # ports are loaded into their corresponding pools. This file is used + # by the readiness probe when the controller is deployed in + # containerized mode. This way the controller pod will not be ready + # until all the pre-created ports have been loaded + try: + with open('/tmp/pools_loaded', 'a'): + LOG.debug("Health check file created for readiness probe") + except IOError: + LOG.exception("I/O error creating the health check file.") + class NeutronVIFPool(BaseVIFPool): """Manages VIFs for Bare Metal Kubernetes Pods.""" @@ -324,6 +336,7 @@ class NeutronVIFPool(BaseVIFPool): pool_key, []).append(port['id']) LOG.info("PORTS POOL: pools updated with pre-created ports") + self._create_healthcheck_file() class NestedVIFPool(BaseVIFPool): @@ -445,6 +458,7 @@ class NestedVIFPool(BaseVIFPool): def _recover_precreated_ports(self): self._precreated_ports(action='recover') LOG.info("PORTS POOL: pools updated with pre-created ports") + self._create_healthcheck_file() def _remove_precreated_ports(self, trunk_ips=None): self._precreated_ports(action='free', trunk_ips=trunk_ips) diff --git a/tools/generate_k8s_resource_definitions.sh b/tools/generate_k8s_resource_definitions.sh index 098ad3cb2..36518f86b 100755 --- a/tools/generate_k8s_resource_definitions.sh +++ b/tools/generate_k8s_resource_definitions.sh @@ -101,5 +101,6 @@ fi generate_kuryr_configmap $OUTPUT_DIR $CONTROLLER_CONF_PATH $CNI_CONF_PATH generate_kuryr_service_account $OUTPUT_DIR -generate_controller_deployment $OUTPUT_DIR +readiness_probe=${KURYR_USE_PORTS_POOLS:-False} +generate_controller_deployment $OUTPUT_DIR $readiness_probe generate_cni_daemon_set $OUTPUT_DIR