From 8b05365955f2ab12b9ae8258451a1bf9142339c1 Mon Sep 17 00:00:00 2001 From: Luis Tomas Bolivar Date: Wed, 8 Nov 2017 10:40:32 +0100 Subject: [PATCH] Add readiness probe to kuryr-controller pod This patch add a readiness probe to the kuryr controller when the ports pool functionality is enabled. This ensures the controller pod is not set to ready until all the precreated ports have been loaded into their respective pools. This helps admins to know when the kuryr-controller pod is prepared to start serving requests. Note the kuryr-controller will reply to request even if it is not on ready status. However, that will lead to trigger port creation for new pods as the already existing ports may not be on their respective pools yet. Change-Id: Id47d3e7450551c19cb19d9278e459bd32bf364cf --- devstack/lib/kuryr_kubernetes | 16 ++++++++++++++++ devstack/plugin.sh | 2 +- doc/source/installation/containerized.rst | 13 +++++++++++-- kuryr_kubernetes/controller/drivers/vif_pool.py | 14 ++++++++++++++ tools/generate_k8s_resource_definitions.sh | 3 ++- 5 files changed, 44 insertions(+), 4 deletions(-) diff --git a/devstack/lib/kuryr_kubernetes b/devstack/lib/kuryr_kubernetes index 981684717..7146014b7 100644 --- a/devstack/lib/kuryr_kubernetes +++ b/devstack/lib/kuryr_kubernetes @@ -404,6 +404,7 @@ EOF function generate_controller_deployment() { output_dir=$1 + readiness_probe=${2:-False} mkdir -p "$output_dir" rm -f ${output_dir}/controller_deployment.yml cat >> "${output_dir}/controller_deployment.yml" << EOF @@ -434,6 +435,21 @@ spec: - name: config-volume mountPath: "/etc/kuryr/kuryr.conf" subPath: kuryr.conf +EOF + # Add readiness probe if ports pool functionality is enabled. The rationale + # behind is to make the controller not ready until the precreated ports are + # loaded into the pools + if [ "$readiness_probe" ]; then + cat >> "${output_dir}/controller_deployment.yml" << EOF + readinessProbe: + exec: + command: + - cat + - /tmp/pools_loaded +EOF + fi + + cat >> "${output_dir}/controller_deployment.yml" << EOF volumes: - name: config-volume configMap: diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 4bec324cd..13aa48fd3 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -107,7 +107,7 @@ function generate_containerized_kuryr_resources { local output_dir="${DATA_DIR}/kuryr-kubernetes" generate_kuryr_configmap $output_dir $KURYR_CONFIG $KURYR_CNI_CONFIG generate_kuryr_service_account $output_dir - generate_controller_deployment $output_dir + generate_controller_deployment $output_dir $KURYR_USE_PORTS_POOLS generate_cni_daemon_set $output_dir $CNI_BIN_DIR $CNI_CONF_DIR } diff --git a/doc/source/installation/containerized.rst b/doc/source/installation/containerized.rst index 2cadbd0f9..657eb1ec5 100644 --- a/doc/source/installation/containerized.rst +++ b/doc/source/installation/containerized.rst @@ -55,6 +55,15 @@ Below is the list of available variables: * ``$KURYR_K8S_BINDING_DRIVER`` - ``[binding]driver`` (default: ``kuryr.lib.binding.drivers.vlan``) * ``$KURYR_K8S_BINDING_IFACE`` - ``[binding]link_iface`` (default: eth0) +In case of using ports pool functionality, we may want to make the +kuryr-controller not ready until the pools are populated with the existing +ports. To achive this a readiness probe must be added to the kuryr-controller +deployment. To add the readiness probe, in addition to the above environment +variables or the kuryr-controller configuration file, and extra environmental +variable must be set: + +* ``$KURYR_USE_PORTS_POOLS`` - ``True`` (default: False) + Example run: :: $ KURYR_K8S_API_ROOT="192.168.0.1:6443" ./tools/generate_k8s_resource_definitions /tmp @@ -73,13 +82,13 @@ To deploy the files on your Kubernetes cluster run: :: $ kubectl apply -f config_map.yml -n kube-system $ kubectl apply -f service_account.yml -n kube-system - $ kubectl apply -f conteoller_deployment.yml -n kube-system + $ kubectl apply -f controller_deployment.yml -n kube-system $ kubectl apply -f cni_ds.yml -n kube-system After successful completion: * kuryr-controller Deployment object, with single replica count, will get - created in default namespace. + created in kube-system namespace. * kuryr-cni gets installed as a daemonset object on all the nodes in kube-system namespace diff --git a/kuryr_kubernetes/controller/drivers/vif_pool.py b/kuryr_kubernetes/controller/drivers/vif_pool.py index 26bd1b6be..cfadb8c9b 100644 --- a/kuryr_kubernetes/controller/drivers/vif_pool.py +++ b/kuryr_kubernetes/controller/drivers/vif_pool.py @@ -211,6 +211,18 @@ class BaseVIFPool(base.VIFPoolDriver): def show_pool(self, pool_key): return self._available_ports_pools.get(pool_key) + def _create_healthcheck_file(self): + # Note(ltomasbo): Create a health check file when the pre-created + # ports are loaded into their corresponding pools. This file is used + # by the readiness probe when the controller is deployed in + # containerized mode. This way the controller pod will not be ready + # until all the pre-created ports have been loaded + try: + with open('/tmp/pools_loaded', 'a'): + LOG.debug("Health check file created for readiness probe") + except IOError: + LOG.exception("I/O error creating the health check file.") + class NeutronVIFPool(BaseVIFPool): """Manages VIFs for Bare Metal Kubernetes Pods.""" @@ -324,6 +336,7 @@ class NeutronVIFPool(BaseVIFPool): pool_key, []).append(port['id']) LOG.info("PORTS POOL: pools updated with pre-created ports") + self._create_healthcheck_file() class NestedVIFPool(BaseVIFPool): @@ -445,6 +458,7 @@ class NestedVIFPool(BaseVIFPool): def _recover_precreated_ports(self): self._precreated_ports(action='recover') LOG.info("PORTS POOL: pools updated with pre-created ports") + self._create_healthcheck_file() def _remove_precreated_ports(self, trunk_ips=None): self._precreated_ports(action='free', trunk_ips=trunk_ips) diff --git a/tools/generate_k8s_resource_definitions.sh b/tools/generate_k8s_resource_definitions.sh index 098ad3cb2..36518f86b 100755 --- a/tools/generate_k8s_resource_definitions.sh +++ b/tools/generate_k8s_resource_definitions.sh @@ -101,5 +101,6 @@ fi generate_kuryr_configmap $OUTPUT_DIR $CONTROLLER_CONF_PATH $CNI_CONF_PATH generate_kuryr_service_account $OUTPUT_DIR -generate_controller_deployment $OUTPUT_DIR +readiness_probe=${KURYR_USE_PORTS_POOLS:-False} +generate_controller_deployment $OUTPUT_DIR $readiness_probe generate_cni_daemon_set $OUTPUT_DIR