From f6d1c0de464d13dea89c9fa397d48528d52aafbc Mon Sep 17 00:00:00 2001 From: Feilong Wang Date: Mon, 2 Jul 2018 15:42:21 +1200 Subject: [PATCH] Fix etcd race condition issue Currently, Magnum is using k8s API /version to check the API availibility which is not a good way because /version only reflects if the basic k8s api is working on not. And it will return response even the etcd service is down. This patch fixes it by using /healthz to replace /version. Task: 22566 Story: 1775759 Change-Id: I45a1bd48a22842a251dafa6c349f0022fd319e3f --- .../templates/kubernetes/fragments/calico-service.sh | 2 +- .../templates/kubernetes/fragments/core-dns-service.sh | 2 +- .../kubernetes/fragments/enable-ingress-traefik.sh | 2 +- .../fragments/enable-prometheus-monitoring.sh | 2 +- .../fragments/kube-apiserver-to-kubelet-role.sh | 2 +- .../kubernetes/fragments/kube-dashboard-service.sh | 10 ++++++++-- .../templates/kubernetes/fragments/wc-notify-master.sh | 7 ++++--- 7 files changed, 17 insertions(+), 10 deletions(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh index 13a326d3fc..6eb2df0feb 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh @@ -456,7 +456,7 @@ if [ "$MASTER_INDEX" != "0" ]; then exit 0 fi -until curl -sf "http://127.0.0.1:8080/healthz" +until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] do echo "Waiting for Kubernetes API..." sleep 5 diff --git a/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh index b8bbe76d56..084195a333 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh @@ -255,7 +255,7 @@ if [ "$MASTER_INDEX" != "0" ]; then fi echo "Waiting for Kubernetes API..." -until curl --silent "http://127.0.0.1:8080/version" +until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] do sleep 5 done diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-traefik.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-traefik.sh index 49362b4b65..16c39de33d 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-traefik.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-traefik.sh @@ -121,7 +121,7 @@ fi # Binary for ingress traefik INGRESS_TRAEFIK_BIN_CONTENT='''#!/bin/sh -until curl -sf "http://127.0.0.1:8080/healthz" +until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] do echo "Waiting for Kubernetes API..." sleep 5 diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh index e05e4a2f28..84283bb63e 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh @@ -369,7 +369,7 @@ GRAFANA_DEF_DASHBOARD_FILE=$GRAFANA_DEF_DASHBOARDS"/default.json" # Write the binary for enable-monitoring KUBE_MON_BIN_CONTENT='''#!/bin/sh -until curl -sf "http://127.0.0.1:8080/healthz" +until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] do echo "Waiting for Kubernetes API..." sleep 5 diff --git a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh index b4b2f52db8..ad191e5a09 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh @@ -12,7 +12,7 @@ if [ "$MASTER_INDEX" != "0" ]; then fi echo "Waiting for Kubernetes API..." -until curl --silent "http://127.0.0.1:8080/version" +until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] do sleep 5 done diff --git a/magnum/drivers/common/templates/kubernetes/fragments/kube-dashboard-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/kube-dashboard-service.sh index 8b037e33cf..a34e37ed4a 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/kube-dashboard-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/kube-dashboard-service.sh @@ -335,8 +335,14 @@ spec: EOF } + if [ "$MASTER_INDEX" != "0" ]; then + exit 0 + fi + + + echo "Waiting for Kubernetes API..." - until curl --silent "http://127.0.0.1:8080/version" + until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] do sleep 5 done @@ -459,7 +465,7 @@ if [ "$MASTER_INDEX" != "0" ]; then fi echo "Waiting for Kubernetes API..." -until curl --silent "http://127.0.0.1:8080/version" +until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] do sleep 5 done diff --git a/magnum/drivers/common/templates/kubernetes/fragments/wc-notify-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/wc-notify-master.sh index 267546a207..fc78415794 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/wc-notify-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/wc-notify-master.sh @@ -13,7 +13,8 @@ WC_NOTIFY_SERVICE=/etc/systemd/system/wc-notify.service cat > $WC_NOTIFY_BIN < $WC_NOTIFY_SERVICE <