From d6a0e0b85c912c478a59d3173ffa0b7392a3f65d Mon Sep 17 00:00:00 2001
From: Pete Birley <pete@port.direct>
Date: Thu, 14 Mar 2019 15:26:30 -0500
Subject: [PATCH] Gate: Permit ceph deployment from outside the cluster

This PS extends the gate scripts to allow ceph to be deployed from
a workstation external to the k8s cluster.

Change-Id: I09b9a11747bab32c19637d8dd076b8caa3b89445
Signed-off-by: Pete Birley <pete@port.direct>
---
 tools/deployment/multinode/030-ceph.sh        | 18 ++++++---------
 .../multinode/035-ceph-ns-activate.sh         |  2 +-
 .../deployment/multinode/kube-node-subnet.sh  | 23 ++++++++++++++++---
 3 files changed, 28 insertions(+), 15 deletions(-)

diff --git a/tools/deployment/multinode/030-ceph.sh b/tools/deployment/multinode/030-ceph.sh
index cbacf7071..e44c4c1a9 100755
--- a/tools/deployment/multinode/030-ceph.sh
+++ b/tools/deployment/multinode/030-ceph.sh
@@ -25,20 +25,17 @@ make ceph-provisioners
 #NOTE: Deploy command
 [ -s /tmp/ceph-fs-uuid.txt ] || uuidgen > /tmp/ceph-fs-uuid.txt
 CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)"
-CEPH_CLUSTER_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)"
+CEPH_CLUSTER_NETWORK="${CEPH_PUBLIC_NETWORK}"
 CEPH_FS_ID="$(cat /tmp/ceph-fs-uuid.txt)"
-#NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this
-# should be set to 'hammer'
-. /etc/os-release
-if [ "x${ID}" == "xubuntu" ] && \
-   [ "$(uname -r | awk -F "." '{ print $2 }')" -lt "5" ]; then
+#NOTE(portdirect): to use RBD devices with kernels < 4.5 this should be set to 'hammer'
+LOWEST_CLUSTER_KERNEL_VERSION=$(kubectl get node  -o go-template='{{range .items}}{{.status.nodeInfo.kernelVersion}}{{"\n"}}{{ end }}' | sort -V | tail -1)
+if [ "$(echo ${LOWEST_CLUSTER_KERNEL_VERSION} | awk -F "." '{ print $1 }')" -lt "4" ] || [ "$(echo ${LOWEST_CLUSTER_KERNEL_VERSION} | awk -F "." '{ print $2 }')" -lt "15" ]; then
+  echo "Using hammer crush tunables"
   CRUSH_TUNABLES=hammer
 else
   CRUSH_TUNABLES=null
 fi
-if [ "x${ID}" == "xcentos" ]; then
-  CRUSH_TUNABLES=hammer
-fi
+NUMBER_OF_OSDS="$(kubectl get nodes -l ceph-osd=enabled --no-headers | wc -l)"
 tee /tmp/ceph.yaml << EOF
 endpoints:
   identity:
@@ -69,8 +66,7 @@ conf:
     crush:
       tunables: ${CRUSH_TUNABLES}
     target:
-      # NOTE(portdirect): 5 nodes, with one osd per node
-      osd: 5
+      osd: ${NUMBER_OF_OSDS}
       pg_per_osd: 100
   storage:
     osd:
diff --git a/tools/deployment/multinode/035-ceph-ns-activate.sh b/tools/deployment/multinode/035-ceph-ns-activate.sh
index 3e8c0f4d9..effbcf5ab 100755
--- a/tools/deployment/multinode/035-ceph-ns-activate.sh
+++ b/tools/deployment/multinode/035-ceph-ns-activate.sh
@@ -18,7 +18,7 @@ set -xe
 
 #NOTE: Deploy command
 CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)"
-CEPH_CLUSTER_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)"
+CEPH_CLUSTER_NETWORK="${CEPH_PUBLIC_NETWORK}"
 tee /tmp/ceph-osh-infra-config.yaml <<EOF
 endpoints:
   ceph_mon:
diff --git a/tools/deployment/multinode/kube-node-subnet.sh b/tools/deployment/multinode/kube-node-subnet.sh
index 2ce60994f..4d4ebbfbb 100755
--- a/tools/deployment/multinode/kube-node-subnet.sh
+++ b/tools/deployment/multinode/kube-node-subnet.sh
@@ -16,10 +16,27 @@
 set -e
 
 UTILS_IMAGE=docker.io/openstackhelm/gate-utils:v0.1.0
-NODE_IPS=$(mktemp --suffix=.txt)
+NODE_IPS=$(mktemp)
 kubectl get nodes -o json | jq -r '.items[].status.addresses[] | select(.type=="InternalIP").address' | sort -V > $NODE_IPS
-FIRST_IP_SUBNET=$(sudo docker run --rm ${UTILS_IMAGE} ipcalc "$(head -n 1 ${NODE_IPS})/24" | awk '/^Network/ { print $2 }')
-LAST_IP_SUBNET=$(sudo docker run --rm ${UTILS_IMAGE} ipcalc "$(tail -n 1 ${NODE_IPS})/24" | awk '/^Network/ { print $2 }')
+function run_and_log_ipcalc {
+  POD_NAME="tmp-$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-z | head -c 5; echo)"
+  kubectl run ${POD_NAME} \
+    --generator=run-pod/v1 \
+    --wait \
+    --image ${UTILS_IMAGE} \
+    --restart=Never \
+    ipcalc -- "$1"
+  end=$(($(date +%s) + 900))
+  until kubectl get pod/${POD_NAME} -o go-template='{{.status.phase}}' | grep -q Succeeded; do
+    now=$(date +%s)
+    [ $now -gt $end ] && echo containers failed to start. && \
+        kubectl get pod/${POD_NAME} -o wide && exit 1
+  done
+  kubectl logs pod/${POD_NAME}
+  kubectl delete pod/${POD_NAME}
+}
+FIRST_IP_SUBNET=$(run_and_log_ipcalc "$(head -n 1 ${NODE_IPS})/24" | awk '/^Network/ { print $2 }')
+LAST_IP_SUBNET=$(run_and_log_ipcalc "$(tail -n 1 ${NODE_IPS})/24" | awk '/^Network/ { print $2 }')
 rm -f $NODE_IPS
 function ip_diff {
   echo $(($(echo $LAST_IP_SUBNET | awk -F '.' "{ print \$$1}") - $(echo $FIRST_IP_SUBNET | awk -F '.' "{ print \$$1}")))