From eebcc9b7a1b5cd11271d31482301e451e1fbce95 Mon Sep 17 00:00:00 2001
From: Bharat Kunwar <brtknr@bath.edu>
Date: Thu, 12 Sep 2019 20:27:24 +0000
Subject: [PATCH] Fix k8s deployment when cluster_user_trust=False

At the moment, cluster deployment fails when cluster_user_trust=False.
This is because the entire SoftwareDeployment exits rather than a single
script fragment. This patch fixes this by scoping the remainder of the
script conditional on whether TRUST_ID is defined.

Finally, default `cloud_provider_enabled` to false when
`cluster_user_trust` is false. Raise an error when
`cloud_provider_enabled` is overridden to true when `cluster_user_trust`
is false. This ensures that the minion kubelet is correctly configured.

Change-Id: Ibd9270c87bfa5d2f490e2e226e33ca56696d9e81
Story: 2006531
Task: 36587
---
 .../fragments/configure-kubernetes-master.sh  |  6 +--
 .../kube-apiserver-to-kubelet-role.sh         |  2 +-
 .../fragments/write-kube-os-config.sh         | 41 ++++++++-----------
 .../drivers/heat/k8s_fedora_template_def.py   |  8 +++-
 4 files changed, 28 insertions(+), 29 deletions(-)

diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
index 6b9014d405..6e34ac930c 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
@@ -127,7 +127,7 @@ if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then
     KUBE_ADMISSION_CONTROL="--admission-control=NodeRestriction,${ADMISSION_CONTROL_LIST}"
 fi
 
-if [ -n "$TRUST_ID" ] && [ "$(echo "${CLOUD_PROVIDER_ENABLED}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
+if [ "$(echo "${CLOUD_PROVIDER_ENABLED}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
     KUBE_API_ARGS="$KUBE_API_ARGS --cloud-provider=external"
 fi
 
@@ -181,7 +181,7 @@ if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then
     KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --service-account-private-key-file=$CERT_DIR/service_account_private.key --root-ca-file=$CERT_DIR/ca.crt"
 fi
 
-if [ -n "$TRUST_ID" ] && [ "$(echo "${CLOUD_PROVIDER_ENABLED}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
+if [ "$(echo "${CLOUD_PROVIDER_ENABLED}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
     KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cloud-provider=external"
     KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --external-cloud-volume-plugin=openstack --cloud-config=/etc/kubernetes/cloud-config"
 fi
@@ -205,7 +205,7 @@ KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=$
 KUBELET_ARGS="${KUBELET_ARGS} --volume-plugin-dir=/var/lib/kubelet/volumeplugins"
 KUBELET_ARGS="${KUBELET_ARGS} ${KUBELET_OPTIONS}"
 
-if [ -n "$TRUST_ID" ] && [ "$(echo "${CLOUD_PROVIDER_ENABLED}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
+if [ "$(echo "${CLOUD_PROVIDER_ENABLED}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
     KUBELET_ARGS="${KUBELET_ARGS} --cloud-provider=external"
 fi
 
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh
index 74bf6ef750..68d3a31180 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh
@@ -151,7 +151,7 @@ kubectl -n kube-system create secret generic os-trustee \
     --from-file=os-certAuthority=/etc/kubernetes/ca-bundle.crt
 
 #TODO: add heat variables for master count to determine leaderelect true/False ?
-if [ -n "${TRUST_ID}" ] && [ "$(echo "${CLOUD_PROVIDER_ENABLED}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
+if [ "$(echo "${CLOUD_PROVIDER_ENABLED}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
     occm_image="${CONTAINER_INFRA_PREFIX:-docker.io/k8scloudprovider/}openstack-cloud-controller-manager:${CLOUD_PROVIDER_TAG}"
     OCCM=/srv/magnum/kubernetes/openstack-cloud-controller-manager.yaml
 
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh b/magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh
index 7f279d4fd7..39b91ac4f8 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh
@@ -5,17 +5,14 @@ set +x
 set -x
 
 $ssh_cmd mkdir -p /etc/kubernetes/
-
-if [ -z "${TRUST_ID}" ]; then
-    exit 0
-fi
-
-KUBE_OS_CLOUD_CONFIG=/etc/kubernetes/cloud-config
 $ssh_cmd cp /etc/pki/tls/certs/ca-bundle.crt /etc/kubernetes/ca-bundle.crt
 
-# Generate a the configuration for Kubernetes services
-# to talk to OpenStack Neutron and Cinder
-CLOUD_CONFIG=$(cat <<EOF
+if [ -n "${TRUST_ID}" ]; then
+    KUBE_OS_CLOUD_CONFIG=/etc/kubernetes/cloud-config
+
+    # Generate a the configuration for Kubernetes services
+    # to talk to OpenStack Neutron and Cinder
+    cat > ${KUBE_OS_CLOUD_CONFIG} <<EOF
 [Global]
 auth-url=$AUTH_URL
 user-id=$TRUSTEE_USER_ID
@@ -33,24 +30,20 @@ monitor-max-retries=3
 [BlockStorage]
 bs-version=v2
 EOF
-)
 
-cat > ${KUBE_OS_CLOUD_CONFIG} <<EOF
-$CLOUD_CONFIG
-EOF
+    # Provide optional region parameter if it's set.
+    if [ -n "${REGION_NAME}" ]; then
+        sed -i '/ca-file/a region='${REGION_NAME}'' $KUBE_OS_CLOUD_CONFIG
+    fi
 
-# Provide optional region parameter if it's set.
-if [ -n "${REGION_NAME}" ]; then
-    sed -i '/ca-file/a region='${REGION_NAME}'' $KUBE_OS_CLOUD_CONFIG
-fi
+    # backwards compatibility, some apps may expect this file from previous magnum versions.
+    $ssh_cmd cp ${KUBE_OS_CLOUD_CONFIG} /etc/kubernetes/kube_openstack_config
 
-# backwards compatibility, some apps may expect this file from previous magnum versions.
-$ssh_cmd cp ${KUBE_OS_CLOUD_CONFIG} /etc/kubernetes/kube_openstack_config
-
-# Append additional networking config to config file provided to openstack
-# cloud controller manager (not supported by in-tree Cinder).
-cat > ${KUBE_OS_CLOUD_CONFIG}-occm <<EOF
-$CLOUD_CONFIG
+    # Append additional networking config to config file provided to openstack
+    # cloud controller manager (not supported by in-tree Cinder).
+    $ssh_cmd cp ${KUBE_OS_CLOUD_CONFIG} ${KUBE_OS_CLOUD_CONFIG}-occm
+    cat >> ${KUBE_OS_CLOUD_CONFIG}-occm <<EOF
 [Networking]
 internal-network-name=$CLUSTER_NETWORK_NAME
 EOF
+fi
diff --git a/magnum/drivers/heat/k8s_fedora_template_def.py b/magnum/drivers/heat/k8s_fedora_template_def.py
index 427149a598..58135242de 100644
--- a/magnum/drivers/heat/k8s_fedora_template_def.py
+++ b/magnum/drivers/heat/k8s_fedora_template_def.py
@@ -110,7 +110,13 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
         # check cloud provider and cinder options. If cinder is selected,
         # the cloud provider needs to be enabled.
         cloud_provider_enabled = cluster.labels.get(
-            'cloud_provider_enabled', 'true').lower()
+            'cloud_provider_enabled',
+            'true' if CONF.trust.cluster_user_trust else 'false').lower()
+        if (not CONF.trust.cluster_user_trust
+                and cloud_provider_enabled == 'true'):
+            raise exception.InvalidParameterValue(_(
+                '"cluster_user_trust" must be set to True in magnum.conf when '
+                '"cloud_provider_enabled" label is set to true.'))
         if (cluster_template.volume_driver == 'cinder'
                 and cloud_provider_enabled == 'false'):
             raise exception.InvalidParameterValue(_(