[k8s_fedora_atomic] Enable TLS in Etcd cluster

With this patch following are done:-
- Configure Etcd with TLS support

Configure Following to commuicate with TLS enabled Etcd:-
- Flannel

Etcd also listens at http://127.0.0.1:2379, so on master nodes
etcdctl and kube apiserver can communicate without using
certificates.

if TLS_DISABLED="True" then TLS is not enabled for etcd.

Change-Id: I2147b67c4e346a4415e1f76c19ac68e94cb0a0fa
Partially-Implements: blueprint secure-etcd-cluster-coe

magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh

@@ -8,18 +8,37 @@ if [ -z "$KUBE_NODE_IP" ]; then
 fi
 
 myip="${KUBE_NODE_IP}"
+cert_dir="/srv/kubernetes"
+protocol="https"
+
+if [ "$TLS_DISABLED" = "True" ]; then
+    protocol="http"
+fi
 
 cat > /etc/etcd/etcd.conf <<EOF
 ETCD_NAME="$myip"
 ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
-ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379"
-ETCD_LISTEN_PEER_URLS="http://$myip:2380"
+ETCD_LISTEN_CLIENT_URLS="$protocol://$myip:2379,http://127.0.0.1:2379"
+ETCD_LISTEN_PEER_URLS="$protocol://$myip:2380"
 
-ETCD_ADVERTISE_CLIENT_URLS="http://$myip:2379"
-ETCD_INITIAL_ADVERTISE_PEER_URLS="http://$myip:2380"
+ETCD_ADVERTISE_CLIENT_URLS="$protocol://$myip:2379,http://127.0.0.1:2379"
+ETCD_INITIAL_ADVERTISE_PEER_URLS="$protocol://$myip:2380"
 ETCD_DISCOVERY="$ETCD_DISCOVERY_URL"
 EOF
 
+if [ "$TLS_DISABLED" = "False" ]; then
+
+cat >> /etc/etcd/etcd.conf <<EOF
+ETCD_CA_FILE=$cert_dir/ca.crt
+ETCD_CERT_FILE=$cert_dir/server.crt
+ETCD_KEY_FILE=$cert_dir/server.key
+ETCD_PEER_CA_FILE=$cert_dir/ca.crt
+ETCD_PEER_CERT_FILE=$cert_dir/server.crt
+ETCD_PEER_KEY_FILE=$cert_dir/server.key
+EOF
+
+fi
+
 if [ -n "$HTTP_PROXY" ]; then
     echo "ETCD_DISCOVERY_PROXY=$HTTP_PROXY" >> /etc/etcd/etcd.conf
 fi

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh

@@ -9,10 +9,6 @@ if [ -z "$KUBE_NODE_IP" ]; then
     KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
 fi
 
-sed -i '
-    /^ETCD_LISTEN_CLIENT_URLS=/ s/=.*/="http:\/\/0.0.0.0:2379"/
-' /etc/etcd/etcd.conf
-
 sed -i '
     /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow-privileged='"$KUBE_ALLOW_PRIV"'"/
 ' /etc/kubernetes/config

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh

@@ -9,9 +9,30 @@ if [ -z "$KUBE_NODE_IP" ]; then
     KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
 fi
 
+CERT_DIR=/srv/kubernetes
+PROTOCOL=https
+FLANNEL_OPTIONS="-etcd-cafile $CERT_DIR/ca.crt \
+-etcd-certfile $CERT_DIR/client.crt \
+-etcd-keyfile $CERT_DIR/client.key"
+ETCD_CURL_OPTIONS="--cacert $CERT_DIR/ca.crt \
+--cert $CERT_DIR/client.crt --key $CERT_DIR/client.key"
 ETCD_SERVER_IP=${ETCD_SERVER_IP:-$KUBE_MASTER_IP}
 KUBE_PROTOCOL="https"
 KUBE_CONFIG=""
+FLANNELD_CONFIG=/etc/sysconfig/flanneld
+
+if [ "$TLS_DISABLED" = "True" ]; then
+    PROTOCOL=http
+    FLANNEL_OPTIONS=""
+    ETCD_CURL_OPTIONS=""
+fi
+
+sed -i '/FLANNEL_OPTIONS/'d $FLANNELD_CONFIG
+
+cat >> $FLANNELD_CONFIG <<EOF
+FLANNEL_OPTIONS="$FLANNEL_OPTIONS"
+EOF
+
 if [ "$TLS_DISABLED" = "True" ]; then
     KUBE_PROTOCOL="http"
 else
@@ -52,12 +73,13 @@ sed -i '
 
 if [ "$NETWORK_DRIVER" = "flannel" ]; then
     sed -i '
-        /^FLANNEL_ETCD=/ s|=.*|="http://'"$ETCD_SERVER_IP"':2379"|
-    ' /etc/sysconfig/flanneld
+        /^FLANNEL_ETCD=/ s|=.*|="'"$PROTOCOL"'://'"$ETCD_SERVER_IP"':2379"|
+    ' $FLANNELD_CONFIG
 
     # Make sure etcd has a flannel configuration
-    . /etc/sysconfig/flanneld
-    until curl -sf "$FLANNEL_ETCD/v2/keys${FLANNEL_ETCD_KEY}/config?quorum=false&recursive=false&sorted=false"
+    . $FLANNELD_CONFIG
+    until curl -sf $ETCD_CURL_OPTIONS \
+        "$FLANNEL_ETCD/v2/keys${FLANNEL_ETCD_KEY}/config?quorum=false&recursive=false&sorted=false"
     do
         echo "Waiting for flannel configuration in etcd..."
         sleep 5

magnum/drivers/common/templates/kubernetes/fragments/make-cert-client.sh

@@ -107,8 +107,14 @@ curl -k -X POST \
     -d "$csr_req" \
     $MAGNUM_URL/certificates | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${CLIENT_CERT}
 
-chmod 500 "${cert_dir}"
-chown -R kube:kube "${cert_dir}"
+# Common certs and key are created for both etcd and kubernetes services.
+# Both etcd and kube user should have permission to access the certs and key.
+groupadd kube_etcd
+usermod -a -G kube_etcd etcd
+usermod -a -G kube_etcd kube
+chmod 550 "${cert_dir}"
+chown -R kube:kube_etcd "${cert_dir}"
+chmod 440 $CLIENT_KEY
 
 sed -i '
     s|CA_CERT|'"$CA_CERT"'|

magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh

@@ -124,5 +124,11 @@ curl -k -X POST \
     -d "$csr_req" \
     $MAGNUM_URL/certificates | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${SERVER_CERT}
 
-chmod 500 "${cert_dir}"
-chown -R kube:kube "${cert_dir}"
+# Common certs and key are created for both etcd and kubernetes services.
+# Both etcd and kube user should have permission to access the certs and key.
+groupadd kube_etcd
+usermod -a -G kube_etcd etcd
+usermod -a -G kube_etcd kube
+chmod 550 "${cert_dir}"
+chown -R kube:kube_etcd "${cert_dir}"
+chmod 440 $SERVER_KEY

magnum/drivers/common/templates/kubernetes/fragments/network-config-service.sh

@@ -5,8 +5,28 @@
 if [ "$NETWORK_DRIVER" != "flannel" ]; then
     exit 0
 fi
+CERT_DIR=/srv/kubernetes
+PROTOCOL=https
+FLANNEL_OPTIONS="-etcd-cafile $CERT_DIR/ca.crt \
+-etcd-certfile $CERT_DIR/server.crt \
+-etcd-keyfile $CERT_DIR/server.key"
+ETCD_CURL_OPTIONS="--cacert $CERT_DIR/ca.crt \
+--cert $CERT_DIR/server.crt --key $CERT_DIR/server.key"
+FLANNELD_CONFIG=/etc/sysconfig/flanneld
 
-. /etc/sysconfig/flanneld
+if [ "$TLS_DISABLED" = "True" ]; then
+    PROTOCOL=http
+    FLANNEL_OPTIONS=""
+    ETCD_CURL_OPTIONS=""
+fi
+
+sed -i '/FLANNEL_OPTIONS/'d $FLANNELD_CONFIG
+
+cat >> $FLANNELD_CONFIG <<EOF
+FLANNEL_OPTIONS="$FLANNEL_OPTIONS"
+EOF
+
+. $FLANNELD_CONFIG
 
 FLANNEL_CONFIG_BIN=/usr/local/bin/flannel-config
 FLANNEL_CONFIG_SERVICE=/etc/systemd/system/flannel-config.service
@@ -27,7 +47,8 @@ if ! [ "$FLANNEL_ETCD" ] && [ "$FLANNEL_ETCD_KEY" ]; then
 fi
 
 echo "creating flanneld config in etcd"
-while ! curl -sf -L $FLANNEL_ETCD/v2/keys${FLANNEL_ETCD_KEY}/config \
+while ! curl -sf -L $ETCD_CURL_OPTIONS \
+        $FLANNEL_ETCD/v2/keys${FLANNEL_ETCD_KEY}/config \
         -X PUT --data-urlencode value@${FLANNEL_JSON}; do
     echo "waiting for etcd"
     sleep 1