[swarm] Enable TLS in Etcd cluster

With this patch following are done:-
- Configure Etcd with TLS support

Configure Following to commuicate with TLS enabled Etcd:-
- Swarm manager
- Swarm agent
- Docker
- Flannel

Etcd also listens at http://127.0.0.1:2379,
so on master nodes etcdctl can be used without certificates.

if TLS_DISABLED="True" then no TLS is enabled for etcd.

Change-Id: I6cadfebcfaaaf7ac7a7660b377b7d96748f0f9f0
Partially-Implements: blueprint secure-etcd-cluster-coe
This commit is contained in:
yatin 2016-12-04 21:52:32 +05:30 committed by yatinkarel
parent c349d2288a
commit ffb751d638
5 changed files with 107 additions and 17 deletions

View File

@ -3,18 +3,37 @@
. /etc/sysconfig/heat-params
myip="$SWARM_NODE_IP"
cert_dir="/etc/docker"
protocol="https"
if [ "$TLS_DISABLED" = "True" ]; then
protocol="http"
fi
cat > /etc/etcd/etcd.conf <<EOF
ETCD_NAME="$myip"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379"
ETCD_LISTEN_PEER_URLS="http://$myip:2380"
ETCD_LISTEN_CLIENT_URLS="$protocol://$myip:2379,http://127.0.0.1:2379"
ETCD_LISTEN_PEER_URLS="$protocol://$myip:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://$myip:2379"
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://$myip:2380"
ETCD_ADVERTISE_CLIENT_URLS="$protocol://$myip:2379,http://127.0.0.1:2379"
ETCD_INITIAL_ADVERTISE_PEER_URLS="$protocol://$myip:2380"
ETCD_DISCOVERY="$ETCD_DISCOVERY_URL"
EOF
if [ "$TLS_DISABLED" = "False" ]; then
cat >> /etc/etcd/etcd.conf <<EOF
ETCD_CA_FILE=$cert_dir/ca.crt
ETCD_CERT_FILE=$cert_dir/server.crt
ETCD_KEY_FILE=$cert_dir/server.key
ETCD_PEER_CA_FILE=$cert_dir/ca.crt
ETCD_PEER_CERT_FILE=$cert_dir/server.crt
ETCD_PEER_KEY_FILE=$cert_dir/server.key
EOF
fi
if [ -n "$HTTP_PROXY" ]; then
echo "ETCD_DISCOVERY_PROXY=$HTTP_PROXY" >> /etc/etcd/etcd.conf
fi

View File

@ -12,11 +12,30 @@ FLANNELD_CONFIG=/etc/sysconfig/flanneld
FLANNEL_CONFIG_BIN=/usr/local/bin/flannel-config
FLANNEL_CONFIG_SERVICE=/etc/systemd/system/flannel-config.service
FLANNEL_JSON=/etc/sysconfig/flannel-network.json
CERT_DIR=/etc/docker
PROTOCOL=https
FLANNEL_OPTIONS="-etcd-cafile $CERT_DIR/ca.crt \
-etcd-certfile $CERT_DIR/server.crt \
-etcd-keyfile $CERT_DIR/server.key"
ETCD_CURL_OPTIONS="--cacert $CERT_DIR/ca.crt \
--cert $CERT_DIR/server.crt --key $CERT_DIR/server.key"
if [ "$TLS_DISABLED" = "True" ]; then
PROTOCOL=http
FLANNEL_OPTIONS=""
ETCD_CURL_OPTIONS=""
fi
sed -i '
/^FLANNEL_ETCD=/ s|=.*|="http://'"$ETCD_SERVER_IP"':2379"|
/^FLANNEL_ETCD=/ s|=.*|="'"$PROTOCOL"'://'"$ETCD_SERVER_IP"':2379"|
' $FLANNELD_CONFIG
sed -i '/FLANNEL_OPTIONS/'d $FLANNELD_CONFIG
cat >> $FLANNELD_CONFIG <<EOF
FLANNEL_OPTIONS="$FLANNEL_OPTIONS"
EOF
. $FLANNELD_CONFIG
echo "creating $FLANNEL_CONFIG_BIN"
@ -34,7 +53,8 @@ if ! [ "$FLANNEL_ETCD" ] && [ "$FLANNEL_ETCD_KEY" ]; then
fi
echo "creating flanneld config in etcd"
while ! curl -sf -L $FLANNEL_ETCD/v2/keys${FLANNEL_ETCD_KEY}/config \
while ! curl -sf -L $ETCD_CURL_OPTIONS \
$FLANNEL_ETCD/v2/keys${FLANNEL_ETCD_KEY}/config \
-X PUT --data-urlencode value@${FLANNEL_JSON}; do
echo "waiting for etcd"
sleep 1

View File

@ -2,11 +2,27 @@
. /etc/sysconfig/heat-params
CERT_DIR=/etc/docker
PROTOCOL=https
FLANNEL_OPTIONS="-etcd-cafile $CERT_DIR/ca.crt \
-etcd-certfile $CERT_DIR/server.crt \
-etcd-keyfile $CERT_DIR/server.key"
DOCKER_NETWORK_OPTIONS="--cluster-store etcd://$ETCD_SERVER_IP:2379 \
--cluster-store-opt kv.cacertfile=$CERT_DIR/ca.crt \
--cluster-store-opt kv.certfile=$CERT_DIR/server.crt \
--cluster-store-opt kv.keyfile=$CERT_DIR/server.key \
--cluster-advertise $SWARM_NODE_IP:9379"
if [ "$TLS_DISABLED" = "True" ]; then
PROTOCOL=http
FLANNEL_OPTIONS=""
DOCKER_NETWORK_OPTIONS="--cluster-store etcd://$ETCD_SERVER_IP:2379 \
--cluster-advertise $SWARM_NODE_IP:9379"
fi
echo "Configuring ${NETWORK_DRIVER} network service ..."
if [ "$NETWORK_DRIVER" == "docker" ]; then
DOCKER_NETWORK_OPTIONS="--cluster-store etcd://$ETCD_SERVER_IP:2379 \
--cluster-advertise $SWARM_NODE_IP:9379"
sed -i "/^DOCKER_NETWORK_OPTIONS=/ s#=.*#='$DOCKER_NETWORK_OPTIONS'#" \
/etc/sysconfig/docker-network
fi
@ -25,9 +41,15 @@ mkdir -p /etc/systemd/system/docker.service.d
mkdir -p /etc/systemd/system/flanneld.service.d
sed -i '
/^FLANNEL_ETCD=/ s|=.*|="http://'"$ETCD_SERVER_IP"':2379"|
/^FLANNEL_ETCD=/ s|=.*|="'"$PROTOCOL"'://'"$ETCD_SERVER_IP"':2379"|
' $FLANNELD_CONFIG
sed -i '/FLANNEL_OPTIONS/'d $FLANNELD_CONFIG
cat >> $FLANNELD_CONFIG <<EOF
FLANNEL_OPTIONS="$FLANNEL_OPTIONS"
EOF
cat >> $FLANNEL_DOCKER_BRIDGE_BIN <<EOF
#!/bin/sh

View File

@ -5,6 +5,16 @@
myip="$SWARM_NODE_IP"
CONF_FILE=/etc/systemd/system/swarm-agent.service
CERT_DIR=/etc/docker
PROTOCOL=https
ETCDCTL_OPTIONS="--ca-file $CERT_DIR/ca.crt \
--cert-file $CERT_DIR/client.crt \
--key-file $CERT_DIR/client.key"
if [ $TLS_DISABLED = 'True' ]; then
PROTOCOL=http
ETCDCTL_OPTIONS=""
fi
cat > $CONF_FILE << EOF
[Unit]
@ -21,18 +31,32 @@ ExecStartPre=-/usr/bin/docker pull swarm:$SWARM_VERSION
ExecStart=/usr/bin/docker run -e http_proxy=$HTTP_PROXY \\
-e https_proxy=$HTTPS_PROXY \\
-e no_proxy=$NO_PROXY \\
-v $CERT_DIR:$CERT_DIR \\
--name swarm-agent \\
swarm:$SWARM_VERSION \\
join \\
--addr $myip:2375 \\
etcd://$ETCD_SERVER_IP:2379/v2/keys/swarm/
EOF
if [ $TLS_DISABLED = 'False' ]; then
cat >> /etc/systemd/system/swarm-agent.service << END_TLS
--discovery-opt kv.cacertfile=$CERT_DIR/ca.crt \\
--discovery-opt kv.certfile=$CERT_DIR/server.crt \\
--discovery-opt kv.keyfile=$CERT_DIR/server.key \\
END_TLS
fi
cat >> /etc/systemd/system/swarm-agent.service << END_SERVICE_BOTTOM
etcd://$ETCD_SERVER_IP:2379/v2/keys/swarm/
Restart=always
ExecStop=/usr/bin/docker stop swarm-agent
ExecStartPost=/usr/local/bin/notify-heat
[Install]
WantedBy=multi-user.target
EOF
END_SERVICE_BOTTOM
chown root:root $CONF_FILE
chmod 644 $CONF_FILE
@ -42,8 +66,8 @@ SCRIPT=/usr/local/bin/notify-heat
cat > $SCRIPT << EOF
#!/bin/sh
until etcdctl \
--peers $ETCD_SERVER_IP:2379 \
--timeout 1s \
--peers $PROTOCOL://$ETCD_SERVER_IP:2379 \
$ETCDCTL_OPTIONS --timeout 1s \
--total-timeout 5s \
ls /v2/keys/swarm/docker/swarm/nodes/$myip:2375
do

View File

@ -1,5 +1,7 @@
#!/bin/sh
CERT_DIR=/etc/docker
cat > /etc/systemd/system/swarm-manager.service << END_SERVICE_TOP
[Unit]
Description=Swarm Manager
@ -13,7 +15,7 @@ ExecStartPre=-/usr/bin/docker kill swarm-manager
ExecStartPre=-/usr/bin/docker rm swarm-manager
ExecStartPre=-/usr/bin/docker pull swarm:$SWARM_VERSION
ExecStart=/usr/bin/docker run --name swarm-manager \\
-v /etc/docker:/etc/docker \\
-v $CERT_DIR:$CERT_DIR \\
-p 2376:2375 \\
-e http_proxy=$HTTP_PROXY \\
-e https_proxy=$HTTPS_PROXY \\
@ -29,9 +31,12 @@ if [ $TLS_DISABLED = 'False' ]; then
cat >> /etc/systemd/system/swarm-manager.service << END_TLS
--tlsverify \\
--tlscacert=/etc/docker/ca.crt \\
--tlskey=/etc/docker/server.key \\
--tlscert=/etc/docker/server.crt \\
--tlscacert=$CERT_DIR/ca.crt \\
--tlskey=$CERT_DIR/server.key \\
--tlscert=$CERT_DIR/server.crt \\
--discovery-opt kv.cacertfile=$CERT_DIR/ca.crt \\
--discovery-opt kv.certfile=$CERT_DIR/server.crt \\
--discovery-opt kv.keyfile=$CERT_DIR/server.key \\
END_TLS
fi