Merge "k8s_atomic: Run all syscontainer with podman" into stable/train
This commit is contained in:
commit
481a90ff6d
|
@ -50,12 +50,30 @@ if [ -n "$ETCD_VOLUME_SIZE" ] && [ "$ETCD_VOLUME_SIZE" -gt 0 ]; then
|
|||
|
||||
fi
|
||||
|
||||
_prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
|
||||
$ssh_cmd atomic install \
|
||||
--system-package no \
|
||||
--system \
|
||||
--storage ostree \
|
||||
--name=etcd ${_prefix}etcd:${ETCD_TAG}
|
||||
cat > /etc/systemd/system/etcd.service <<EOF
|
||||
[Unit]
|
||||
Description=Etcd server
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
ExecStartPre=mkdir -p /var/lib/etcd
|
||||
ExecStartPre=-/bin/podman rm etcd
|
||||
ExecStart=/bin/podman run \\
|
||||
--name etcd \\
|
||||
--volume /etc/pki/ca-trust/extracted/pem:/etc/ssl/certs:ro,z \\
|
||||
--volume /etc/etcd:/etc/etcd:ro,z \\
|
||||
--volume /var/lib/etcd:/var/lib/etcd:rshared,z \\
|
||||
--net=host \\
|
||||
${CONTAINER_INFRA_PREFIX:-"k8s.gcr.io/"}etcd:${ETCD_TAG} \\
|
||||
/usr/local/bin/etcd \\
|
||||
--config-file /etc/etcd/etcd.conf.yaml
|
||||
ExecStop=/bin/podman stop etcd
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
|
||||
if [ -z "$KUBE_NODE_IP" ]; then
|
||||
# FIXME(yuanying): Set KUBE_NODE_IP correctly
|
||||
|
@ -70,34 +88,69 @@ if [ "$TLS_DISABLED" = "True" ]; then
|
|||
protocol="http"
|
||||
fi
|
||||
|
||||
cat > /etc/etcd/etcd.conf <<EOF
|
||||
ETCD_NAME="$myip"
|
||||
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
|
||||
ETCD_LISTEN_CLIENT_URLS="$protocol://$myip:2379,http://127.0.0.1:2379"
|
||||
ETCD_LISTEN_PEER_URLS="$protocol://$myip:2380"
|
||||
cat > /etc/etcd/etcd.conf.yaml <<EOF
|
||||
# This is the configuration file for the etcd server.
|
||||
|
||||
# Human-readable name for this member.
|
||||
name: "${INSTANCE_NAME}"
|
||||
|
||||
# Path to the data directory.
|
||||
data-dir: /var/lib/etcd/default.etcd
|
||||
|
||||
# List of comma separated URLs to listen on for peer traffic.
|
||||
listen-peer-urls: "$protocol://$myip:2380"
|
||||
|
||||
# List of comma separated URLs to listen on for client traffic.
|
||||
listen-client-urls: "$protocol://$myip:2379,http://127.0.0.1:2379"
|
||||
|
||||
# List of this member's peer URLs to advertise to the rest of the cluster.
|
||||
# The URLs needed to be a comma-separated list.
|
||||
initial-advertise-peer-urls: "$protocol://$myip:2380"
|
||||
|
||||
# List of this member's client URLs to advertise to the public.
|
||||
# The URLs needed to be a comma-separated list.
|
||||
advertise-client-urls: "$protocol://$myip:2379,http://127.0.0.1:2379"
|
||||
|
||||
# Discovery URL used to bootstrap the cluster.
|
||||
discovery: "$ETCD_DISCOVERY_URL"
|
||||
|
||||
ETCD_ADVERTISE_CLIENT_URLS="$protocol://$myip:2379,http://127.0.0.1:2379"
|
||||
ETCD_INITIAL_ADVERTISE_PEER_URLS="$protocol://$myip:2380"
|
||||
ETCD_DISCOVERY="$ETCD_DISCOVERY_URL"
|
||||
EOF
|
||||
|
||||
if [ -n "$HTTP_PROXY" ]; then
|
||||
cat >> /etc/etcd/etcd.conf.yaml <<EOF
|
||||
# HTTP proxy to use for traffic to discovery service.
|
||||
discovery-proxy: $HTTP_PROXY
|
||||
|
||||
EOF
|
||||
fi
|
||||
|
||||
if [ "$TLS_DISABLED" = "False" ]; then
|
||||
|
||||
cat >> /etc/etcd/etcd.conf <<EOF
|
||||
ETCD_CA_FILE=$cert_dir/ca.crt
|
||||
ETCD_TRUSTED_CA_FILE=$cert_dir/ca.crt
|
||||
ETCD_CERT_FILE=$cert_dir/server.crt
|
||||
ETCD_KEY_FILE=$cert_dir/server.key
|
||||
ETCD_CLIENT_CERT_AUTH=true
|
||||
ETCD_PEER_CA_FILE=$cert_dir/ca.crt
|
||||
ETCD_PEER_TRUSTED_CA_FILE=$cert_dir/ca.crt
|
||||
ETCD_PEER_CERT_FILE=$cert_dir/server.crt
|
||||
ETCD_PEER_KEY_FILE=$cert_dir/server.key
|
||||
ETCD_PEER_CLIENT_CERT_AUTH=true
|
||||
cat >> /etc/etcd/etcd.conf.yaml <<EOF
|
||||
client-transport-security:
|
||||
# Path to the client server TLS cert file.
|
||||
cert-file: $cert_dir/server.crt
|
||||
|
||||
# Path to the client server TLS key file.
|
||||
key-file: $cert_dir/server.key
|
||||
|
||||
# Enable client cert authentication.
|
||||
client-cert-auth: true
|
||||
|
||||
# Path to the client server TLS trusted CA cert file.
|
||||
trusted-ca-file: $cert_dir/ca.crt
|
||||
|
||||
peer-transport-security:
|
||||
# Path to the peer server TLS cert file.
|
||||
cert-file: $cert_dir/server.crt
|
||||
|
||||
# Path to the peer server TLS key file.
|
||||
key-file: $cert_dir/server.key
|
||||
|
||||
# Enable peer client cert authentication.
|
||||
client-cert-auth: true
|
||||
|
||||
# Path to the peer server TLS trusted CA cert file.
|
||||
trusted-ca-file: $cert_dir/ca.crt
|
||||
EOF
|
||||
|
||||
fi
|
||||
|
||||
if [ -n "$HTTP_PROXY" ]; then
|
||||
echo "ETCD_DISCOVERY_PROXY=$HTTP_PROXY" >> /etc/etcd/etcd.conf
|
||||
fi
|
||||
|
|
|
@ -21,14 +21,11 @@ if [ ! -z "$NO_PROXY" ]; then
|
|||
export NO_PROXY
|
||||
fi
|
||||
|
||||
_prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
|
||||
|
||||
$ssh_cmd rm -rf /etc/cni/net.d/*
|
||||
$ssh_cmd rm -rf /var/lib/cni/*
|
||||
$ssh_cmd rm -rf /opt/cni/*
|
||||
$ssh_cmd mkdir -p /opt/cni
|
||||
$ssh_cmd mkdir -p /opt/cni/bin
|
||||
$ssh_cmd mkdir -p /etc/cni/net.d/
|
||||
_addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]},{"type":"bind","source":"/var/lib/docker","destination":"/var/lib/docker","options":["bind","rw","slave","mode=755"]}'
|
||||
|
||||
if [ "$NETWORK_DRIVER" = "calico" ]; then
|
||||
echo "net.ipv4.conf.all.rp_filter = 1" >> /etc/sysctl.conf
|
||||
|
@ -49,16 +46,193 @@ fi
|
|||
|
||||
|
||||
mkdir -p /srv/magnum/kubernetes/
|
||||
cat > /srv/magnum/kubernetes/install-kubernetes.sh <<EOF
|
||||
#!/bin/bash -x
|
||||
atomic install --storage ostree --system --set=ADDTL_MOUNTS='${_addtl_mounts}' --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
|
||||
atomic install --storage ostree --system --system-package=no --name=kube-apiserver ${_prefix}kubernetes-apiserver:${KUBE_TAG}
|
||||
atomic install --storage ostree --system --system-package=no --name=kube-controller-manager ${_prefix}kubernetes-controller-manager:${KUBE_TAG}
|
||||
atomic install --storage ostree --system --system-package=no --name=kube-scheduler ${_prefix}kubernetes-scheduler:${KUBE_TAG}
|
||||
atomic install --storage ostree --system --system-package=no --name=kube-proxy ${_prefix}kubernetes-proxy:${KUBE_TAG}
|
||||
cat > /etc/kubernetes/config <<EOF
|
||||
KUBE_LOGTOSTDERR="--logtostderr=true"
|
||||
KUBE_LOG_LEVEL="--v=3"
|
||||
KUBE_MASTER="--master=http://127.0.0.1:8080"
|
||||
EOF
|
||||
cat > /etc/kubernetes/kubelet <<EOF
|
||||
KUBELET_ARGS="--fail-swap-on=false"
|
||||
EOF
|
||||
|
||||
cat > /etc/kubernetes/apiserver <<EOF
|
||||
KUBE_API_ADDRESS="--insecure-bind-address=127.0.0.1"
|
||||
KUBE_ETCD_SERVERS="--etcd-servers=http://127.0.0.1:2379,http://127.0.0.1:4001"
|
||||
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"
|
||||
KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
|
||||
KUBE_API_ARGS=""
|
||||
EOF
|
||||
|
||||
cat > /etc/kubernetes/controller-manager <<EOF
|
||||
KUBE_CONTROLLER_MANAGER_ARGS=""
|
||||
EOF
|
||||
cat > /etc/kubernetes/scheduler<<EOF
|
||||
KUBE_SCHEDULER_ARGS=""
|
||||
EOF
|
||||
cat > /etc/kubernetes/proxy <<EOF
|
||||
KUBE_PROXY_ARGS=""
|
||||
EOF
|
||||
|
||||
|
||||
cat > /etc/systemd/system/kube-apiserver.service <<EOF
|
||||
[Unit]
|
||||
Description=kube-apiserver via Hyperkube
|
||||
[Service]
|
||||
EnvironmentFile=/etc/sysconfig/heat-params
|
||||
EnvironmentFile=/etc/kubernetes/config
|
||||
EnvironmentFile=/etc/kubernetes/apiserver
|
||||
ExecStartPre=/bin/mkdir -p /etc/kubernetes/
|
||||
ExecStartPre=-/usr/bin/podman rm kube-apiserver
|
||||
ExecStartPre=-/bin/bash -c '/usr/bin/podman run --privileged --user root --net host --rm --volume /usr/local/bin:/host/usr/local/bin \${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:\${KUBE_TAG} /bin/sh -c "cp /usr/local/bin/kubectl /host/usr/local/bin/kubectl"'
|
||||
ExecStart=/bin/bash -c '/usr/bin/podman run --name kube-apiserver \\
|
||||
--net host \\
|
||||
--volume /etc/kubernetes:/etc/kubernetes:ro,z \\
|
||||
--volume /usr/lib/os-release:/etc/os-release:ro \\
|
||||
--volume /etc/ssl/certs:/etc/ssl/certs:ro \\
|
||||
--volume /run:/run \\
|
||||
--volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \\
|
||||
\${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:\${KUBE_TAG} \\
|
||||
/hyperkube kube-apiserver \\
|
||||
\$KUBE_LOGTOSTDERR \$KUBE_LOG_LEVEL \$KUBE_ETCD_SERVERS \$KUBE_API_ADDRESS \$KUBE_API_PORT \$KUBELET_PORT \$KUBE_SERVICE_ADDRESSES \$KUBE_ADMISSION_CONTROL \$KUBE_API_ARGS'
|
||||
ExecStop=-/usr/bin/podman stop kube-apiserver
|
||||
Delegate=yes
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
cat > /etc/systemd/system/kube-controller-manager.service <<EOF
|
||||
[Unit]
|
||||
Description=kube-controller-manager via Hyperkube
|
||||
[Service]
|
||||
EnvironmentFile=/etc/sysconfig/heat-params
|
||||
EnvironmentFile=/etc/kubernetes/config
|
||||
EnvironmentFile=/etc/kubernetes/controller-manager
|
||||
ExecStartPre=/bin/mkdir -p /etc/kubernetes/
|
||||
ExecStartPre=-/usr/bin/podman rm kube-controller-manager
|
||||
ExecStart=/bin/bash -c '/usr/bin/podman run --name kube-controller-manager \\
|
||||
--net host \\
|
||||
--volume /etc/kubernetes:/etc/kubernetes:ro,z \\
|
||||
--volume /usr/lib/os-release:/etc/os-release:ro \\
|
||||
--volume /etc/ssl/certs:/etc/ssl/certs:ro \\
|
||||
--volume /run:/run \\
|
||||
--volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \\
|
||||
\${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:\${KUBE_TAG} \\
|
||||
/hyperkube kube-controller-manager \\
|
||||
--secure-port=0 \\
|
||||
\$KUBE_LOGTOSTDERR \$KUBE_LOG_LEVEL \$KUBE_MASTER \$KUBE_CONTROLLER_MANAGER_ARGS'
|
||||
ExecStop=-/usr/bin/podman stop kube-controller-manager
|
||||
Delegate=yes
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
cat > /etc/systemd/system/kube-scheduler.service <<EOF
|
||||
[Unit]
|
||||
Description=kube-scheduler via Hyperkube
|
||||
[Service]
|
||||
EnvironmentFile=/etc/sysconfig/heat-params
|
||||
EnvironmentFile=/etc/kubernetes/config
|
||||
EnvironmentFile=/etc/kubernetes/scheduler
|
||||
ExecStartPre=/bin/mkdir -p /etc/kubernetes/
|
||||
ExecStartPre=-/usr/bin/podman rm kube-scheduler
|
||||
ExecStart=/bin/bash -c '/usr/bin/podman run --name kube-scheduler \\
|
||||
--net host \\
|
||||
--volume /etc/kubernetes:/etc/kubernetes:ro,z \\
|
||||
--volume /usr/lib/os-release:/etc/os-release:ro \\
|
||||
--volume /etc/ssl/certs:/etc/ssl/certs:ro \\
|
||||
--volume /run:/run \\
|
||||
--volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \\
|
||||
\${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:\${KUBE_TAG} \\
|
||||
/hyperkube kube-scheduler \\
|
||||
\$KUBE_LOGTOSTDERR \$KUBE_LOG_LEVEL \$KUBE_MASTER \$KUBE_SCHEDULER_ARGS'
|
||||
ExecStop=-/usr/bin/podman stop kube-scheduler
|
||||
Delegate=yes
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
|
||||
|
||||
cat > /etc/systemd/system/kubelet.service <<EOF
|
||||
[Unit]
|
||||
Description=Kubelet via Hyperkube (System Container)
|
||||
[Service]
|
||||
EnvironmentFile=/etc/sysconfig/heat-params
|
||||
EnvironmentFile=/etc/kubernetes/config
|
||||
EnvironmentFile=/etc/kubernetes/kubelet
|
||||
ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
|
||||
ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
|
||||
ExecStartPre=/bin/mkdir -p /var/lib/calico
|
||||
ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
|
||||
ExecStartPre=/bin/mkdir -p /opt/cni/bin
|
||||
ExecStartPre=-/usr/bin/podman rm kubelet
|
||||
ExecStart=/bin/bash -c '/usr/bin/podman run --name kubelet \\
|
||||
--privileged \\
|
||||
--pid host \\
|
||||
--network host \\
|
||||
--volume /etc/cni/net.d:/etc/cni/net.d:ro,z \\
|
||||
--volume /etc/kubernetes:/etc/kubernetes:ro,z \\
|
||||
--volume /usr/lib/os-release:/etc/os-release:ro \\
|
||||
--volume /etc/ssl/certs:/etc/ssl/certs:ro \\
|
||||
--volume /lib/modules:/lib/modules:ro \\
|
||||
--volume /run:/run \\
|
||||
--volume /sys/fs/cgroup:/sys/fs/cgroup:ro \\
|
||||
--volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \\
|
||||
--volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \\
|
||||
--volume /var/lib/calico:/var/lib/calico \\
|
||||
--volume /var/lib/docker:/var/lib/docker \\
|
||||
--volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \\
|
||||
--volume /var/log:/var/log \\
|
||||
--volume /var/run:/var/run \\
|
||||
--volume /var/run/lock:/var/run/lock:z \\
|
||||
--volume /opt/cni/bin:/opt/cni/bin:z \\
|
||||
\${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:\${KUBE_TAG} \\
|
||||
/hyperkube kubelet \\
|
||||
\$KUBE_LOGTOSTDERR \$KUBE_LOG_LEVEL \$KUBELET_API_SERVER \$KUBELET_ADDRESS \$KUBELET_PORT \$KUBELET_HOSTNAME \$KUBELET_ARGS'
|
||||
ExecStop=-/usr/bin/podman stop kubelet
|
||||
Delegate=yes
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
cat > /etc/systemd/system/kube-proxy.service <<EOF
|
||||
[Unit]
|
||||
Description=kube-proxy via Hyperkube
|
||||
[Service]
|
||||
EnvironmentFile=/etc/sysconfig/heat-params
|
||||
EnvironmentFile=/etc/kubernetes/config
|
||||
EnvironmentFile=/etc/kubernetes/proxy
|
||||
ExecStartPre=/bin/mkdir -p /etc/kubernetes/
|
||||
ExecStartPre=-/usr/bin/podman rm kube-proxy
|
||||
ExecStart=/bin/bash -c '/usr/bin/podman run --name kube-proxy \\
|
||||
--privileged \\
|
||||
--net host \\
|
||||
--volume /etc/kubernetes:/etc/kubernetes:ro,z \\
|
||||
--volume /usr/lib/os-release:/etc/os-release:ro \\
|
||||
--volume /etc/ssl/certs:/etc/ssl/certs:ro \\
|
||||
--volume /run:/run \\
|
||||
--volume /sys/fs/cgroup:/sys/fs/cgroup:ro \\
|
||||
--volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \\
|
||||
--volume /lib/modules:/lib/modules:ro \\
|
||||
--volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \\
|
||||
\${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:\${KUBE_TAG} \\
|
||||
/hyperkube kube-proxy \\
|
||||
\$KUBE_LOGTOSTDERR \$KUBE_LOG_LEVEL \$KUBE_MASTER \$KUBE_PROXY_ARGS'
|
||||
ExecStop=-/usr/bin/podman stop kube-proxy
|
||||
Delegate=yes
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
chmod +x /srv/magnum/kubernetes/install-kubernetes.sh
|
||||
$ssh_cmd "/srv/magnum/kubernetes/install-kubernetes.sh"
|
||||
|
||||
|
||||
CERT_DIR=/etc/kubernetes/certs
|
||||
|
@ -199,7 +373,7 @@ sed -i '
|
|||
sed -i '/^KUBE_SCHEDULER_ARGS=/ s/=.*/="--leader-elect=true"/' /etc/kubernetes/scheduler
|
||||
|
||||
$ssh_cmd mkdir -p /etc/kubernetes/manifests
|
||||
KUBELET_ARGS="--register-node=true --pod-manifest-path=/etc/kubernetes/manifests --cadvisor-port=0 --hostname-override=${INSTANCE_NAME}"
|
||||
KUBELET_ARGS="--register-node=true --pod-manifest-path=/etc/kubernetes/manifests --hostname-override=${INSTANCE_NAME}"
|
||||
KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}pause:3.0"
|
||||
KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}"
|
||||
KUBELET_ARGS="${KUBELET_ARGS} --volume-plugin-dir=/var/lib/kubelet/volumeplugins"
|
||||
|
@ -281,7 +455,7 @@ fi
|
|||
KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only-port=0 --anonymous-auth=false --authorization-mode=Webhook --authentication-token-webhook=true"
|
||||
|
||||
sed -i '
|
||||
/^KUBELET_ADDRESS=/ s/=.*/="--address=${KUBE_NODE_IP}"/
|
||||
/^KUBELET_ADDRESS=/ s/=.*/=""/
|
||||
/^KUBELET_HOSTNAME=/ s/=.*/=""/
|
||||
/^KUBELET_ARGS=/ s|=.*|="'"\$(/etc/kubernetes/get_require_kubeconfig.sh) ${KUBELET_ARGS}"'"|
|
||||
/^KUBELET_ARGS=/ s|=.*|="'"${KUBELET_ARGS}"'"|
|
||||
' /etc/kubernetes/kubelet
|
||||
|
|
|
@ -21,12 +21,11 @@ if [ ! -z "$NO_PROXY" ]; then
|
|||
export NO_PROXY
|
||||
fi
|
||||
|
||||
_prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
|
||||
|
||||
$ssh_cmd rm -rf /etc/cni/net.d/*
|
||||
$ssh_cmd rm -rf /var/lib/cni/*
|
||||
$ssh_cmd rm -rf /opt/cni/*
|
||||
$ssh_cmd mkdir -p /opt/cni
|
||||
$ssh_cmd mkdir -p /opt/cni/bin
|
||||
$ssh_cmd mkdir -p /etc/cni/net.d/
|
||||
_addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]},{"type":"bind","source":"/var/lib/docker","destination":"/var/lib/docker","options":["bind","rw","slave","mode=755"]}'
|
||||
|
||||
|
@ -48,13 +47,91 @@ EOF
|
|||
fi
|
||||
|
||||
mkdir -p /srv/magnum/kubernetes/
|
||||
cat > /srv/magnum/kubernetes/install-kubernetes.sh <<EOF
|
||||
#!/bin/bash -x
|
||||
atomic install --storage ostree --system --system-package=no --set=ADDTL_MOUNTS='${_addtl_mounts}' --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
|
||||
atomic install --storage ostree --system --system-package=no --name=kube-proxy ${_prefix}kubernetes-proxy:${KUBE_TAG}
|
||||
cat > /etc/kubernetes/config <<EOF
|
||||
KUBE_LOGTOSTDERR="--logtostderr=true"
|
||||
KUBE_LOG_LEVEL="--v=3"
|
||||
KUBE_MASTER="--master=http://127.0.0.1:8080"
|
||||
EOF
|
||||
cat > /etc/kubernetes/kubelet <<EOF
|
||||
KUBELET_ARGS="--fail-swap-on=false"
|
||||
EOF
|
||||
cat > /etc/kubernetes/proxy <<EOF
|
||||
KUBE_PROXY_ARGS=""
|
||||
EOF
|
||||
cat > /etc/systemd/system/kubelet.service <<EOF
|
||||
[Unit]
|
||||
Description=Kubelet via Hyperkube (System Container)
|
||||
[Service]
|
||||
EnvironmentFile=/etc/sysconfig/heat-params
|
||||
EnvironmentFile=/etc/kubernetes/config
|
||||
EnvironmentFile=/etc/kubernetes/kubelet
|
||||
ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
|
||||
ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
|
||||
ExecStartPre=/bin/mkdir -p /var/lib/calico
|
||||
ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
|
||||
ExecStartPre=/bin/mkdir -p /opt/cni/bin
|
||||
ExecStartPre=-/usr/bin/podman rm kubelet
|
||||
ExecStart=/bin/bash -c '/usr/bin/podman run --name kubelet \\
|
||||
--privileged \\
|
||||
--pid host \\
|
||||
--network host \\
|
||||
--volume /etc/cni/net.d:/etc/cni/net.d:ro,z \\
|
||||
--volume /etc/kubernetes:/etc/kubernetes:ro,z \\
|
||||
--volume /usr/lib/os-release:/etc/os-release:ro \\
|
||||
--volume /etc/ssl/certs:/etc/ssl/certs:ro \\
|
||||
--volume /lib/modules:/lib/modules:ro \\
|
||||
--volume /run:/run \\
|
||||
--volume /sys/fs/cgroup:/sys/fs/cgroup:ro \\
|
||||
--volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \\
|
||||
--volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \\
|
||||
--volume /var/lib/calico:/var/lib/calico \\
|
||||
--volume /var/lib/docker:/var/lib/docker \\
|
||||
--volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \\
|
||||
--volume /var/log:/var/log \\
|
||||
--volume /var/run:/var/run \\
|
||||
--volume /var/run/lock:/var/run/lock:z \\
|
||||
--volume /opt/cni/bin:/opt/cni/bin:z \\
|
||||
\${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:\${KUBE_TAG} \\
|
||||
/hyperkube kubelet \\
|
||||
\$KUBE_LOGTOSTDERR \$KUBE_LOG_LEVEL \$KUBELET_API_SERVER \$KUBELET_ADDRESS \$KUBELET_PORT \$KUBELET_HOSTNAME \$KUBELET_ARGS'
|
||||
ExecStop=-/usr/bin/podman stop kubelet
|
||||
Delegate=yes
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
cat > /etc/systemd/system/kube-proxy.service <<EOF
|
||||
[Unit]
|
||||
Description=kube-proxy via Hyperkube
|
||||
[Service]
|
||||
EnvironmentFile=/etc/sysconfig/heat-params
|
||||
EnvironmentFile=/etc/kubernetes/config
|
||||
EnvironmentFile=/etc/kubernetes/proxy
|
||||
ExecStartPre=/bin/mkdir -p /etc/kubernetes/
|
||||
ExecStartPre=-/usr/bin/podman rm kube-proxy
|
||||
ExecStart=/bin/bash -c '/usr/bin/podman run --name kube-proxy \\
|
||||
--privileged \\
|
||||
--net host \\
|
||||
--volume /etc/kubernetes:/etc/kubernetes:ro,z \\
|
||||
--volume /usr/lib/os-release:/etc/os-release:ro \\
|
||||
--volume /etc/ssl/certs:/etc/ssl/certs:ro \\
|
||||
--volume /run:/run \\
|
||||
--volume /sys/fs/cgroup:/sys/fs/cgroup:ro \\
|
||||
--volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \\
|
||||
--volume /lib/modules:/lib/modules:ro \\
|
||||
--volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \\
|
||||
\${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:\${KUBE_TAG} \\
|
||||
/hyperkube kube-proxy \\
|
||||
\$KUBE_LOGTOSTDERR \$KUBE_LOG_LEVEL \$KUBE_MASTER \$KUBE_PROXY_ARGS'
|
||||
ExecStop=-/usr/bin/podman stop kube-proxy
|
||||
Delegate=yes
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
chmod +x /srv/magnum/kubernetes/install-kubernetes.sh
|
||||
$ssh_cmd "/srv/magnum/kubernetes/install-kubernetes.sh"
|
||||
|
||||
CERT_DIR=/etc/kubernetes/certs
|
||||
ETCD_SERVER_IP=${ETCD_SERVER_IP:-$KUBE_MASTER_IP}
|
||||
|
@ -139,7 +216,7 @@ sed -i '
|
|||
# the option --hostname-override for kubelet uses the hostname to register the node.
|
||||
# Using any other name will break the load balancer and cinder volume features.
|
||||
mkdir -p /etc/kubernetes/manifests
|
||||
KUBELET_ARGS="--pod-manifest-path=/etc/kubernetes/manifests --cadvisor-port=0 --kubeconfig ${KUBELET_KUBECONFIG} --hostname-override=${INSTANCE_NAME}"
|
||||
KUBELET_ARGS="--pod-manifest-path=/etc/kubernetes/manifests --kubeconfig ${KUBELET_KUBECONFIG} --hostname-override=${INSTANCE_NAME}"
|
||||
KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only-port=0 --anonymous-auth=false --authorization-mode=Webhook --authentication-token-webhook=true"
|
||||
KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}"
|
||||
KUBELET_ARGS="${KUBELET_ARGS} --volume-plugin-dir=/var/lib/kubelet/volumeplugins"
|
||||
|
@ -183,24 +260,13 @@ fi
|
|||
$ssh_cmd systemctl daemon-reload
|
||||
$ssh_cmd systemctl enable docker
|
||||
|
||||
cat > /etc/kubernetes/get_require_kubeconfig.sh <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
KUBE_VERSION=\$(kubelet --version | awk '{print \$2}')
|
||||
min_version=v1.8.0
|
||||
if [[ "\${min_version}" != \$(echo -e "\${min_version}\n\${KUBE_VERSION}" | sort -s -t. -k 1,1 -k 2,2n -k 3,3n | head -n1) && "\${KUBE_VERSION}" != "devel" ]]; then
|
||||
echo "--require-kubeconfig"
|
||||
fi
|
||||
EOF
|
||||
chmod +x /etc/kubernetes/get_require_kubeconfig.sh
|
||||
|
||||
KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
|
||||
|
||||
sed -i '
|
||||
/^KUBELET_ADDRESS=/ s/=.*/="--address=0.0.0.0"/
|
||||
/^KUBELET_HOSTNAME=/ s/=.*/=""/
|
||||
s/^KUBELET_API_SERVER=.*$//
|
||||
/^KUBELET_ARGS=/ s|=.*|="'"\$(/etc/kubernetes/get_require_kubeconfig.sh) ${KUBELET_ARGS}"'"|
|
||||
/^KUBELET_ARGS=/ s|=.*|="'"${KUBELET_ARGS}"'"|
|
||||
' /etc/kubernetes/kubelet
|
||||
|
||||
KUBE_PROXY_ARGS="--kubeconfig=${PROXY_KUBECONFIG} --cluster-cidr=${PODS_NETWORK_CIDR} --hostname-override=${INSTANCE_NAME}"
|
||||
|
|
|
@ -19,7 +19,7 @@ echo "starting services"
|
|||
for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler kubelet kube-proxy; do
|
||||
echo "activating service $service"
|
||||
$ssh_cmd systemctl enable $service
|
||||
$ssh_cmd systemctl --no-block restart $service
|
||||
$ssh_cmd systemctl restart $service
|
||||
done
|
||||
|
||||
# Label self as master
|
||||
|
|
|
@ -9,7 +9,6 @@ ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
|
|||
# be re-created using the flannel-provided subnet).
|
||||
echo "stopping docker"
|
||||
$ssh_cmd systemctl stop docker
|
||||
$ssh_cmd ip link del docker0
|
||||
|
||||
# make sure we pick up any modified unit files
|
||||
$ssh_cmd systemctl daemon-reload
|
||||
|
@ -17,5 +16,5 @@ $ssh_cmd systemctl daemon-reload
|
|||
for service in docker kubelet kube-proxy; do
|
||||
echo "activating service $service"
|
||||
$ssh_cmd systemctl enable $service
|
||||
$ssh_cmd systemctl --no-block start $service
|
||||
$ssh_cmd systemctl start $service
|
||||
done
|
||||
|
|
|
@ -50,12 +50,43 @@ systemctl restart sshd
|
|||
|
||||
|
||||
_prefix="${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}"
|
||||
atomic install \
|
||||
--storage ostree \
|
||||
--system \
|
||||
--system-package no \
|
||||
--set REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt \
|
||||
--name heat-container-agent \
|
||||
"${_prefix}heat-container-agent:${HEAT_CONTAINER_AGENT_TAG}"
|
||||
|
||||
cat > /etc/systemd/system/heat-container-agent.service <<EOF
|
||||
[Unit]
|
||||
Description=Run heat-container-agent
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
ExecStartPre=mkdir -p /var/lib/heat-container-agent
|
||||
ExecStartPre=mkdir -p /var/run/heat-config
|
||||
ExecStartPre=mkdir -p /var/run/os-collect-config
|
||||
ExecStartPre=mkdir -p /opt/stack/os-config-refresh
|
||||
ExecStartPre=mkdir -p /srv/magnum
|
||||
ExecStartPre=-/bin/podman kill heat-container-agent
|
||||
ExecStartPre=-/bin/podman rm heat-container-agent
|
||||
ExecStartPre=-/bin/podman pull docker.io/openstackmagnum/heat-container-agent:train-dev
|
||||
ExecStart=/bin/podman run \\
|
||||
--name heat-container-agent \\
|
||||
--net=host \\
|
||||
--privileged \\
|
||||
--volume /srv/magnum:/srv/magnum \\
|
||||
--volume /opt/stack/os-config-refresh:/opt/stack/os-config-refresh \\
|
||||
--volume /run/systemd:/run/systemd \\
|
||||
--volume /etc/:/etc/ \\
|
||||
--volume /var/lib:/var/lib \\
|
||||
--volume /var/run:/var/run \\
|
||||
--volume /var/log:/var/log \\
|
||||
--volume /tmp:/tmp \\
|
||||
--volume /dev:/dev \\
|
||||
--env REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt \\
|
||||
${_prefix}heat-container-agent:${HEAT_CONTAINER_AGENT_TAG} \\
|
||||
/usr/bin/start-heat-container-agent
|
||||
ExecStop=/bin/podman stop heat-container-agent
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
systemctl enable heat-container-agent
|
||||
systemctl start heat-container-agent
|
||||
|
|
|
@ -4,49 +4,40 @@
|
|||
set -x
|
||||
|
||||
ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
|
||||
kubecontrol="/var/lib/containers/atomic/heat-container-agent.0/rootfs/usr/bin/kubectl --kubeconfig /etc/kubernetes/kubelet-config.yaml"
|
||||
KUBECONFIG="/etc/kubernetes/kubelet-config.yaml"
|
||||
new_kube_tag="$kube_tag_input"
|
||||
|
||||
if [ ${new_kube_tag}!=${KUBE_TAG} ]; then
|
||||
# If there is only one master and this is the master node, skip the drain, just cordon it
|
||||
# If there is only one worker and this is the worker node, skip the drain, just cordon it
|
||||
all_masters=$(${ssh_cmd} ${kubecontrol} get nodes --selector=node-role.kubernetes.io/master= -o name)
|
||||
all_workers=$(${ssh_cmd} ${kubecontrol} get nodes --selector=node-role.kubernetes.io/master!= -o name)
|
||||
all_masters=$(kubectl get nodes --selector=node-role.kubernetes.io/master= -o name)
|
||||
all_workers=$(kubectl get nodes --selector=node-role.kubernetes.io/master!= -o name)
|
||||
if [ "node/${INSTANCE_NAME}" != "${all_masters}" ] && [ "node/${INSTANCE_NAME}" != "${all_workers}" ]; then
|
||||
${ssh_cmd} ${kubecontrol} drain ${INSTANCE_NAME} --ignore-daemonsets --delete-local-data --force
|
||||
kubectl drain ${INSTANCE_NAME} --ignore-daemonsets --delete-local-data --force
|
||||
else
|
||||
${ssh_cmd} ${kubecontrol} cordon ${INSTANCE_NAME}
|
||||
kubectl cordon ${INSTANCE_NAME}
|
||||
fi
|
||||
|
||||
declare -A service_image_mapping
|
||||
service_image_mapping=( ["kubelet"]="kubernetes-kubelet" ["kube-controller-manager"]="kubernetes-controller-manager" ["kube-scheduler"]="kubernetes-scheduler" ["kube-proxy"]="kubernetes-proxy" ["kube-apiserver"]="kubernetes-apiserver" )
|
||||
|
||||
SERVICE_LIST=$($ssh_cmd atomic containers list -f container=kube -q --no-trunc)
|
||||
SERVICE_LIST=$($ssh_cmd podman ps -f name=kube --format {{.Names}})
|
||||
|
||||
for service in ${SERVICE_LIST}; do
|
||||
${ssh_cmd} systemctl stop ${service}
|
||||
${ssh_cmd} podman rm ${service}
|
||||
done
|
||||
|
||||
for service in ${SERVICE_LIST}; do
|
||||
${ssh_cmd} atomic pull --storage ostree "docker.io/openstackmagnum/${service_image_mapping[${service}]}:${new_kube_tag}"
|
||||
done
|
||||
|
||||
for service in ${SERVICE_LIST}; do
|
||||
${ssh_cmd} atomic containers update --rebase docker.io/openstackmagnum/${service_image_mapping[${service}]}:${new_kube_tag} ${service}
|
||||
done
|
||||
|
||||
for service in ${SERVICE_LIST}; do
|
||||
systemctl restart ${service}
|
||||
done
|
||||
|
||||
${ssh_cmd} /var/lib/containers/atomic/heat-container-agent.0/rootfs/usr/bin/kubectl --kubeconfig /etc/kubernetes/kubelet-config.yaml uncordon ${INSTANCE_NAME}
|
||||
|
||||
for service in ${SERVICE_LIST}; do
|
||||
${ssh_cmd} atomic --assumeyes images "delete docker.io/openstackmagnum/${service_image_mapping[${service}]}:${KUBE_TAG}"
|
||||
done
|
||||
|
||||
${ssh_cmd} atomic images prune
|
||||
|
||||
# Appending the new KUBE_TAG into the heat-parms to log and indicate the current k8s version
|
||||
${ssh_cmd} podman rmi ${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:${KUBE_TAG}
|
||||
echo "KUBE_TAG=$new_kube_tag" >> /etc/sysconfig/heat-params
|
||||
|
||||
for service in ${SERVICE_LIST}; do
|
||||
${ssh_cmd} systemctl start ${service}
|
||||
done
|
||||
|
||||
i=0
|
||||
until kubectl uncordon ${INSTANCE_NAME}
|
||||
do
|
||||
((i++))
|
||||
[ $i -lt 30 ] || break;
|
||||
echo "Trying to uncordon node..."
|
||||
sleep 5s
|
||||
done
|
||||
fi
|
||||
|
|
|
@ -434,7 +434,7 @@ parameters:
|
|||
etcd_tag:
|
||||
type: string
|
||||
description: tag of the etcd system container
|
||||
default: v3.2.7
|
||||
default: 3.2.26
|
||||
|
||||
coredns_tag:
|
||||
type: string
|
||||
|
|
Loading…
Reference in New Issue