From 6f0d8b12ee7c69ee01a87fa9d384b4d6c87f9be1 Mon Sep 17 00:00:00 2001 From: Kevin Fox Date: Thu, 20 Oct 2016 10:57:37 -0700 Subject: [PATCH] Multinode Gate & Misc fixes This PS adds support for deploying a working gate in a 2-node arrangement. It also: * Removes the explicit cinderv2 endpoint override as kolla now is patched * Uses zuul-cloner for faster git access. * Turns off keystone caching to make the gate reliable. Keystone caching is racy. Change-Id: I9aaf4d1f85df78eca2295928ed1435de33ef1c4e --- tests/bin/ceph_workflow.sh | 4 +++ tests/bin/endpoint_test.sh | 2 ++ tests/bin/fix_gate_iptables.sh | 2 +- tests/bin/gate_capture_logs.sh | 4 +++ tests/bin/setup_canal.sh | 2 +- tests/bin/setup_config.sh | 17 +++++++++- tools/setup_gate.sh | 60 +++++++++++++++++++++++++++++++--- tools/setup_kubernetes.sh | 32 +++++++++++++----- 8 files changed, 107 insertions(+), 16 deletions(-) diff --git a/tests/bin/ceph_workflow.sh b/tests/bin/ceph_workflow.sh index 3eef86f60..364a1f0a2 100755 --- a/tests/bin/ceph_workflow.sh +++ b/tests/bin/ceph_workflow.sh @@ -120,6 +120,10 @@ $DIR/tools/wait_for_pods.sh kolla kollakube res create pod neutron-dhcp-agent neutron-l3-agent-network \ neutron-openvswitch-agent-network neutron-metadata-agent-network +[ "x$1" == "xceph-multi" ] && kollakube res \ + create pod openvswitch-ovsdb-compute openvswitch-vswitchd-compute \ + neutron-openvswitch-agent-compute + kollakube res create bootstrap openvswitch-set-external-ip kollakube res create pod nova-libvirt kollakube res create pod nova-compute diff --git a/tests/bin/endpoint_test.sh b/tests/bin/endpoint_test.sh index 0edef171f..6416fccda 100755 --- a/tests/bin/endpoint_test.sh +++ b/tests/bin/endpoint_test.sh @@ -9,3 +9,5 @@ OS_TOKEN=$(openstack token issue -f value -c id) curl -H "X-Auth-Token:$OS_TOKEN" $OS_AUTH_URL/endpoints -o /tmp/$$ jq -r '.endpoints[] | .service_id' /tmp/$$ | sort | uniq -c > /tmp/$$.1 awk '{if($1 != 3){exit -1}}' /tmp/$$.1 || endpoints_dump_and_fail +[ -d $WORKSPACE/logs ] && cp /tmp/$$ $WORKSPACE/logs/endpoints.txt +[ -d $WORKSPACE/logs ] && cp /tmp/$$.1 $WORKSPACE/logs/endpoints1.txt diff --git a/tests/bin/fix_gate_iptables.sh b/tests/bin/fix_gate_iptables.sh index e2ae2bce6..598b65001 100755 --- a/tests/bin/fix_gate_iptables.sh +++ b/tests/bin/fix_gate_iptables.sh @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/bash -xe l=$(sudo iptables -L INPUT --line-numbers | grep openstack-INPUT | \ awk '{print $1}') diff --git a/tests/bin/gate_capture_logs.sh b/tests/bin/gate_capture_logs.sh index a5f4d30d1..498112b50 100755 --- a/tests/bin/gate_capture_logs.sh +++ b/tests/bin/gate_capture_logs.sh @@ -84,4 +84,8 @@ kubectl get secret nova-libvirt-bin --namespace=kolla -o yaml openstack volume list > $WORKSPACE/logs/volumes.txt cp -a /etc/kolla $WORKSPACE/logs/ cp /usr/bin/rbd $WORKSPACE/logs/rbd.sh +[ -f /etc/nodepool/sub_nodes_private ] && cat /etc/nodepool/sub_nodes_private | while read line; do + ssh $line sudo journalctl -u kubelet > $WORKSPACE/logs/kubelet-$line.txt + ssh $line ps ax > $WORKSPACE/logs/ps-$line.txt +done exit -1 diff --git a/tests/bin/setup_canal.sh b/tests/bin/setup_canal.sh index 9a0f8540a..024e6435c 100755 --- a/tests/bin/setup_canal.sh +++ b/tests/bin/setup_canal.sh @@ -7,7 +7,7 @@ url="$url/k8s-install/kubeadm/canal.yaml" curl "$url" -o /tmp/canal.yaml -sed -i "s@192.168.0.0/16@172.16.130.0/23@" /tmp/canal.yaml +sed -i "s@192.168.0.0/16@172.16.130.0/22@" /tmp/canal.yaml sed -i "s@100.78.232.136@172.16.128.100@" /tmp/canal.yaml kubectl create -f /tmp/canal.yaml diff --git a/tests/bin/setup_config.sh b/tests/bin/setup_config.sh index 63b7b47ac..1b907e172 100755 --- a/tests/bin/setup_config.sh +++ b/tests/bin/setup_config.sh @@ -2,6 +2,8 @@ NODE=$(hostname -s) +TYPE="$2" + echo "kolla_base_distro: $1" >> kolla/etc/kolla/globals.yml cat tests/conf/ceph-all-in-one/kolla_config >> kolla/etc/kolla/globals.yml IP=172.18.0.1 @@ -16,15 +18,28 @@ cat tests/conf/ceph-all-in-one/kolla_kubernetes_config \ sed -i "s/initial_mon:.*/initial_mon: $NODE/" \ etc/kolla-kubernetes/kolla-kubernetes.yml +if [ "x$TYPE" == "xceph-multi" ]; then + interface=$(netstat -ie | grep -B1 \ + $(cat /etc/nodepool/primary_node_private) \ + | head -n 1 | awk -F: '{print $1}') + echo "tunnel_interface: $interface" >> kolla/etc/kolla/globals.yml + echo "storage_interface: $interface" >> \ + etc/kolla-kubernetes/kolla-kubernetes.yml + sed -i "s/172.17.0.1/$(cat /etc/nodepool/primary_node_private)/" \ + etc/kolla-kubernetes/kolla-kubernetes.yml +fi + kolla/tools/generate_passwords.py kolla/tools/kolla-ansible genconfig -crudini --set /etc/kolla/nova-compute/nova.conf cinder catalog_info volumev2:cinderv2:internalURL crudini --set /etc/kolla/nova-compute/nova.conf libvirt virt_type qemu crudini --set /etc/kolla/nova-compute/nova.conf libvirt rbd_user nova UUID=$(awk '{if($1 == "rbd_secret_uuid:"){print $2}}' /etc/kolla/passwords.yml) crudini --set /etc/kolla/nova-compute/nova.conf libvirt rbd_secret_uuid $UUID +# Keystone does not seem to invalidate its cache on entry point addition. +crudini --set /etc/kolla/keystone/keystone.conf cache enabled False + sed -i 's/log_outputs = "3:/log_outputs = "1:/' /etc/kolla/nova-libvirt/libvirtd.conf sed -i 's/log_level = 3/log_level = 1/' /etc/kolla/nova-libvirt/libvirtd.conf diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh index 74030acf0..0ca160454 100755 --- a/tools/setup_gate.sh +++ b/tools/setup_gate.sh @@ -5,6 +5,7 @@ trap 'tests/bin/gate_capture_logs.sh "$?"' ERR mkdir -p $WORKSPACE/logs/ +env > $WORKSPACE/logs/env sudo iptables-save > $WORKSPACE/logs/iptables-before.txt tests/bin/fix_gate_iptables.sh @@ -12,7 +13,18 @@ tests/bin/fix_gate_iptables.sh virtualenv .venv . .venv/bin/activate +cat > /tmp/clonemap <<"EOF" +clonemap: + - name: openstack/kolla + dest: kolla +EOF + +[ -x /usr/zuul-env/bin/zuul-cloner ] && \ +/usr/zuul-env/bin/zuul-cloner -m /tmp/clonemap --workspace `pwd` \ + --branch master --cache-dir /opt/git git://git.openstack.org \ + openstack/kolla || \ git clone https://github.com/openstack/kolla.git + sudo ln -s `pwd`/kolla/etc/kolla /etc/kolla sudo ln -s `pwd`/kolla /usr/share/kolla sudo ln -s `pwd`/etc/kolla-kubernetes /etc/kolla-kubernetes @@ -34,17 +46,57 @@ popd pip install -r requirements.txt pip install . -tests/bin/setup_config.sh "$2" +tests/bin/setup_config.sh "$2" "$4" tests/bin/setup_gate_loopback.sh -tools/setup_kubernetes.sh +tools/setup_kubernetes.sh master kubectl taint nodes --all dedicated- +# Turn up kube-proxy logging +# kubectl -n kube-system get ds -l 'component=kube-proxy-amd64' -o json \ +# | sed 's/--v=4/--v=9/' \ +# | kubectl apply -f - && kubectl -n kube-system delete pods -l 'component=kube-proxy-amd64' + +if [ "x$4" == "xceph-multi" ]; then + NODES=1 + cat /etc/nodepool/sub_nodes_private | while read line; do + NODES=$((NODES+1)) + echo $line + scp tools/setup_kubernetes.sh $line: + scp tests/bin/fix_gate_iptables.sh $line: + scp /usr/bin/kubectl $line:kubectl + ssh $line bash fix_gate_iptables.sh + ssh $line sudo iptables-save > $WORKSPACE/logs/iptables-$line.txt + ssh $line sudo setenforce 0 + ssh $line sudo mv kubectl /usr/bin/ + ssh $line bash setup_kubernetes.sh slave "$(cat /etc/kubernetes/token.txt)" "$(cat /etc/kubernetes/ip.txt)" + ssh $line sudo sed -i "'s@KUBELET_EXTRA_ARGS=@KUBELET_EXTRA_ARGS=--hostname-override=$line @'" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + ssh $line sudo systemctl daemon-reload + ssh $line sudo systemctl restart kubelet + set +xe + count=0 + while true; do + c=$(kubectl get nodes --no-headers=true | wc -l) + [ $c -ge $NODES ] && break + count=$((count+1)) + [ $count -gt 30 ] && break + sleep 1 + done + [ $count -gt 30 ] && echo Node failed to join. && exit -1 + set -xe + kubectl get nodes + kubectl label node $line kolla_compute=true + done +fi + NODE=$(hostname -s) kubectl label node $NODE kolla_controller=true -kubectl label node $NODE kolla_compute=true + +if [ "x$4" != "xceph-multi" ]; then + kubectl label node $NODE kolla_compute=true +fi tests/bin/setup_canal.sh @@ -74,7 +126,7 @@ kubectl exec ceph-admin -c main --namespace=kolla -- /bin/bash \ tools/setup_simple_ceph_users.sh tools/setup_rbd_volumes.sh --yes-i-really-really-mean-it -tests/bin/ceph_workflow.sh +tests/bin/ceph_workflow.sh "$4" . ~/keystonerc_admin kubectl get pods --namespace=kolla diff --git a/tools/setup_kubernetes.sh b/tools/setup_kubernetes.sh index 45f4b126b..6307ecfd9 100755 --- a/tools/setup_kubernetes.sh +++ b/tools/setup_kubernetes.sh @@ -27,7 +27,20 @@ EOF fi cat >> /tmp/setup.$$ <<"EOF" systemctl start docker -kubeadm init --service-cidr 172.16.128.0/24 +EOF +if [ "$1" == "master" ]; then + cat >> /tmp/setup.$$ <<"EOF" +kubeadm init --service-cidr 172.16.128.0/24 --api-advertise-addresses $(cat /etc/nodepool/primary_node_private) | tee /tmp/kubeout +grep 'kubeadm join --token' /tmp/kubeout | awk '{print $4}' > /etc/kubernetes/token.txt +grep 'kubeadm join --token' /tmp/kubeout | awk '{print $5}' > /etc/kubernetes/ip.txt +rm -f /tmp/kubeout +EOF +else + cat >> /tmp/setup.$$ <> /tmp/setup.$$ <<"EOF" sed -i 's/100.64.0.10/172.16.128.10/g' /etc/systemd/system/kubelet.service.d/10-kubeadm.conf systemctl daemon-reload systemctl stop kubelet @@ -37,11 +50,12 @@ EOF sudo bash /tmp/setup.$$ sudo docker ps -a -count=0 -while true; do - kubectl get pods > /dev/null 2>&1 && break || true - sleep 1 - count=$((count + 1)) - [ $count -gt 30 ] && echo kube-apiserver failed to come back up. && exit -1 -done - +if [ "$1" == "master" ]; then + count=0 + while true; do + kubectl get pods > /dev/null 2>&1 && break || true + sleep 1 + count=$((count + 1)) + [ $count -gt 30 ] && echo kube-apiserver failed to come back up. && exit -1 + done +fi