From e3203bd7fe373f530d6e4d4498619ca9aa75c16f Mon Sep 17 00:00:00 2001 From: Gage Hugo Date: Wed, 8 Sep 2021 15:25:17 -0500 Subject: [PATCH] Improve osh-infra-deploy helm v3 job This change improves the osh-infra-deploy job to successfully deploy minikube with helm v3 along with the necessary namespaces. Future changes will modify the install scripts for each job to make them helm v3 compatible. Change-Id: I08a94046f86f7c92be7580fbf10751150d2fcecc --- tools/gate/deploy-k8s.sh | 100 +++++++++++++++++++++------------------ zuul.d/jobs.yaml | 4 ++ 2 files changed, 59 insertions(+), 45 deletions(-) diff --git a/tools/gate/deploy-k8s.sh b/tools/gate/deploy-k8s.sh index 549a32376..c84a6380d 100755 --- a/tools/gate/deploy-k8s.sh +++ b/tools/gate/deploy-k8s.sh @@ -14,63 +14,52 @@ set -ex : "${HELM_VERSION:="v3.6.3"}" -: "${KUBE_VERSION:="v1.22.0"}" +: "${KUBE_VERSION:="v1.21.5"}" : "${MINIKUBE_VERSION:="v1.22.0"}" : "${CALICO_VERSION:="v3.20"}" : "${YQ_VERSION:="v4.6.0"}" -: "${HTTP_PROXY:=""}" -: "${HTTPS_PROXY:=""}" -: "${NO_PROXY:=""}" - export DEBCONF_NONINTERACTIVE_SEEN=true export DEBIAN_FRONTEND=noninteractive sudo swapoff -a -# Note: Including fix from https://review.opendev.org/c/openstack/openstack-helm-infra/+/763619/ echo "DefaultLimitMEMLOCK=16384" | sudo tee -a /etc/systemd/system.conf sudo systemctl daemon-reexec -# Function to help generate a resolv.conf formatted file. -# Arguments are positional: -# 1st is location of file to be generated -# 2nd is a custom nameserver that should be used exclusively if avalible. -function generate_resolvconf() { - local target - target="${1}" - local priority_nameserver - priority_nameserver="${2}" - if [[ ${priority_nameserver} ]]; then - sudo -E tee "${target}" < /etc/resolv.conf" + if [ -z "${HTTP_PROXY}" ]; then + sudo bash -c "printf 'nameserver 8.8.8.8\nnameserver 8.8.4.4\n' > /run/systemd/resolve/resolv.conf" + sudo bash -c "printf 'nameserver 8.8.8.8\nnameserver 8.8.4.4\n' >> /etc/resolv.conf" else - sudo -E tee --append "${target}" < /run/systemd/resolve/resolv.conf" + sudo bash -c "echo \"${old_ns}\" >> /etc/resolv.conf" fi + + for file in /etc/resolv.conf /run/systemd/resolve/resolv.conf; do + sudo bash -c "echo 'search svc.cluster.local cluster.local' >> ${file}" + sudo bash -c "echo 'options ndots:5 timeout:1 attempts:1' >> ${file}" + done } # NOTE: Clean Up hosts file sudo sed -i '/^127.0.0.1/c\127.0.0.1 localhost localhost.localdomain localhost4localhost4.localdomain4' /etc/hosts sudo sed -i '/^::1/c\::1 localhost6 localhost6.localdomain6' /etc/hosts +configure_resolvconf + # shellcheck disable=SC1091 . /etc/os-release @@ -83,8 +72,7 @@ sudo add-apt-repository \ stable" # NOTE: Configure docker -docker_resolv="$(mktemp -d)/resolv.conf" -generate_resolvconf "${docker_resolv}" +docker_resolv="/run/systemd/resolve/resolv.conf" docker_dns_list="$(awk '/^nameserver/ { printf "%s%s",sep,"\"" $NF "\""; sep=", "} END{print ""}' "${docker_resolv}")" sudo -E mkdir -p /etc/docker @@ -155,9 +143,6 @@ sudo -E bash -c \ sudo -E mv "${TMP_DIR}"/helm /usr/local/bin/helm rm -rf "${TMP_DIR}" -sudo -E mkdir -p /etc/kubernetes -generate_resolvconf /etc/kubernetes/kubelet_resolv.conf - # NOTE: Deploy kubernetes using minikube. A CNI that supports network policy is # required for validation; use calico for simplicity. sudo -E minikube config set kubernetes-version "${KUBE_VERSION}" @@ -176,8 +161,8 @@ sudo -E minikube start \ --extra-config=controller-manager.cluster-cidr=192.168.0.0/16 \ --extra-config=kube-proxy.mode=ipvs \ --extra-config=apiserver.service-node-port-range=1-65535 \ - --extra-config=kubelet.resolv-conf=/etc/kubernetes/kubelet_resolv.conf \ --extra-config=kubelet.cgroup-driver=systemd \ + --extra-config=kubelet.resolv-conf=/run/systemd/resolve/resolv.conf \ --embed-certs sudo -E systemctl enable --now kubelet @@ -231,7 +216,32 @@ kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app # Remove stable repo, if present, to improve build time helm repo remove stable || true -# Add labels to the core namespaces +# Add labels to the core namespaces & nodes kubectl label --overwrite namespace default name=default kubectl label --overwrite namespace kube-system name=kube-system -kubectl label --overwrite namespace kube-public name=kube-public \ No newline at end of file +kubectl label --overwrite namespace kube-public name=kube-public +kubectl label nodes --all openstack-control-plane=enabled +kubectl label nodes --all openstack-compute-node=enabled +kubectl label nodes --all openvswitch=enabled +kubectl label nodes --all linuxbridge=enabled +kubectl label nodes --all ceph-mon=enabled +kubectl label nodes --all ceph-osd=enabled +kubectl label nodes --all ceph-mds=enabled +kubectl label nodes --all ceph-rgw=enabled +kubectl label nodes --all ceph-mgr=enabled + +for NAMESPACE in ceph openstack osh-infra; do +tee /tmp/${NAMESPACE}-ns.yaml << EOF +apiVersion: v1 +kind: Namespace +metadata: + labels: + kubernetes.io/metadata.name: ${NAMESPACE} + name: ${NAMESPACE} + name: ${NAMESPACE} +EOF + +kubectl create -f /tmp/${NAMESPACE}-ns.yaml +done + +make all diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index accc487b7..c1be464db 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -53,6 +53,10 @@ post-run: playbooks/osh-infra-collect-logs.yaml nodeset: openstack-helm-single-node vars: + osh_params: + openstack_release: train + container_distro_name: ubuntu + container_distro_version: bionic gate_scripts_relative_path: ../openstack-helm-infra gate_scripts: - ./tools/deployment/common/000-install-packages.sh