From 5b1879aa099cde8d1451a2f7614aa264eff7beb3 Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Sun, 24 Mar 2024 22:23:49 -0500 Subject: [PATCH] Refactor deploy-env role - Make it less mixed. Each task file deploys one feature. - Deploy Metallb - Deploy Openstack provider network gateway Change-Id: I41f0353b286f817cb562b3bd59992e4baa473568 --- roles/deploy-env/README.md | 36 ++++---- roles/deploy-env/defaults/main.yaml | 17 ++++ roles/deploy-env/files/calico_patch.yaml | 3 +- roles/deploy-env/files/cluster_resolv.conf | 1 + roles/deploy-env/files/kubeadm_config.yaml | 2 + roles/deploy-env/files/nginx_tcp_proxy.conf | 25 ++++++ roles/deploy-env/files/resolv.conf | 5 +- roles/deploy-env/tasks/calico.yaml | 55 ++++++++++++ .../tasks/client_cluster_tunnel.yaml | 73 ++++++++++++++++ roles/deploy-env/tasks/containerd.yaml | 24 ++---- roles/deploy-env/tasks/control-plane.yaml | 84 ------------------- roles/deploy-env/tasks/coredns_resolver.yaml | 37 ++++++++ roles/deploy-env/tasks/k8s_client.yaml | 54 ++++++++++++ .../{common_k8s.yaml => k8s_common.yaml} | 74 ++++++---------- roles/deploy-env/tasks/k8s_control_plane.yaml | 39 +++++++++ roles/deploy-env/tasks/main.yaml | 75 +++++++++++------ roles/deploy-env/tasks/metallb.yaml | 64 ++++++++++++++ .../tasks/openstack_metallb_endpoint.yaml | 77 +++++++++++++++++ .../tasks/openstack_provider_gateway.yaml | 78 +++++++++++++++++ roles/deploy-env/tasks/prerequisites.yaml | 77 ++++++++--------- zuul.d/jobs.yaml | 19 +++++ zuul.d/project.yaml | 3 + 22 files changed, 687 insertions(+), 235 deletions(-) create mode 100644 roles/deploy-env/files/cluster_resolv.conf create mode 100644 roles/deploy-env/files/nginx_tcp_proxy.conf create mode 100644 roles/deploy-env/tasks/calico.yaml create mode 100644 roles/deploy-env/tasks/client_cluster_tunnel.yaml delete mode 100644 roles/deploy-env/tasks/control-plane.yaml create mode 100644 roles/deploy-env/tasks/coredns_resolver.yaml create mode 100644 roles/deploy-env/tasks/k8s_client.yaml rename roles/deploy-env/tasks/{common_k8s.yaml => k8s_common.yaml} (59%) create mode 100644 roles/deploy-env/tasks/k8s_control_plane.yaml create mode 100644 roles/deploy-env/tasks/metallb.yaml create mode 100644 roles/deploy-env/tasks/openstack_metallb_endpoint.yaml create mode 100644 roles/deploy-env/tasks/openstack_provider_gateway.yaml diff --git a/roles/deploy-env/README.md b/roles/deploy-env/README.md index fccb39a04..00bd7b188 100644 --- a/roles/deploy-env/README.md +++ b/roles/deploy-env/README.md @@ -1,8 +1,9 @@ This role is used to deploy test environment which includes - install necessary prerequisites including Helm - deploy Containerd and a container runtime for Kubernetes -- deploy Kubernetes using Kubeadm with a single control plain node +- deploy Kubernetes using Kubeadm with a single control plane node - install Calico as a Kubernetes networking +- establish tunnel between primary node and K8s control plane ndoe The role works both for singlenode and multinode inventories and assumes the inventory has the node called `primary` and the group called `nodes`. @@ -11,27 +12,32 @@ See for example: ```yaml all: + vars: + ansible_port: 22 + ansible_user: ubuntu + ansible_ssh_private_key_file: /home/ubuntu/.ssh/id_rsa + ansible_ssh_extra_args: -o StrictHostKeyChecking=no children: - ungrouped: + primary: hosts: primary: - ansible_port: 22 ansible_host: 10.10.10.10 - ansible_user: ubuntu - ansible_ssh_private_key_file: /home/ubuntu/.ssh/id_rsa - ansible_ssh_extra_args: -o StrictHostKeyChecking=no - nodes: + k8s_cluster: hosts: node-1: - ansible_port: 22 ansible_host: 10.10.10.11 - ansible_user: ubuntu - ansible_ssh_private_key_file: /home/ubuntu/.ssh/id_rsa - ansible_ssh_extra_args: -o StrictHostKeyChecking=no node-2: - ansible_port: 22 ansible_host: 10.10.10.12 - ansible_user: ubuntu - ansible_ssh_private_key_file: /home/ubuntu/.ssh/id_rsa - ansible_ssh_extra_args: -o StrictHostKeyChecking=no + node-3: + ansible_host: 10.10.10.13 + k8s_control-plane: + hosts: + node-1: + ansible_host: 10.10.10.11 + k8s_nodes: + hosts: + node-2: + ansible_host: 10.10.10.12 + node-3: + ansible_host: 10.10.10.13 ``` diff --git a/roles/deploy-env/defaults/main.yaml b/roles/deploy-env/defaults/main.yaml index 9ff9ee10f..ba637dd2d 100644 --- a/roles/deploy-env/defaults/main.yaml +++ b/roles/deploy-env/defaults/main.yaml @@ -35,4 +35,21 @@ loopback_setup: false loopback_device: /dev/loop100 loopback_image: /var/lib/openstack-helm/ceph-loop.img loopback_image_size: 12G + +metallb_setup: false +metallb_pool_cidr: "172.24.128.0/24" +metallb_openstack_endpoint_cidr: "172.24.128.100/24" + +client_ssh_user: zuul +client_ssh_key_file: /home/zuul/.ssh/id_rsa + +cluster_ssh_user: zuul + +openstack_provider_gateway_setup: false +openstack_provider_network_cidr: "172.24.4.0/24" +openstack_provider_gateway_cidr: "172.24.4.1/24" + +tunnel_network_cidr: "172.24.5.0/24" +tunnel_client_cidr: "172.24.5.2/24" +tunnel_cluster_cidr: "172.24.5.1/24" ... diff --git a/roles/deploy-env/files/calico_patch.yaml b/roles/deploy-env/files/calico_patch.yaml index cdb38bb15..bdada7422 100644 --- a/roles/deploy-env/files/calico_patch.yaml +++ b/roles/deploy-env/files/calico_patch.yaml @@ -15,9 +15,8 @@ spec: value: "9091" - name: FELIX_IGNORELOOSERPF value: "true" - # We assign IP on br-ex interface while testing the deployed Openstack cluster and # we need Calico to skip this interface while discovering the # network changes on the host to prevent announcing unnecessary networks. - name: IP_AUTODETECTION_METHOD - value: "skip-interface=br-ex" + value: "skip-interface=br-ex|provider.*|client.*" ... diff --git a/roles/deploy-env/files/cluster_resolv.conf b/roles/deploy-env/files/cluster_resolv.conf new file mode 100644 index 000000000..eb91d36e0 --- /dev/null +++ b/roles/deploy-env/files/cluster_resolv.conf @@ -0,0 +1 @@ +nameserver 10.96.0.10 diff --git a/roles/deploy-env/files/kubeadm_config.yaml b/roles/deploy-env/files/kubeadm_config.yaml index 8c3d8ef91..e314a2dc7 100644 --- a/roles/deploy-env/files/kubeadm_config.yaml +++ b/roles/deploy-env/files/kubeadm_config.yaml @@ -2,6 +2,8 @@ apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration mode: ipvs +ipvs: + strictARP: true ... --- apiVersion: kubeadm.k8s.io/v1beta3 diff --git a/roles/deploy-env/files/nginx_tcp_proxy.conf b/roles/deploy-env/files/nginx_tcp_proxy.conf new file mode 100644 index 000000000..3d64369df --- /dev/null +++ b/roles/deploy-env/files/nginx_tcp_proxy.conf @@ -0,0 +1,25 @@ +user nginx; +worker_processes auto; + +error_log /dev/stdout warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +stream { + access_log off; + + server { + listen {{ openstack_provider_gateway_cidr | ipaddr('address') }}:80; + proxy_pass {{ metallb_openstack_endpoint_cidr | ipaddr('address') }}:80; + proxy_bind {{ openstack_provider_gateway_cidr | ipaddr('address') }} transparent; + } + + server { + listen {{ openstack_provider_gateway_cidr | ipaddr('address') }}:443; + proxy_pass {{ metallb_openstack_endpoint_cidr | ipaddr('address') }}:443; + proxy_bind {{ openstack_provider_gateway_cidr | ipaddr('address') }} transparent; + } +} diff --git a/roles/deploy-env/files/resolv.conf b/roles/deploy-env/files/resolv.conf index 5f9818c77..12f016893 100644 --- a/roles/deploy-env/files/resolv.conf +++ b/roles/deploy-env/files/resolv.conf @@ -1,4 +1 @@ -nameserver 8.8.8.8 -nameserver 8.8.4.4 -search svc.cluster.local cluster.local -options ndots:5 timeout:1 attempts:1 +nameserver {{ nameserver_ip }} diff --git a/roles/deploy-env/tasks/calico.yaml b/roles/deploy-env/tasks/calico.yaml new file mode 100644 index 000000000..f79d6311c --- /dev/null +++ b/roles/deploy-env/tasks/calico.yaml @@ -0,0 +1,55 @@ +--- +# We download Calico manifest on all nodes because we then want to download +# Calico images BEFORE deploying it, so that `kubectl wait` timeout +# for `k8s-app=kube-dns` isn't reached by slow download speeds +- name: Download Calico manifest + when: inventory_hostname in (groups['k8s_cluster'] | default([])) + shell: | + curl -LSs {{ calico_manifest_url }} -o /tmp/calico.yaml + sed -i -e 's#docker.io/calico/#quay.io/calico/#g' /tmp/calico.yaml + export CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock + export IMAGE_SERVICE_ENDPOINT=unix:///run/containerd/containerd.sock + awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} crictl pull {} + args: + executable: /bin/bash + +- name: Deploy Calico + become: false + when: inventory_hostname in (groups['primary'] | default([])) + block: + - name: Download Calico manifest + shell: | + if [[ ! -f /tmp/calico.yaml ]]; then + curl -LSs {{ calico_manifest_url }} -o /tmp/calico.yaml + sed -i -e 's#docker.io/calico/#quay.io/calico/#g' /tmp/calico.yaml + fi + args: + executable: /bin/bash + + - name: Deploy Calico + command: kubectl apply -f /tmp/calico.yaml + + - name: Sleep before trying to check Calico pods + pause: + seconds: 30 + + - name: Wait for Calico pods ready + command: kubectl -n kube-system wait --timeout=20s --for=condition=Ready pods -l k8s-app=calico-node + register: calico_pods_wait + until: calico_pods_wait is succeeded + retries: 10 + + - name: Prepare Calico patch + copy: + src: files/calico_patch.yaml + dest: /tmp/calico_patch.yaml + + - name: Patch Calico + command: kubectl -n kube-system patch daemonset calico-node --patch-file /tmp/calico_patch.yaml + + - name: Wait for Calico pods ready (after patch) + command: kubectl -n kube-system wait --timeout=20s --for=condition=Ready pods -l k8s-app=calico-node + register: calico_pods_wait + until: calico_pods_wait is succeeded + retries: 10 +... diff --git a/roles/deploy-env/tasks/client_cluster_tunnel.yaml b/roles/deploy-env/tasks/client_cluster_tunnel.yaml new file mode 100644 index 000000000..41daac0bd --- /dev/null +++ b/roles/deploy-env/tasks/client_cluster_tunnel.yaml @@ -0,0 +1,73 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Set cluster IP + set_fact: + cluster_default_ip: "{{ (groups['k8s_control_plane'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']))[0] }}" + +- name: Set client IP + set_fact: + client_default_ip: "{{ (groups['primary'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']))[0] }}" + +- name: Setup wireguard tunnel between primary and cluster control-plane node + when: (groups['primary'] | difference(groups['k8s_control_plane']) | length > 0) + block: + - name: Generate wireguard key pair + shell: | + wg genkey | tee /root/wg-private-key | wg pubkey > /root/wg-public-key + chmod 600 /root/wg-private-key + when: (inventory_hostname in (groups['primary'] | default([]))) or (inventory_hostname in (groups['k8s_control_plane'] | default([]))) + + - name: Register public wireguard key variable + command: cat /root/wg-public-key + register: wg_public_key + when: (inventory_hostname in (groups['primary'] | default([]))) or (inventory_hostname in (groups['k8s_control_plane'] | default([]))) + + - name: Set primary wireguard public key + set_fact: + client_wg_public_key: "{{ (groups['primary'] | map('extract', hostvars, ['wg_public_key', 'stdout']))[0] }}" + when: inventory_hostname in (groups['k8s_control_plane'] | default([])) + + - name: Set cluster wireguard public key + set_fact: + cluster_wg_public_key: "{{ (groups['k8s_control_plane'] | map('extract', hostvars, ['wg_public_key', 'stdout']))[0] }}" + when: inventory_hostname in (groups['primary'] | default([])) + + - name: Set up wireguard tunnel on cluster control-plane node + shell: | + cat > /tmp/configure_cluster_tunnel.sh < /tmp/configure_client_tunnel.sh < "${PATCH}" - kubectl patch configmap coredns -n kube-system --patch-file "${PATCH}" - kubectl set image deployment coredns -n kube-system "coredns=registry.k8s.io/coredns/coredns:v1.9.4" - kubectl rollout restart -n kube-system deployment/coredns - sleep 10 - kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=kube-dns - rm -f "${PATCH}" - args: - executable: /bin/bash -... diff --git a/roles/deploy-env/tasks/coredns_resolver.yaml b/roles/deploy-env/tasks/coredns_resolver.yaml new file mode 100644 index 000000000..e540369ae --- /dev/null +++ b/roles/deploy-env/tasks/coredns_resolver.yaml @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Enable recursive queries for coredns + become: false + shell: | + PATCH=$(mktemp) + kubectl get configmap coredns -n kube-system -o json | jq -r "{data: .data}" | sed 's/ready\\n/header \{\\n response set ra\\n \}\\n ready\\n/g' > "${PATCH}" + kubectl patch configmap coredns -n kube-system --patch-file "${PATCH}" + kubectl set image deployment coredns -n kube-system "coredns=registry.k8s.io/coredns/coredns:v1.9.4" + kubectl rollout restart -n kube-system deployment/coredns + sleep 30 + kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=kube-dns + rm -f "${PATCH}" + args: + executable: /bin/bash + when: inventory_hostname in (groups['primary'] | default([])) + +- name: Use coredns as default DNS resolver + copy: + src: files/cluster_resolv.conf + dest: /etc/resolv.conf + owner: root + group: root + mode: 0644 + when: inventory_hostname in (groups['k8s_cluster'] | default([])) +... diff --git a/roles/deploy-env/tasks/k8s_client.yaml b/roles/deploy-env/tasks/k8s_client.yaml new file mode 100644 index 000000000..629f390da --- /dev/null +++ b/roles/deploy-env/tasks/k8s_client.yaml @@ -0,0 +1,54 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Install Kubectl + apt: + state: present + update_cache: true + allow_downgrade: true + pkg: + - "kubectl={{ kube_version }}" + +- name: "Setup kubeconfig directory for {{ kubectl.user }} user" + shell: | + mkdir -p /home/{{ kubectl.user }}/.kube + +- name: "Copy kube_config file for {{ kubectl.user }} user" + synchronize: + src: /tmp/kube_config + dest: /home/{{ kubectl.user }}/.kube/config + +- name: "Set kubconfig file ownership for {{ kubectl.user }} user" + shell: | + chown -R {{ kubectl.user }}:{{ kubectl.group }} /home/{{ kubectl.user }}/.kube + +- name: Deploy Helm + block: + - name: Install Helm + shell: | + TMP_DIR=$(mktemp -d) + curl -sSL https://get.helm.sh/helm-{{ helm_version }}-linux-amd64.tar.gz | tar -zxv --strip-components=1 -C ${TMP_DIR} + mv "${TMP_DIR}"/helm /usr/local/bin/helm + rm -rf "${TMP_DIR}" + args: + executable: /bin/bash + + # This is to improve build time + - name: Remove stable Helm repo + command: helm repo remove stable + ignore_errors: true + +- name: Untaint Kubernetes control plane node + become: false + command: kubectl taint nodes -l 'node-role.kubernetes.io/control-plane' node-role.kubernetes.io/control-plane- +... diff --git a/roles/deploy-env/tasks/common_k8s.yaml b/roles/deploy-env/tasks/k8s_common.yaml similarity index 59% rename from roles/deploy-env/tasks/common_k8s.yaml rename to roles/deploy-env/tasks/k8s_common.yaml index 2fa4f0350..ac1871523 100644 --- a/roles/deploy-env/tasks/common_k8s.yaml +++ b/roles/deploy-env/tasks/k8s_common.yaml @@ -43,6 +43,16 @@ state: present ignore_errors: true +- name: Configure number of inotify instances + sysctl: + name: "{{ item }}" + value: "0" + state: present + loop: + - net.ipv4.conf.all.rp_filter + - net.ipv4.conf.default.rp_filter + ignore_errors: true + - name: Remove swapfile from /etc/fstab mount: name: "{{ item }}" @@ -56,27 +66,6 @@ command: swapoff -a when: ansible_swaptotal_mb > 0 -- name: Ensure dependencies are installed - apt: - name: - - apt-transport-https - - ca-certificates - - gnupg2 - - ipvsadm - - jq - state: present - -- name: Add Kubernetes apt repository key - apt_key: - url: "https://pkgs.k8s.io/core:/stable:/{{ kube_version_repo }}/deb/Release.key" - state: present - -- name: Add Kubernetes apt repository - apt_repository: - repo: "deb https://pkgs.k8s.io/core:/stable:/{{ kube_version_repo }}/deb/ /" - state: present - filename: kubernetes.list - - name: Install Kubernetes binaries apt: state: present @@ -93,36 +82,27 @@ daemon_reload: yes state: restarted +- name: Configure resolv.conf + template: + src: files/resolv.conf + dest: /etc/resolv.conf + owner: root + group: root + mode: 0644 + vars: + nameserver_ip: "8.8.8.8" + - name: Disable systemd-resolved service: name: systemd-resolved enabled: false state: stopped + ignore_errors: true -- name: Configure resolv.conf - copy: - src: files/resolv.conf - dest: "{{ item }}" - loop: - - /etc/resolv.conf - - /run/systemd/resolve/resolv.conf - -# We download Calico manifest on all nodes because we then want to download -# Calico images BEFORE deploying it -- name: Download Calico manifest - shell: | - curl -LSs {{ calico_manifest_url }} -o /tmp/calico.yaml - sed -i -e 's#docker.io/calico/#quay.io/calico/#g' /tmp/calico.yaml - args: - executable: /bin/bash - -# Download images needed for calico before applying manifests, so that `kubectl wait` timeout -# for `k8s-app=kube-dns` isn't reached by slow download speeds -- name: Download Calico images - shell: | - export CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock - export IMAGE_SERVICE_ENDPOINT=unix:///run/containerd/containerd.sock - awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} crictl pull {} - args: - executable: /bin/bash +- name: Disable unbound + service: + name: unbound + enabled: false + state: stopped + ignore_errors: true ... diff --git a/roles/deploy-env/tasks/k8s_control_plane.yaml b/roles/deploy-env/tasks/k8s_control_plane.yaml new file mode 100644 index 000000000..563f09b2d --- /dev/null +++ b/roles/deploy-env/tasks/k8s_control_plane.yaml @@ -0,0 +1,39 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Mount tmpfs to /var/lib/etcd + mount: + path: /var/lib/etcd + src: tmpfs + fstype: tmpfs + opts: size=1g + state: mounted + +- name: Prepare kubeadm config + template: + src: files/kubeadm_config.yaml + dest: /tmp/kubeadm_config.yaml + +- name: Initialize the Kubernetes cluster using kubeadm + command: kubeadm init --config /tmp/kubeadm_config.yaml + +- name: Generate join command + command: kubeadm token create --print-join-command + register: join_command + +- name: "Copy kube config to localhost" + synchronize: + mode: pull + src: /etc/kubernetes/admin.conf + dest: /tmp/kube_config +... diff --git a/roles/deploy-env/tasks/main.yaml b/roles/deploy-env/tasks/main.yaml index 7e3478ee4..397c38dbf 100644 --- a/roles/deploy-env/tasks/main.yaml +++ b/roles/deploy-env/tasks/main.yaml @@ -20,42 +20,63 @@ src: files/hosts dest: /etc/hosts +- name: Loop devices + include_tasks: + file: loopback_devices.yaml + when: loopback_setup and inventory_hostname in (groups['k8s_cluster'] | default([])) + - name: Deploy Containerd include_tasks: file: containerd.yaml -- name: Common K8s tasks +- name: Include K8s common tasks include_tasks: - file: common_k8s.yaml + file: k8s_common.yaml + when: inventory_hostname in (groups['k8s_cluster'] | default([])) -- name: Include control-plane tasks +- name: Include K8s control-plane tasks include_tasks: - file: control-plane.yaml - when: inventory_hostname == 'primary' + file: k8s_control_plane.yaml + when: inventory_hostname in (groups['k8s_control_plane'] | default([])) - name: Join workload nodes to cluster - command: "{{ hostvars['primary']['join_command'].stdout_lines[0] }}" - when: inventory_hostname in (groups['nodes'] | default([])) + command: "{{ (groups['k8s_control_plane'] | map('extract', hostvars, ['join_command', 'stdout_lines', 0]))[0] }}" + when: inventory_hostname in (groups['k8s_nodes'] | default([])) -- name: Wait for cluster is ready - become: false - block: - - name: Sleep 10 before checking calico nodes - pause: - seconds: 10 - - name: Wait for Calico pods ready - command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=calico-node - when: inventory_hostname == 'primary' - -- name: Add coredns to /etc/resolv.conf - lineinfile: - line: nameserver 10.96.0.10 - path: /etc/resolv.conf - state: present - insertbefore: "BOF" - -- name: Loop devices +- name: Include K8s client tasks include_tasks: - file: loopback_devices.yaml - when: loopback_setup + file: k8s_client.yaml + when: inventory_hostname in (groups['primary'] | default([])) + +- name: Include K8s Calico tasks + include_tasks: + file: calico.yaml + +- name: Include coredns resolver tasks + include_tasks: + file: coredns_resolver.yaml + +- name: Include Openstack provider gateway tasks + include_tasks: + file: openstack_provider_gateway.yaml + when: + - openstack_provider_gateway_setup + - inventory_hostname in (groups['k8s_control_plane'] | default([])) + +- name: Include Metallb tasks + include_tasks: + file: metallb.yaml + when: metallb_setup + +- name: Include Openstack Metallb endpoint tasks + include_tasks: + file: openstack_metallb_endpoint.yaml + when: + - metallb_setup + - inventory_hostname in (groups['primary'] | default([])) + +- name: Include client-to-cluster tunnel tasks + include_tasks: + file: client_cluster_tunnel.yaml + when: (groups['primary'] | difference(groups['k8s_control_plane']) | length > 0) ... diff --git a/roles/deploy-env/tasks/metallb.yaml b/roles/deploy-env/tasks/metallb.yaml new file mode 100644 index 000000000..9be2a32ad --- /dev/null +++ b/roles/deploy-env/tasks/metallb.yaml @@ -0,0 +1,64 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Deploy MetalLB + become: false + when: inventory_hostname in (groups['primary'] | default([])) + block: + - name: Add MetalLB chart repo + kubernetes.core.helm_repository: + name: metallb + repo_url: "https://metallb.github.io/metallb" + + - name: Install MetalLB + kubernetes.core.helm: + name: metallb + chart_ref: metallb/metallb + namespace: metallb-system + create_namespace: true + + - name: Sleep before trying to check MetalLB pods + pause: + seconds: 30 + + - name: Wait for MetalLB pods ready + command: kubectl -n metallb-system wait --timeout=240s --for=condition=Ready pods -l 'app.kubernetes.io/name=metallb' + + - name: Create MetalLB address pool + shell: | + tee > /tmp/metallb_ipaddresspool.yaml < /tmp/metallb_l2advertisement.yaml < /tmp/openstack_endpoint_service.yaml <