Add deploy-env role
This role works both for singlenode and multinode inventories. The role installs all necessary prerequisites and deploys K8s with Containerd as a container runtime. The idea is to use this role to deploy all test singlenode/multinode environments for all test jobs. This PR wraps into a role playbooks that we are currently using for multinode compute-kit tests. Change-Id: I41bbe80d806e614a155e6775c4505a4d81a086e8
This commit is contained in:
Vladimir Kozhukalov
parent
f234218dba
commit
bda43dfff8
|
|
@ -0,0 +1,37 @@
|
|||
This role is used to deploy test environment which includes
|
||||
- install necessary prerequisites including Helm
|
||||
- deploy Containerd and a container runtime for Kubernetes
|
||||
- deploy Kubernetes using Kubeadm with a single control plain node
|
||||
- install Calico as a Kubernetes networking
|
||||
|
||||
The role works both for singlenode and multinode inventories and
|
||||
assumes the inventory has the node called `primary` and the group called `nodes`.
|
||||
|
||||
See for example:
|
||||
|
||||
```yaml
|
||||
all:
|
||||
children:
|
||||
ungrouped:
|
||||
hosts:
|
||||
primary:
|
||||
ansible_port: 22
|
||||
ansible_host: 10.10.10.10
|
||||
ansible_user: ubuntu
|
||||
ansible_ssh_private_key_file: /home/ubuntu/.ssh/id_rsa.pub
|
||||
ansible_ssh_extra_args: -o StrictHostKeyChecking=no
|
||||
nodes:
|
||||
hosts:
|
||||
node-1:
|
||||
ansible_port: 22
|
||||
ansible_host: 10.10.10.11
|
||||
ansible_user: ubuntu
|
||||
ansible_ssh_private_key_file: /home/ubuntu/.ssh/id_rsa.pub
|
||||
ansible_ssh_extra_args: -o StrictHostKeyChecking=no
|
||||
node-2:
|
||||
ansible_port: 22
|
||||
ansible_host: 10.10.10.12
|
||||
ansible_user: ubuntu
|
||||
ansible_ssh_private_key_file: /home/ubuntu/.ssh/id_rsa.pub
|
||||
ansible_ssh_extra_args: -o StrictHostKeyChecking=no
|
||||
```
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
---
|
||||
kubectl:
|
||||
user: zuul
|
||||
group: zuul
|
||||
...
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
---
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9091"
|
||||
spec:
|
||||
containers:
|
||||
- name: calico-node
|
||||
env:
|
||||
- name: FELIX_PROMETHEUSMETRICSENABLED
|
||||
value: "true"
|
||||
- name: FELIX_PROMETHEUSMETRICSPORT
|
||||
value: "9091"
|
||||
- name: FELIX_IGNORELOOSERPF
|
||||
value: "true"
|
||||
# We assign IP on br-ex interface while testing the deployed Openstack cluster and
|
||||
# we need Calico to skip this interface while discovering the
|
||||
# network changes on the host to prevent announcing unnecessary networks.
|
||||
- name: IP_AUTODETECTION_METHOD
|
||||
value: "skip-interface=br-ex"
|
||||
...
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
version = 2
|
||||
disabled_plugins = []
|
||||
[plugins."io.containerd.grpc.v1.cri".registry]
|
||||
config_path = "/etc/containerd/certs.d"
|
||||
|
||||
{% for item in registry_namespaces %}
|
||||
{% if item.auth is defined %}
|
||||
[plugins."io.containerd.grpc.v1.cri".registry.configs."{{ item.namespace }}".auth]
|
||||
auth = "{{ item.auth }}"
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"exec-opts": ["native.cgroupdriver=systemd"],
|
||||
"log-driver": "json-file",
|
||||
"log-opts": {
|
||||
"max-size": "100m"
|
||||
},
|
||||
"storage-driver": "overlay2",
|
||||
"live-restore": true
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
{% if item.skip_server is not defined or not item.skip_server %}
|
||||
server = "{{ item.server | default('https://' + item.namespace) }}"
|
||||
{% endif %}
|
||||
|
||||
[host."{{ item.mirror }}"]
|
||||
capabilities = ["pull", "resolve", "push"]
|
||||
{% if item.ca is defined %}
|
||||
ca = "{{ item.ca }}"
|
||||
{% endif %}
|
||||
{% if item.skip_verify is defined and item.skip_verify %}
|
||||
skip_verify = true
|
||||
{% endif %}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
---
|
||||
apiVersion: kubeproxy.config.k8s.io/v1alpha1
|
||||
kind: KubeProxyConfiguration
|
||||
mode: ipvs
|
||||
...
|
||||
---
|
||||
apiVersion: kubeadm.k8s.io/v1beta2
|
||||
kind: ClusterConfiguration
|
||||
networking:
|
||||
serviceSubnet: "10.96.0.0/16"
|
||||
podSubnet: "10.244.0.0/24" # --pod-network-cidr
|
||||
dnsDomain: "cluster.local"
|
||||
...
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
nameserver 8.8.8.8
|
||||
nameserver 8.8.4.4
|
||||
search svc.cluster.local cluster.local
|
||||
options ndots:5 timeout:1 attempts:1
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
---
|
||||
- name: Load necessary modules
|
||||
modprobe:
|
||||
name: "{{ item }}"
|
||||
state: present
|
||||
with_items:
|
||||
- overlay
|
||||
- br_netfilter
|
||||
|
||||
- name: Configure sysctl
|
||||
sysctl:
|
||||
name: "{{ item }}"
|
||||
value: "1"
|
||||
state: present
|
||||
loop:
|
||||
- net.ipv6.conf.default.disable_ipv6
|
||||
- net.ipv6.conf.all.disable_ipv6
|
||||
- net.ipv6.conf.lo.disable_ipv6
|
||||
- net.bridge.bridge-nf-call-iptables
|
||||
- net.bridge.bridge-nf-call-ip6tables
|
||||
- net.ipv4.ip_forward
|
||||
ignore_errors: true
|
||||
|
||||
- name: Remove swapfile from /etc/fstab
|
||||
mount:
|
||||
name: "{{ item }}"
|
||||
fstype: swap
|
||||
state: absent
|
||||
with_items:
|
||||
- swap
|
||||
- none
|
||||
|
||||
- name: Disable swap
|
||||
command: swapoff -a
|
||||
when: ansible_swaptotal_mb > 0
|
||||
|
||||
- name: Ensure dependencies are installed
|
||||
apt:
|
||||
name:
|
||||
- apt-transport-https
|
||||
- ca-certificates
|
||||
- gnupg2
|
||||
- ipvsadm
|
||||
- jq
|
||||
state: present
|
||||
|
||||
- name: Add Kubernetes apt repository key
|
||||
apt_key:
|
||||
url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
|
||||
state: present
|
||||
|
||||
- name: Add Kubernetes apt repository
|
||||
apt_repository:
|
||||
repo: deb https://apt.kubernetes.io/ kubernetes-xenial main
|
||||
state: present
|
||||
filename: kubernetes.list
|
||||
|
||||
- name: Install Kubernetes binaries
|
||||
apt:
|
||||
state: present
|
||||
update_cache: true
|
||||
allow_downgrade: true
|
||||
pkg:
|
||||
- "kubelet={{ kube_version }}"
|
||||
- "kubeadm={{ kube_version }}"
|
||||
- "kubectl={{ kube_version }}"
|
||||
|
||||
- name: Restart kubelet
|
||||
service:
|
||||
name: kubelet
|
||||
daemon_reload: yes
|
||||
state: restarted
|
||||
|
||||
- name: Disable systemd-resolved
|
||||
service:
|
||||
name: systemd-resolved
|
||||
enabled: false
|
||||
state: stopped
|
||||
|
||||
- name: Configure resolv.conf
|
||||
copy:
|
||||
src: files/resolv.conf
|
||||
dest: "{{ item }}"
|
||||
loop:
|
||||
- /etc/resolv.conf
|
||||
- /run/systemd/resolve/resolv.conf
|
||||
|
||||
# We download Calico manifest on all nodes because we then want to download
|
||||
# Calico images BEFORE deploying it
|
||||
- name: Download Calico manifest
|
||||
shell: |
|
||||
curl -LSs https://docs.projectcalico.org/archive/{{ calico_version }}/manifests/calico.yaml -o /tmp/calico.yaml
|
||||
sed -i -e 's#docker.io/calico/#quay.io/calico/#g' /tmp/calico.yaml
|
||||
args:
|
||||
executable: /bin/bash
|
||||
|
||||
# Download images needed for calico before applying manifests, so that `kubectl wait` timeout
|
||||
# for `k8s-app=kube-dns` isn't reached by slow download speeds
|
||||
- name: Download Calico images
|
||||
shell: |
|
||||
export CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock
|
||||
export IMAGE_SERVICE_ENDPOINT=unix:///run/containerd/containerd.sock
|
||||
awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} crictl pull {}
|
||||
args:
|
||||
executable: /bin/bash
|
||||
...
|
||||
|
|
@ -0,0 +1,148 @@
|
|||
---
|
||||
- name: Remove old docker packages
|
||||
apt:
|
||||
pkg:
|
||||
- docker.io
|
||||
- docker-doc
|
||||
- docker-compose
|
||||
- podman-docker
|
||||
- containerd
|
||||
- runc
|
||||
state: absent
|
||||
|
||||
- name: Ensure dependencies are installed
|
||||
apt:
|
||||
name:
|
||||
- apt-transport-https
|
||||
- ca-certificates
|
||||
- gnupg2
|
||||
state: present
|
||||
|
||||
- name: Add Docker apt repository key
|
||||
apt_key:
|
||||
url: https://download.docker.com/linux/ubuntu/gpg
|
||||
keyring: /etc/apt/trusted.gpg.d/docker.gpg
|
||||
state: present
|
||||
|
||||
- name: Get dpkg arch
|
||||
command: dpkg --print-architecture
|
||||
register: dpkg_architecture
|
||||
|
||||
- name: Add Docker apt repository
|
||||
apt_repository:
|
||||
repo: deb [arch="{{ dpkg_architecture.stdout }}" signed-by=/etc/apt/trusted.gpg.d/docker.gpg] https://download.docker.com/linux/ubuntu "{{ ansible_distribution_release }}" stable
|
||||
state: present
|
||||
filename: docker.list
|
||||
|
||||
- name: Install docker packages
|
||||
apt:
|
||||
pkg:
|
||||
- docker-ce
|
||||
- docker-ce-cli
|
||||
- containerd.io
|
||||
- docker-buildx-plugin
|
||||
- docker-compose-plugin
|
||||
state: present
|
||||
update_cache: true
|
||||
|
||||
- name: Install Crictl
|
||||
shell: |
|
||||
wget https://github.com/kubernetes-sigs/cri-tools/releases/download/{{crictl_version}}/crictl-{{crictl_version}}-linux-amd64.tar.gz
|
||||
sudo tar zxvf crictl-{{crictl_version}}-linux-amd64.tar.gz -C /usr/local/bin
|
||||
rm -f crictl-{{crictl_version}}-linux-amd64.tar.gz
|
||||
args:
|
||||
executable: /bin/bash
|
||||
|
||||
- name: Configure Docker daemon
|
||||
copy:
|
||||
src: files/daemon.json
|
||||
dest: /etc/docker/daemon.json
|
||||
|
||||
- name: Restart docker
|
||||
service:
|
||||
name: docker
|
||||
daemon_reload: yes
|
||||
state: restarted
|
||||
|
||||
- name: Set mirror_fqdn fact
|
||||
when:
|
||||
- registry_mirror is not defined
|
||||
- zuul_site_mirror_fqdn is defined
|
||||
set_fact:
|
||||
registry_mirror: "http://{{ zuul_site_mirror_fqdn }}:8082"
|
||||
|
||||
- name: Set regitstry namespaces
|
||||
set_fact:
|
||||
registry_namespaces:
|
||||
- namespace: "_default"
|
||||
mirror: "{{ registry_mirror }}"
|
||||
skip_server: true
|
||||
skip_verify: true
|
||||
when: registry_mirror is defined
|
||||
|
||||
- name: Buildset registry namespace
|
||||
when: buildset_registry is defined
|
||||
block:
|
||||
- name: Buildset registry alias
|
||||
include_tasks:
|
||||
file: buildset_registry_alias.yaml
|
||||
|
||||
- name: Write buildset registry TLS certificate
|
||||
copy:
|
||||
content: "{{ buildset_registry.cert }}"
|
||||
dest: "/usr/local/share/ca-certificates/{{ buildset_registry_alias }}.crt"
|
||||
mode: 0644
|
||||
register: buildset_registry_tls_ca
|
||||
|
||||
- name: Update CA certs
|
||||
command: "update-ca-certificates"
|
||||
when: buildset_registry_tls_ca is changed
|
||||
|
||||
- name: Set buildset registry namespace
|
||||
set_fact:
|
||||
buildset_registry_namespace:
|
||||
namespace: '{{ buildset_registry_alias }}:{{ buildset_registry.port }}'
|
||||
mirror: 'https://{{ buildset_registry_alias }}:{{ buildset_registry.port }}'
|
||||
ca: "/usr/local/share/ca-certificates/{{ buildset_registry_alias }}.crt"
|
||||
auth: "{{ (buildset_registry.username + ':' + buildset_registry.password) | b64encode }}"
|
||||
|
||||
- name: Init registry_namespaces if not defined
|
||||
set_fact:
|
||||
registry_namespaces: "[]"
|
||||
when: not registry_namespaces is defined
|
||||
|
||||
- name: Append buildset_registry to registry namespaces
|
||||
when:
|
||||
- buildset_registry_namespace is defined
|
||||
- registry_namespaces is defined
|
||||
set_fact:
|
||||
registry_namespaces: "{{ registry_namespaces + [ buildset_registry_namespace ] }}"
|
||||
|
||||
- name: Configure containerd
|
||||
template:
|
||||
src: files/containerd_config.toml
|
||||
dest: /etc/containerd/config.toml
|
||||
|
||||
- name: Create containerd config directory hierarchy
|
||||
file:
|
||||
state: directory
|
||||
path: /etc/containerd/certs.d
|
||||
|
||||
- name: Create host namespace directory
|
||||
file:
|
||||
state: directory
|
||||
path: "/etc/containerd/certs.d/{{ item.namespace }}"
|
||||
loop: "{{ registry_namespaces }}"
|
||||
|
||||
- name: Create hosts.toml file
|
||||
template:
|
||||
src: files/hosts.toml
|
||||
dest: "/etc/containerd/certs.d/{{ item.namespace }}/hosts.toml"
|
||||
loop: "{{ registry_namespaces }}"
|
||||
|
||||
- name: Restart containerd
|
||||
service:
|
||||
name: containerd
|
||||
daemon_reload: yes
|
||||
state: restarted
|
||||
...
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
---
|
||||
- name: Mount tmpfs to /var/lib/etcd
|
||||
mount:
|
||||
path: /var/lib/etcd
|
||||
src: tmpfs
|
||||
fstype: tmpfs
|
||||
opts: size=1g
|
||||
state: mounted
|
||||
|
||||
- name: Prepare kubeadm config
|
||||
copy:
|
||||
src: files/kubeadm_config.yaml
|
||||
dest: /tmp/kubeadm_config.yaml
|
||||
|
||||
- name: Initialize the Kubernetes cluster using kubeadm
|
||||
command: kubeadm init --config /tmp/kubeadm_config.yaml
|
||||
|
||||
- name: "Setup kubeconfig for {{ kubectl.user }} user"
|
||||
shell: |
|
||||
mkdir -p /home/{{ kubectl.user }}/.kube
|
||||
cp -i /etc/kubernetes/admin.conf /home/{{ kubectl.user }}/.kube/config
|
||||
chown {{ kubectl.user }}:{{ kubectl.group }} /home/{{ kubectl.user }}/.kube/config
|
||||
args:
|
||||
executable: /bin/bash
|
||||
|
||||
- name: Deploy Calico
|
||||
become: false
|
||||
command: kubectl apply -f /tmp/calico.yaml
|
||||
|
||||
- name: Sleep before trying to check Calico pods
|
||||
pause:
|
||||
seconds: 20
|
||||
|
||||
- name: Wait for Calico pods ready
|
||||
become: false
|
||||
command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=calico-node
|
||||
|
||||
- name: Prepare Calico patch
|
||||
copy:
|
||||
src: files/calico_patch.yaml
|
||||
dest: /tmp/calico_patch.yaml
|
||||
|
||||
- name: Patch Calico
|
||||
become: false
|
||||
command: kubectl -n kube-system patch daemonset calico-node --patch-file /tmp/calico_patch.yaml
|
||||
|
||||
- name: Wait for Calico pods ready
|
||||
become: false
|
||||
command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=calico-node
|
||||
|
||||
- name: Generate join command
|
||||
command: kubeadm token create --print-join-command
|
||||
register: join_command
|
||||
|
||||
- name: Untaint Kubernetes control plane node
|
||||
become: false
|
||||
command: kubectl taint nodes -l 'node-role.kubernetes.io/control-plane' node-role.kubernetes.io/control-plane-
|
||||
|
||||
- name: Enable recursive queries for coredns
|
||||
become: false
|
||||
shell: |
|
||||
PATCH=$(mktemp)
|
||||
kubectl get configmap coredns -n kube-system -o json | jq -r "{data: .data}" | sed 's/ready\\n/header \{\\n response set ra\\n \}\\n ready\\n/g' > "${PATCH}"
|
||||
kubectl patch configmap coredns -n kube-system --patch-file "${PATCH}"
|
||||
kubectl set image deployment coredns -n kube-system "coredns=registry.k8s.io/coredns/coredns:v1.9.4"
|
||||
kubectl rollout restart -n kube-system deployment/coredns
|
||||
rm -f "${PATCH}"
|
||||
args:
|
||||
executable: /bin/bash
|
||||
...
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
---
|
||||
- name: Include prerequisites tasks
|
||||
include_tasks:
|
||||
file: prerequisites.yaml
|
||||
|
||||
- name: Include common tasks
|
||||
include_tasks:
|
||||
file: containerd.yaml
|
||||
|
||||
- name: Include common tasks
|
||||
include_tasks:
|
||||
file: common_k8s.yaml
|
||||
|
||||
- name: Include control-plane tasks
|
||||
include_tasks:
|
||||
file: control-plane.yaml
|
||||
when: inventory_hostname == 'primary'
|
||||
|
||||
- name: Join workload nodes to cluster
|
||||
command: "{{ hostvars['primary']['join_command'].stdout_lines[0] }}"
|
||||
when: inventory_hostname in (groups['nodes'] | default([]))
|
||||
|
||||
- name: Wait for cluster is ready
|
||||
become: false
|
||||
block:
|
||||
- name: Wait for Calico pods ready
|
||||
command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=calico-node
|
||||
|
||||
- name: Wait for Coredns pods ready
|
||||
command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=kube-dns
|
||||
when: inventory_hostname == 'primary'
|
||||
|
||||
- name: Add coredns to /etc/resolv.conf
|
||||
lineinfile:
|
||||
line: nameserver 10.96.0.10
|
||||
path: /etc/resolv.conf
|
||||
state: present
|
||||
insertbefore: "BOF"
|
||||
...
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
---
|
||||
- name: Add Ceph apt repository key
|
||||
apt_key:
|
||||
url: https://download.ceph.com/keys/release.asc
|
||||
state: present
|
||||
|
||||
- name: Add Ceph apt repository
|
||||
apt_repository:
|
||||
repo: deb https://download.ceph.com/debian-reef/ "{{ ansible_distribution_release }}" main
|
||||
state: present
|
||||
filename: ceph.list
|
||||
|
||||
- name: Install necessary packages
|
||||
apt:
|
||||
pkg:
|
||||
- socat
|
||||
- jq
|
||||
- util-linux
|
||||
- bridge-utils
|
||||
- iptables
|
||||
- conntrack
|
||||
- libffi-dev
|
||||
- ipvsadm
|
||||
- make
|
||||
- bc
|
||||
- git-review
|
||||
- notary
|
||||
- ceph-common
|
||||
- rbd-nbd
|
||||
- nfs-common
|
||||
- ethtool
|
||||
- python3-dev
|
||||
- ca-certificates
|
||||
- git
|
||||
- nmap
|
||||
- curl
|
||||
- uuid-runtime
|
||||
- net-tools
|
||||
- less
|
||||
- telnet
|
||||
- tcpdump
|
||||
- vim
|
||||
- lvm2
|
||||
|
||||
- name: Deploy Helm
|
||||
when: inventory_hostname == 'primary'
|
||||
block:
|
||||
- name: Install Helm
|
||||
shell: |
|
||||
TMP_DIR=$(mktemp -d)
|
||||
curl -sSL https://get.helm.sh/helm-{{ helm_version }}-linux-amd64.tar.gz | tar -zxv --strip-components=1 -C ${TMP_DIR}
|
||||
mv "${TMP_DIR}"/helm /usr/local/bin/helm
|
||||
rm -rf "${TMP_DIR}"
|
||||
args:
|
||||
executable: /bin/bash
|
||||
|
||||
# This is to improve build time
|
||||
- name: Remove stable Helm repo
|
||||
command: helm repo remove stable
|
||||
ignore_errors: true
|
||||
...
|
||||
Loading…
Reference in New Issue