Configure containerd mirrors for multinode tests

The compute-kit jobs are used to test new images
which are published to buildset registry. We have
to configure containerd which is used for multinode
compute-kit jobs to use this buildset registry.

The role use-buildset-registry that we used before
does not properly configure containerd. So we
extended deploy-docker playbook to configure
both buildset registry and registry mirror
if they are defined.

Change-Id: Idb892a3fcaf51385998d466dbdff8de36d9dd338
This commit is contained in:
Vladimir Kozhukalov 2023-08-11 14:41:57 +03:00
parent 2024cc361e
commit c39638a148
7 changed files with 177 additions and 23 deletions

View File

@ -0,0 +1,19 @@
- name: Configure /etc/hosts for buildset_registry to workaround docker not understanding ipv6 addresses
lineinfile:
path: /etc/hosts
state: present
regex: "^{{ buildset_registry.host }}\tzuul-jobs.buildset-registry$"
line: "{{ buildset_registry.host }}\tzuul-jobs.buildset-registry"
insertafter: EOF
when:
- buildset_registry.host | ipaddr
- name: Set buildset_registry alias variable when using ip
set_fact:
buildset_registry_alias: zuul-jobs.buildset-registry
when:
- buildset_registry.host | ipaddr
- name: Set buildset_registry alias variable when using name
set_fact:
buildset_registry_alias: "{{ buildset_registry.host }}"
when:
- not ( buildset_registry.host | ipaddr )

View File

@ -48,25 +48,103 @@
state: present state: present
update_cache: true update_cache: true
- name: Install Crictl
shell: |
wget https://github.com/kubernetes-sigs/cri-tools/releases/download/{{crictl_version}}/crictl-{{crictl_version}}-linux-amd64.tar.gz
sudo tar zxvf crictl-{{crictl_version}}-linux-amd64.tar.gz -C /usr/local/bin
rm -f crictl-{{crictl_version}}-linux-amd64.tar.gz
args:
executable: /bin/bash
- name: Configure Docker daemon - name: Configure Docker daemon
copy: copy:
src: files/daemon.json src: files/daemon.json
dest: /etc/docker/daemon.json dest: /etc/docker/daemon.json
- name: Remove /etc/containerd/config.toml
file:
path: /etc/containerd/config.toml
state: absent
ignore_errors: true
- name: Restart containerd
service:
name: containerd
daemon_reload: yes
state: restarted
- name: Restart docker - name: Restart docker
service: service:
name: docker name: docker
daemon_reload: yes daemon_reload: yes
state: restarted state: restarted
- name: Set mirror_fqdn fact
when:
- registry_mirror is not defined
- zuul_site_mirror_fqdn is defined
set_fact:
registry_mirror: "http://{{ zuul_site_mirror_fqdn }}:8082"
- name: Set regitstry namespaces
set_fact:
registry_namespaces:
- namespace: "_default"
mirror: "{{ registry_mirror }}"
skip_server: true
skip_verify: true
when: registry_mirror is defined
- name: Buildset registry namespace
when: buildset_registry is defined
block:
- name: Buildset registry alias
include_tasks:
file: buildset_registry_alias.yaml
- name: Write buildset registry TLS certificate
copy:
content: "{{ buildset_registry.cert }}"
dest: "/usr/local/share/ca-certificates/{{ buildset_registry_alias }}.crt"
mode: 0644
register: buildset_registry_tls_ca
- name: Update CA certs
command: "update-ca-certificates"
when: buildset_registry_tls_ca is changed
- name: Set buildset registry namespace
set_fact:
buildset_registry_namespace:
namespace: '{{ buildset_registry_alias }}:{{ buildset_registry.port }}'
mirror: 'https://{{ buildset_registry_alias }}:{{ buildset_registry.port }}'
ca: "/usr/local/share/ca-certificates/{{ buildset_registry_alias }}.crt"
auth: "{{ (buildset_registry.username + ':' + buildset_registry.password) | b64encode }}"
- name: Init registry_namespaces if not defined
set_fact:
registry_namespaces: "[]"
when: not registry_namespaces is defined
- name: Append buildset_registry to registry namespaces
when:
- buildset_registry_namespace is defined
- registry_namespaces is defined
set_fact:
registry_namespaces: "{{ registry_namespaces + [ buildset_registry_namespace ] }}"
- name: Configure containerd
template:
src: files/containerd_config.toml
dest: /etc/containerd/config.toml
- name: Create containerd config directory hierarchy
file:
state: directory
path: /etc/containerd/certs.d
- name: Create host namespace directory
file:
state: directory
path: "/etc/containerd/certs.d/{{ item.namespace }}"
loop: "{{ registry_namespaces }}"
- name: Create hosts.toml file
template:
src: files/hosts.toml
dest: "/etc/containerd/certs.d/{{ item.namespace }}/hosts.toml"
loop: "{{ registry_namespaces }}"
- name: Restart containerd
service:
name: containerd
daemon_reload: yes
state: restarted

View File

@ -118,6 +118,7 @@
executable: /bin/bash executable: /bin/bash
- hosts: all - hosts: all
become: true
tasks: tasks:
# We download Calico manifest on all nodes because we then want to download # We download Calico manifest on all nodes because we then want to download
# Calico images BEFORE deploying it # Calico images BEFORE deploying it
@ -132,7 +133,9 @@
# for `k8s-app=kube-dns` isn't reached by slow download speeds # for `k8s-app=kube-dns` isn't reached by slow download speeds
- name: Download Calico images - name: Download Calico images
shell: | shell: |
awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} sudo docker pull {} export CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock
export IMAGE_SERVICE_ENDPOINT=unix:///run/containerd/containerd.sock
awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} crictl pull {}
args: args:
executable: /bin/bash executable: /bin/bash

View File

@ -0,0 +1,11 @@
version = 2
disabled_plugins = []
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"
{% for item in registry_namespaces %}
{% if item.auth is defined %}
[plugins."io.containerd.grpc.v1.cri".registry.configs."{{ item.namespace }}".auth]
auth = "{{ item.auth }}"
{% endif %}
{% endfor %}

View File

@ -0,0 +1,12 @@
{% if item.skip_server is not defined or not item.skip_server %}
server = "{{ item.server | default('https://' + item.namespace) }}"
{% endif %}
[host."{{ item.mirror }}"]
capabilities = ["pull", "resolve", "push"]
{% if item.ca is defined %}
ca = "{{ item.ca }}"
{% endif %}
{% if item.skip_verify %}
skip_verify = true
{% endif %}

View File

@ -11,18 +11,48 @@
# limitations under the License. # limitations under the License.
--- ---
- hosts: all
tasks:
- name: Override images
include_role:
name: override-images
when: buildset_registry is defined
- name: Use docker mirror
include_role:
name: use-docker-mirror
- hosts: primary - hosts: primary
tasks: tasks:
- name: Override images
when: buildset_registry is defined
vars:
work_dir: "{{ zuul.project.src_dir }}"
block:
- name: Buildset registry alias
include_tasks:
file: buildset_registry_alias.yaml
- name: Print zuul
debug:
var: zuul
- name: Override proposed images from artifacts
shell: >
find {{ override_paths | join(" ") }} -type f -exec sed -Ei
"s#['\"]?docker\.io/({{ repo }}):({{ tag }})['\"]?\$#{{ buildset_registry_alias }}:{{ buildset_registry.port }}/\1:\2#g" {} +
loop: "{{ zuul.artifacts | default([]) }}"
args:
chdir: "{{ work_dir }}"
loop_control:
loop_var: zj_zuul_artifact
when: "'metadata' in zj_zuul_artifact and zj_zuul_artifact.metadata.type | default('') == 'container_image'"
vars:
tag: "{{ zj_zuul_artifact.metadata.tag }}"
repo: "{{ zj_zuul_artifact.metadata.repository }}"
override_paths:
- ../openstack-helm*/*/values*
- ../openstack-helm-infra/tools/deployment/
- name: Diff
shell: |
set -ex;
for dir in openstack-helm openstack-helm-infra; do
path="{{ work_dir }}/../${dir}/"
if [ ! -d "${path}" ]; then continue; fi
echo "${dir} diff"
cd "${path}"; git diff; cd -;
done
- name: "creating directory for run artifacts" - name: "creating directory for run artifacts"
file: file:
path: "/tmp/artifacts" path: "/tmp/artifacts"

View File

@ -69,6 +69,7 @@
calico_version: "v3.25" calico_version: "v3.25"
helm_version: "v3.6.3" helm_version: "v3.6.3"
yq_version: "v4.6.0" yq_version: "v4.6.0"
crictl_version: "v1.26.1"
zuul_osh_infra_relative_path: ../openstack-helm-infra zuul_osh_infra_relative_path: ../openstack-helm-infra
gate_scripts_relative_path: ../openstack-helm gate_scripts_relative_path: ../openstack-helm