From 6e67de3ed8035955dda57698d0a30f6a1f2b017e Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Sat, 27 Apr 2024 00:26:16 -0500 Subject: [PATCH] [WIP] Refactor Openstack image build Change-Id: I7c63ee98125bf0988b3926e40dd916087cbb9476 --- zuul.d/openstack-2023.2.yaml | 46 +++++----- zuul.d/openstack-2024.1.yaml | 107 ++++++---------------- zuul.d/openstack-base.yaml | 2 +- zuul.d/playbooks/build-openstack-base.yml | 37 ++++++++ zuul.d/playbooks/build-openstack.yml | 90 +++++++++--------- zuul.d/playbooks/vars-openstack.yml | 33 ++++--- 6 files changed, 154 insertions(+), 161 deletions(-) create mode 100644 zuul.d/playbooks/build-openstack-base.yml diff --git a/zuul.d/openstack-2023.2.yaml b/zuul.d/openstack-2023.2.yaml index 135b0541..4ccdb4b0 100644 --- a/zuul.d/openstack-2023.2.yaml +++ b/zuul.d/openstack-2023.2.yaml @@ -11,29 +11,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -- project: - check: - jobs: - - openstack-helm-images-build-openstack-2023-2-ubuntu_jammy - - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy - - openstack-helm-images-cinder-2023-2-ubuntu_jammy - gate: - jobs: - - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy - - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy - - openstack-helm-images-cinder-2023-2-ubuntu_jammy - promote: - jobs: - - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy - periodic-weekly: - jobs: - - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy - - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy: - dependencies: - - name: openstack-helm-images-compute-kit-2023-2-ubuntu_jammy - - name: openstack-helm-images-cinder-2023-2-ubuntu_jammy - - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy - - openstack-helm-images-cinder-2023-2-ubuntu_jammy +# - project: +# check: +# jobs: +# - openstack-helm-images-build-openstack-2023-2-ubuntu_jammy +# - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy +# - openstack-helm-images-cinder-2023-2-ubuntu_jammy +# gate: +# jobs: +# - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy +# - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy +# - openstack-helm-images-cinder-2023-2-ubuntu_jammy +# promote: +# jobs: +# - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy +# periodic-weekly: +# jobs: +# - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy +# - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy: +# dependencies: +# - name: openstack-helm-images-compute-kit-2023-2-ubuntu_jammy +# - name: openstack-helm-images-cinder-2023-2-ubuntu_jammy +# - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy +# - openstack-helm-images-cinder-2023-2-ubuntu_jammy - job: name: openstack-helm-images-build-openstack-2023-2-ubuntu_jammy diff --git a/zuul.d/openstack-2024.1.yaml b/zuul.d/openstack-2024.1.yaml index ca684a56..3d162a6d 100644 --- a/zuul.d/openstack-2024.1.yaml +++ b/zuul.d/openstack-2024.1.yaml @@ -15,14 +15,11 @@ - project: check: jobs: + - openstack-helm-images-build-openstack-base-2024-1-ubuntu_jammy - openstack-helm-images-build-openstack-2024-1-ubuntu_jammy - # - openstack-helm-images-compute-kit-2024-1-ubuntu_jammy - # - openstack-helm-images-cinder-2024-1-ubuntu_jammy gate: jobs: - openstack-helm-images-upload-openstack-2024-1-ubuntu_jammy - # - openstack-helm-images-compute-kit-2024-1-ubuntu_jammy - # - openstack-helm-images-cinder-2024-1-ubuntu_jammy promote: jobs: - openstack-helm-images-promote-openstack-2024-1-ubuntu_jammy @@ -32,37 +29,42 @@ - openstack-helm-images-promote-openstack-2024-1-ubuntu_jammy: dependencies: - name: openstack-helm-images-upload-openstack-2024-1-ubuntu_jammy - # - name: openstack-helm-images-compute-kit-2024-1-ubuntu_jammy - # - name: openstack-helm-images-cinder-2024-1-ubuntu_jammy - # - openstack-helm-images-compute-kit-2024-1-ubuntu_jammy - # - openstack-helm-images-cinder-2024-1-ubuntu_jammy + +- job: + name: openstack-helm-images-build-openstack-base-2024-1-ubuntu_jammy + description: Build base 2024.1 Jammy images + parent: openstack-helm-images-build-openstack-ubuntu_jammy + run: + - zuul.d/playbooks/build-openstack-base.yml - job: name: openstack-helm-images-build-openstack-2024-1-ubuntu_jammy description: Build 2024.1 Jammy images parent: openstack-helm-images-build-openstack-ubuntu_jammy + dependencies: + - openstack-helm-images-build-openstack-base-2024-1-ubuntu_jammy vars: &build_vars branchname: "stable/2024.1" shortbranchname: "2024.1" openstack_docker_images: - - requirements + # - requirements - keystone - - glance - - cinder - - designate - - nova - - ironic - - neutron - - horizon - - heat - - barbican - - placement - - cyborg - - magnum - - masakari - - masakari_monitors - - manila - - tacker + # - glance + # - cinder + # - designate + # - nova + # - ironic + # - neutron + # - horizon + # - heat + # - barbican + # - placement + # - cyborg + # - magnum + # - masakari + # - masakari_monitors + # - manila + # - tacker files: &openstack_files - ^openstack/loci/.* - zuul.d/openstack-2024.1.yaml @@ -85,58 +87,3 @@ vars: *build_vars files: *openstack_files -- job: - name: openstack-helm-images-compute-kit-2024-1-ubuntu_jammy - parent: openstack-helm-compute-kit-2024-1-ubuntu_jammy - dependencies: - - name: openstack-helm-images-build-openvswitch - soft: true - - name: openstack-helm-images-build-openstack-2024-1-ubuntu_jammy - soft: true - - name: openstack-helm-images-build-mariadb - soft: true - - name: openstack-helm-images-build-libvirt - soft: true - - name: openstack-helm-images-upload-openvswitch - soft: true - - name: openstack-helm-images-upload-openstack-2024-1-ubuntu_jammy - soft: true - - name: openstack-helm-images-upload-mariadb - soft: true - - name: openstack-helm-images-upload-libvirt - soft: true - files: - - openvswitch/.* - - zuul.d/openvswitch.yaml - - mariadb/.* - - zuul.d/mariadb.yaml - - libvirt/.* - - zuul.d/libvirt.yaml - - ^openstack/loci/.* - - zuul.d/openstack-2024.1.yaml - - zuul.d/openstack-base.yaml - - zuul.d/playbooks/build-openstack.yml - - zuul.d/playbooks/vars-openstack.yml - - zuul.d/playbooks/Dockerfile_base - -- job: - name: openstack-helm-images-cinder-2024-1-ubuntu_jammy - parent: openstack-helm-cinder-2024-1-ubuntu_jammy - dependencies: - - name: openstack-helm-images-build-openstack-2024-1-ubuntu_jammy - soft: true - - name: openstack-helm-images-build-mariadb - soft: true - - name: openstack-helm-images-upload-openstack-2024-1-ubuntu_jammy - soft: true - - name: openstack-helm-images-upload-mariadb - soft: true - files: - - mariadb/.* - - zuul.d/mariadb.yaml - - ^openstack/loci/.* - - zuul.d/openstack-2024.1.yaml - - zuul.d/openstack-base.yaml - - zuul.d/playbooks/build-openstack.yml - - zuul.d/playbooks/vars-openstack.yml - - zuul.d/playbooks/Dockerfile_base diff --git a/zuul.d/openstack-base.yaml b/zuul.d/openstack-base.yaml index d214d2c1..aeb5ed24 100644 --- a/zuul.d/openstack-base.yaml +++ b/zuul.d/openstack-base.yaml @@ -28,7 +28,7 @@ ceph_repo: "" ceph_key: "" openstack_docker_images: - - requirements + # - requirements - keystone - glance - cinder diff --git a/zuul.d/playbooks/build-openstack-base.yml b/zuul.d/playbooks/build-openstack-base.yml new file mode 100644 index 00000000..b73b90d4 --- /dev/null +++ b/zuul.d/playbooks/build-openstack-base.yml @@ -0,0 +1,37 @@ +--- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: all[0] + gather_facts: true + vars_files: + - vars-openstack.yml + + pre_tasks: + - name: Use buildset registry + include_role: + name: use-buildset-registry + + tasks: + - name: Build base image + include_role: + name: build-docker-image + vars: + zuul_work_dir: "{{ zuul.project.src_dir }}" + docker_images: "{{ base_docker_images }}" + + - name: Build requirements image + include_role: + name: build-docker-image + vars: + zuul_work_dir: "src/opendev.org/openstack/loci" + docker_images: "{{ requirements_docker_images }}" diff --git a/zuul.d/playbooks/build-openstack.yml b/zuul.d/playbooks/build-openstack.yml index 29c9d406..ec8cd52a 100644 --- a/zuul.d/playbooks/build-openstack.yml +++ b/zuul.d/playbooks/build-openstack.yml @@ -17,60 +17,62 @@ - vars-openstack.yml vars: docker_images: "{{ openstack_docker_images | map('extract', openstack_docker_images_dict) | list }}" + wheels_location: "{{ buildset_registry_alias }}/requirements:{{ shortbranchname }}-{{ distroname }}" + pre_tasks: - name: Use buildset registry include_role: name: use-buildset-registry + tasks: - - name: Export the right vars - shell: | - source /etc/ci/mirror_info.sh - echo ${NODEPOOL_OPENSUSE_MIRROR}/ - echo ${NODEPOOL_PYPI_MIRROR} - echo ${NODEPOOL_MIRROR_HOST} - args: - executable: /bin/bash - changed_when: False - register: _zuul_mirrors - - name: Set fact with the exported variables - set_fact: - opensuse_package_mirror: "{{ _zuul_mirrors.stdout_lines[0] }}" - pip_index_url: "{{ _zuul_mirrors.stdout_lines[1] | default('https://pypi.python.org/simple/') }}" - pip_trusted_host: "{{ _zuul_mirrors.stdout_lines[2] | default('pypi.python.org') }}" + # - name: Export the right vars + # shell: | + # source /etc/ci/mirror_info.sh + # echo ${NODEPOOL_PYPI_MIRROR} + # echo ${NODEPOOL_MIRROR_HOST} + # args: + # executable: /bin/bash + # changed_when: False + # register: _zuul_mirrors - - name: Build base image - include_role: - name: build-docker-image - vars: - zuul_work_dir: "{{ zuul.project.src_dir }}" - docker_images: "{{ base_docker_images }}" + # - name: Set fact with the exported variables + # set_fact: + # pip_index_url: "{{ _zuul_mirrors.stdout_lines[0] | default('https://pypi.python.org/simple/') }}" + # pip_trusted_host: "{{ _zuul_mirrors.stdout_lines[1] | default('pypi.python.org') }}" - - name: Build base requirements image - include_role: - name: build-docker-image - vars: - zuul_work_dir: "src/opendev.org/openstack/loci" - docker_images: "{{ requirements_docker_images }}" + # - name: Build base image + # include_role: + # name: build-docker-image + # vars: + # zuul_work_dir: "{{ zuul.project.src_dir }}" + # docker_images: "{{ base_docker_images }}" - - name: Build webserver serving wheels - include_role: - name: build-docker-image - vars: - zuul_work_dir: "{{ zuul.project.src_dir }}" - docker_images: - - context: zuul.d/playbooks/ - repository: openstackhelm/loci_wheels + # - name: Build requirements image + # include_role: + # name: build-docker-image + # vars: + # zuul_work_dir: "src/opendev.org/openstack/loci" + # docker_images: "{{ requirements_docker_images }}" - - name: Allow access to wheels server - iptables: - action: insert - chain: INPUT - in_interface: docker0 - jump: ACCEPT - become: yes + # - name: Build webserver serving wheels + # include_role: + # name: build-docker-image + # vars: + # zuul_work_dir: "{{ zuul.project.src_dir }}" + # docker_images: + # - context: zuul.d/playbooks/ + # repository: openstackhelm/loci_wheels - - name: Run http server containing wheels - command: docker run -d -p 0.0.0.0:8080:80 openstackhelm/loci_wheels:latest + # - name: Allow access to wheels server + # iptables: + # action: insert + # chain: INPUT + # in_interface: docker0 + # jump: ACCEPT + # become: yes + + # - name: Run http server containing wheels + # command: docker run -d -p 0.0.0.0:8080:80 openstackhelm/loci_wheels:latest - name: Build project images include_role: diff --git a/zuul.d/playbooks/vars-openstack.yml b/zuul.d/playbooks/vars-openstack.yml index b8bd0760..da9972c5 100644 --- a/zuul.d/playbooks/vars-openstack.yml +++ b/zuul.d/playbooks/vars-openstack.yml @@ -14,8 +14,13 @@ currentdate: "{{ now(utc=True,fmt='%Y%m%d') }}" python3: "yes" #172.17.0.1 is default docker0 ip. -wheels_location: "http://172.17.0.1:8080/wheels.tar.gz" +# wheels_location: "http://172.17.0.1:8080/wheels.tar.gz" +# wheels_location: "{{ buildset_registry_alias }}/requirements:{{ shortbranchname }}-{{ distroname }}" base_repository: "base" +requirements_repository: "requirements" +tags: &imagetag + - "{{ shortbranchname }}-{{ distroname }}" + - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}" base_docker_images: - context: "zuul.d/playbooks/" @@ -25,6 +30,7 @@ base_docker_images: - "FROM='{{ from }}'" - "CEPH_REPO='{{ ceph_repo }}'" - "CEPH_KEY='{{ ceph_key }}'" + tags: *imagetag requirements_docker_images: - context: "." @@ -34,24 +40,25 @@ requirements_docker_images: - "PROJECT='requirements'" - "PROJECT_REF={{ branchname }}" - "KEEP_ALL_WHEELS=yes" + tags: *imagetag openstack_docker_images_dict: - requirements: - context: "." - repository: openstackhelm/requirements - build_args: - - "FROM='{{ base_repository }}'" - - "PROJECT='requirements'" - - "PROJECT_REF={{ branchname }}" - - "KEEP_ALL_WHEELS=yes" - tags: &imagetag - - "{{ shortbranchname }}-{{ distroname }}" - - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}" # Frozen tag + # requirements: + # context: "." + # repository: openstackhelm/requirements + # build_args: + # - "FROM='{{ base_repository }}'" + # - "PROJECT='requirements'" + # - "PROJECT_REF={{ branchname }}" + # - "KEEP_ALL_WHEELS=yes" + # tags: &imagetag + # - "{{ shortbranchname }}-{{ distroname }}" + # - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}" # Frozen tag keystone: context: "." repository: openstackhelm/keystone build_args: - - "FROM='{{ base_repository }}'" + - "FROM='{{ base_repository }}:{{ shortbranchname }}-{{ distroname }}'" - "PROJECT='keystone'" - "PROJECT_REF={{ branchname }}" - "PROFILES='fluent apache ldap {{ shortbranchname }}'"