From 1251b508b6bc68a39e129fcfe862c76ef7e4ae7a Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Sat, 27 Apr 2024 00:26:16 -0500 Subject: [PATCH] [WIP] Refactor Openstack image build Change-Id: I7c63ee98125bf0988b3926e40dd916087cbb9476 --- zuul.d/base.yaml | 8 +- zuul.d/openstack-2023.2.yaml | 46 ++++----- zuul.d/openstack-2024.1.yaml | 113 ++++++---------------- zuul.d/openstack-base.yaml | 2 +- zuul.d/playbooks/build-openstack-base.yml | 52 ++++++++++ zuul.d/playbooks/build-openstack.yml | 12 ++- zuul.d/playbooks/vars-openstack.yml | 33 ++++--- 7 files changed, 139 insertions(+), 127 deletions(-) create mode 100644 zuul.d/playbooks/build-openstack-base.yml diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml index 428af3be..7d3d78e0 100644 --- a/zuul.d/base.yaml +++ b/zuul.d/base.yaml @@ -56,9 +56,9 @@ - openstack-helm-images-compute-kit-2023-1-ubuntu_jammy - openstack-helm-images-cinder-2023-1-ubuntu_jammy - openstack-helm-images-horizon-2023-1-ubuntu_jammy - templates: - - publish-openstack-docs-pti - - release-notes-jobs-python3 + # templates: + # - publish-openstack-docs-pti + # - release-notes-jobs-python3 # This takes the same structure as other infra jobs. # This dict keyname matter. See also job definition for more explanations. @@ -92,7 +92,7 @@ name: openstack-helm-images-buildset-registry parent: opendev-buildset-registry timeout: 10800 - voting: false + # voting: false # In docker_images: # `context` is the folder in osh-images diff --git a/zuul.d/openstack-2023.2.yaml b/zuul.d/openstack-2023.2.yaml index 135b0541..4ccdb4b0 100644 --- a/zuul.d/openstack-2023.2.yaml +++ b/zuul.d/openstack-2023.2.yaml @@ -11,29 +11,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -- project: - check: - jobs: - - openstack-helm-images-build-openstack-2023-2-ubuntu_jammy - - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy - - openstack-helm-images-cinder-2023-2-ubuntu_jammy - gate: - jobs: - - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy - - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy - - openstack-helm-images-cinder-2023-2-ubuntu_jammy - promote: - jobs: - - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy - periodic-weekly: - jobs: - - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy - - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy: - dependencies: - - name: openstack-helm-images-compute-kit-2023-2-ubuntu_jammy - - name: openstack-helm-images-cinder-2023-2-ubuntu_jammy - - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy - - openstack-helm-images-cinder-2023-2-ubuntu_jammy +# - project: +# check: +# jobs: +# - openstack-helm-images-build-openstack-2023-2-ubuntu_jammy +# - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy +# - openstack-helm-images-cinder-2023-2-ubuntu_jammy +# gate: +# jobs: +# - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy +# - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy +# - openstack-helm-images-cinder-2023-2-ubuntu_jammy +# promote: +# jobs: +# - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy +# periodic-weekly: +# jobs: +# - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy +# - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy: +# dependencies: +# - name: openstack-helm-images-compute-kit-2023-2-ubuntu_jammy +# - name: openstack-helm-images-cinder-2023-2-ubuntu_jammy +# - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy +# - openstack-helm-images-cinder-2023-2-ubuntu_jammy - job: name: openstack-helm-images-build-openstack-2023-2-ubuntu_jammy diff --git a/zuul.d/openstack-2024.1.yaml b/zuul.d/openstack-2024.1.yaml index ca684a56..621ec706 100644 --- a/zuul.d/openstack-2024.1.yaml +++ b/zuul.d/openstack-2024.1.yaml @@ -15,14 +15,11 @@ - project: check: jobs: - - openstack-helm-images-build-openstack-2024-1-ubuntu_jammy - # - openstack-helm-images-compute-kit-2024-1-ubuntu_jammy - # - openstack-helm-images-cinder-2024-1-ubuntu_jammy + - openstack-helm-images-build-openstack-base-2024-1-ubuntu_jammy + # - openstack-helm-images-build-openstack-2024-1-ubuntu_jammy gate: jobs: - openstack-helm-images-upload-openstack-2024-1-ubuntu_jammy - # - openstack-helm-images-compute-kit-2024-1-ubuntu_jammy - # - openstack-helm-images-cinder-2024-1-ubuntu_jammy promote: jobs: - openstack-helm-images-promote-openstack-2024-1-ubuntu_jammy @@ -32,37 +29,46 @@ - openstack-helm-images-promote-openstack-2024-1-ubuntu_jammy: dependencies: - name: openstack-helm-images-upload-openstack-2024-1-ubuntu_jammy - # - name: openstack-helm-images-compute-kit-2024-1-ubuntu_jammy - # - name: openstack-helm-images-cinder-2024-1-ubuntu_jammy - # - openstack-helm-images-compute-kit-2024-1-ubuntu_jammy - # - openstack-helm-images-cinder-2024-1-ubuntu_jammy + +- job: + name: openstack-helm-images-build-openstack-base-2024-1-ubuntu_jammy + description: Build base 2024.1 Jammy images + parent: openstack-helm-images-build-openstack-ubuntu_jammy + vars: + branchname: "stable/2024.1" + shortbranchname: "2024.1" + openstack_docker_images: [] + # run: + # - zuul.d/playbooks/build-openstack-base.yml - job: name: openstack-helm-images-build-openstack-2024-1-ubuntu_jammy description: Build 2024.1 Jammy images parent: openstack-helm-images-build-openstack-ubuntu_jammy + dependencies: + - openstack-helm-images-build-openstack-base-2024-1-ubuntu_jammy vars: &build_vars branchname: "stable/2024.1" shortbranchname: "2024.1" openstack_docker_images: - - requirements + # - requirements - keystone - - glance - - cinder - - designate - - nova - - ironic - - neutron - - horizon - - heat - - barbican - - placement - - cyborg - - magnum - - masakari - - masakari_monitors - - manila - - tacker + # - glance + # - cinder + # - designate + # - nova + # - ironic + # - neutron + # - horizon + # - heat + # - barbican + # - placement + # - cyborg + # - magnum + # - masakari + # - masakari_monitors + # - manila + # - tacker files: &openstack_files - ^openstack/loci/.* - zuul.d/openstack-2024.1.yaml @@ -85,58 +91,3 @@ vars: *build_vars files: *openstack_files -- job: - name: openstack-helm-images-compute-kit-2024-1-ubuntu_jammy - parent: openstack-helm-compute-kit-2024-1-ubuntu_jammy - dependencies: - - name: openstack-helm-images-build-openvswitch - soft: true - - name: openstack-helm-images-build-openstack-2024-1-ubuntu_jammy - soft: true - - name: openstack-helm-images-build-mariadb - soft: true - - name: openstack-helm-images-build-libvirt - soft: true - - name: openstack-helm-images-upload-openvswitch - soft: true - - name: openstack-helm-images-upload-openstack-2024-1-ubuntu_jammy - soft: true - - name: openstack-helm-images-upload-mariadb - soft: true - - name: openstack-helm-images-upload-libvirt - soft: true - files: - - openvswitch/.* - - zuul.d/openvswitch.yaml - - mariadb/.* - - zuul.d/mariadb.yaml - - libvirt/.* - - zuul.d/libvirt.yaml - - ^openstack/loci/.* - - zuul.d/openstack-2024.1.yaml - - zuul.d/openstack-base.yaml - - zuul.d/playbooks/build-openstack.yml - - zuul.d/playbooks/vars-openstack.yml - - zuul.d/playbooks/Dockerfile_base - -- job: - name: openstack-helm-images-cinder-2024-1-ubuntu_jammy - parent: openstack-helm-cinder-2024-1-ubuntu_jammy - dependencies: - - name: openstack-helm-images-build-openstack-2024-1-ubuntu_jammy - soft: true - - name: openstack-helm-images-build-mariadb - soft: true - - name: openstack-helm-images-upload-openstack-2024-1-ubuntu_jammy - soft: true - - name: openstack-helm-images-upload-mariadb - soft: true - files: - - mariadb/.* - - zuul.d/mariadb.yaml - - ^openstack/loci/.* - - zuul.d/openstack-2024.1.yaml - - zuul.d/openstack-base.yaml - - zuul.d/playbooks/build-openstack.yml - - zuul.d/playbooks/vars-openstack.yml - - zuul.d/playbooks/Dockerfile_base diff --git a/zuul.d/openstack-base.yaml b/zuul.d/openstack-base.yaml index d214d2c1..aeb5ed24 100644 --- a/zuul.d/openstack-base.yaml +++ b/zuul.d/openstack-base.yaml @@ -28,7 +28,7 @@ ceph_repo: "" ceph_key: "" openstack_docker_images: - - requirements + # - requirements - keystone - glance - cinder diff --git a/zuul.d/playbooks/build-openstack-base.yml b/zuul.d/playbooks/build-openstack-base.yml new file mode 100644 index 00000000..4b993404 --- /dev/null +++ b/zuul.d/playbooks/build-openstack-base.yml @@ -0,0 +1,52 @@ +--- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: all[0] + gather_facts: true + vars_files: + - vars-openstack.yml + + pre_tasks: + - name: Use buildset registry + include_role: + name: use-buildset-registry + + tasks: + - name: Export the right vars + shell: | + source /etc/ci/mirror_info.sh + echo ${NODEPOOL_PYPI_MIRROR} + echo ${NODEPOOL_MIRROR_HOST} + args: + executable: /bin/bash + changed_when: False + register: _zuul_mirrors + + - name: Set fact with the exported variables + set_fact: + pip_index_url: "{{ _zuul_mirrors.stdout_lines[0] | default('https://pypi.python.org/simple/') }}" + pip_trusted_host: "{{ _zuul_mirrors.stdout_lines[1] | default('pypi.python.org') }}" + + - name: Build base image + include_role: + name: build-docker-image + vars: + zuul_work_dir: "{{ zuul.project.src_dir }}" + docker_images: "{{ base_docker_images }}" + + - name: Build requirements image + include_role: + name: build-docker-image + vars: + zuul_work_dir: "src/opendev.org/openstack/loci" + docker_images: "{{ requirements_docker_images }}" diff --git a/zuul.d/playbooks/build-openstack.yml b/zuul.d/playbooks/build-openstack.yml index 29c9d406..3fa6fd41 100644 --- a/zuul.d/playbooks/build-openstack.yml +++ b/zuul.d/playbooks/build-openstack.yml @@ -17,26 +17,28 @@ - vars-openstack.yml vars: docker_images: "{{ openstack_docker_images | map('extract', openstack_docker_images_dict) | list }}" + wheels_location: "{{ buildset_registry_alias }}/requirements:{{ shortbranchname }}-{{ distroname }}" + pre_tasks: - name: Use buildset registry include_role: name: use-buildset-registry + tasks: - name: Export the right vars shell: | source /etc/ci/mirror_info.sh - echo ${NODEPOOL_OPENSUSE_MIRROR}/ echo ${NODEPOOL_PYPI_MIRROR} echo ${NODEPOOL_MIRROR_HOST} args: executable: /bin/bash changed_when: False register: _zuul_mirrors + - name: Set fact with the exported variables set_fact: - opensuse_package_mirror: "{{ _zuul_mirrors.stdout_lines[0] }}" - pip_index_url: "{{ _zuul_mirrors.stdout_lines[1] | default('https://pypi.python.org/simple/') }}" - pip_trusted_host: "{{ _zuul_mirrors.stdout_lines[2] | default('pypi.python.org') }}" + pip_index_url: "{{ _zuul_mirrors.stdout_lines[0] | default('https://pypi.python.org/simple/') }}" + pip_trusted_host: "{{ _zuul_mirrors.stdout_lines[1] | default('pypi.python.org') }}" - name: Build base image include_role: @@ -45,7 +47,7 @@ zuul_work_dir: "{{ zuul.project.src_dir }}" docker_images: "{{ base_docker_images }}" - - name: Build base requirements image + - name: Build requirements image include_role: name: build-docker-image vars: diff --git a/zuul.d/playbooks/vars-openstack.yml b/zuul.d/playbooks/vars-openstack.yml index b8bd0760..da9972c5 100644 --- a/zuul.d/playbooks/vars-openstack.yml +++ b/zuul.d/playbooks/vars-openstack.yml @@ -14,8 +14,13 @@ currentdate: "{{ now(utc=True,fmt='%Y%m%d') }}" python3: "yes" #172.17.0.1 is default docker0 ip. -wheels_location: "http://172.17.0.1:8080/wheels.tar.gz" +# wheels_location: "http://172.17.0.1:8080/wheels.tar.gz" +# wheels_location: "{{ buildset_registry_alias }}/requirements:{{ shortbranchname }}-{{ distroname }}" base_repository: "base" +requirements_repository: "requirements" +tags: &imagetag + - "{{ shortbranchname }}-{{ distroname }}" + - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}" base_docker_images: - context: "zuul.d/playbooks/" @@ -25,6 +30,7 @@ base_docker_images: - "FROM='{{ from }}'" - "CEPH_REPO='{{ ceph_repo }}'" - "CEPH_KEY='{{ ceph_key }}'" + tags: *imagetag requirements_docker_images: - context: "." @@ -34,24 +40,25 @@ requirements_docker_images: - "PROJECT='requirements'" - "PROJECT_REF={{ branchname }}" - "KEEP_ALL_WHEELS=yes" + tags: *imagetag openstack_docker_images_dict: - requirements: - context: "." - repository: openstackhelm/requirements - build_args: - - "FROM='{{ base_repository }}'" - - "PROJECT='requirements'" - - "PROJECT_REF={{ branchname }}" - - "KEEP_ALL_WHEELS=yes" - tags: &imagetag - - "{{ shortbranchname }}-{{ distroname }}" - - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}" # Frozen tag + # requirements: + # context: "." + # repository: openstackhelm/requirements + # build_args: + # - "FROM='{{ base_repository }}'" + # - "PROJECT='requirements'" + # - "PROJECT_REF={{ branchname }}" + # - "KEEP_ALL_WHEELS=yes" + # tags: &imagetag + # - "{{ shortbranchname }}-{{ distroname }}" + # - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}" # Frozen tag keystone: context: "." repository: openstackhelm/keystone build_args: - - "FROM='{{ base_repository }}'" + - "FROM='{{ base_repository }}:{{ shortbranchname }}-{{ distroname }}'" - "PROJECT='keystone'" - "PROJECT_REF={{ branchname }}" - "PROFILES='fluent apache ldap {{ shortbranchname }}'"