[WIP] Refactor Openstack image build

Change-Id: I7c63ee98125bf0988b3926e40dd916087cbb9476
2024-04-27 00:26:16 -05:00 · 2024-04-27 00:26:16 -05:00 · 6e67de3ed8
parent 9c19b92dea
commit 6e67de3ed8
6 changed files with 154 additions and 161 deletions
--- a/zuul.d/openstack-2023.2.yaml
+++ b/zuul.d/openstack-2023.2.yaml
@ -11,29 +11,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

- project:
-    check:
-      jobs:
-        - openstack-helm-images-build-openstack-2023-2-ubuntu_jammy
-        - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
-        - openstack-helm-images-cinder-2023-2-ubuntu_jammy
-    gate:
-      jobs:
-        - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
-        - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
-        - openstack-helm-images-cinder-2023-2-ubuntu_jammy
-    promote:
-      jobs:
-        - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy
-    periodic-weekly:
-      jobs:
-        - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
-        - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy:
-            dependencies:
-              - name: openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
-              - name: openstack-helm-images-cinder-2023-2-ubuntu_jammy
-        - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
-        - openstack-helm-images-cinder-2023-2-ubuntu_jammy
+# - project:
+#     check:
+#       jobs:
+#         - openstack-helm-images-build-openstack-2023-2-ubuntu_jammy
+#         - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+#         - openstack-helm-images-cinder-2023-2-ubuntu_jammy
+#     gate:
+#       jobs:
+#         - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
+#         - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+#         - openstack-helm-images-cinder-2023-2-ubuntu_jammy
+#     promote:
+#       jobs:
+#         - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy
+#     periodic-weekly:
+#       jobs:
+#         - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
+#         - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy:
+#             dependencies:
+#               - name: openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+#               - name: openstack-helm-images-cinder-2023-2-ubuntu_jammy
+#         - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+#         - openstack-helm-images-cinder-2023-2-ubuntu_jammy

 - job:
    name: openstack-helm-images-build-openstack-2023-2-ubuntu_jammy
--- a/zuul.d/openstack-2024.1.yaml
+++ b/zuul.d/openstack-2024.1.yaml
@ -15,14 +15,11 @@
 - project:
    check:
      jobs:
+        - openstack-helm-images-build-openstack-base-2024-1-ubuntu_jammy
        - openstack-helm-images-build-openstack-2024-1-ubuntu_jammy
-        # - openstack-helm-images-compute-kit-2024-1-ubuntu_jammy
-        # - openstack-helm-images-cinder-2024-1-ubuntu_jammy
    gate:
      jobs:
        - openstack-helm-images-upload-openstack-2024-1-ubuntu_jammy
-        # - openstack-helm-images-compute-kit-2024-1-ubuntu_jammy
-        # - openstack-helm-images-cinder-2024-1-ubuntu_jammy
    promote:
      jobs:
        - openstack-helm-images-promote-openstack-2024-1-ubuntu_jammy
@ -32,37 +29,42 @@
        - openstack-helm-images-promote-openstack-2024-1-ubuntu_jammy:
            dependencies:
              - name: openstack-helm-images-upload-openstack-2024-1-ubuntu_jammy
-              # - name: openstack-helm-images-compute-kit-2024-1-ubuntu_jammy
-              # - name: openstack-helm-images-cinder-2024-1-ubuntu_jammy
-        # - openstack-helm-images-compute-kit-2024-1-ubuntu_jammy
-        # - openstack-helm-images-cinder-2024-1-ubuntu_jammy
+
+- job:
+    name: openstack-helm-images-build-openstack-base-2024-1-ubuntu_jammy
+    description: Build base 2024.1 Jammy images
+    parent: openstack-helm-images-build-openstack-ubuntu_jammy
+    run:
+      - zuul.d/playbooks/build-openstack-base.yml

 - job:
    name: openstack-helm-images-build-openstack-2024-1-ubuntu_jammy
    description: Build 2024.1 Jammy images
    parent: openstack-helm-images-build-openstack-ubuntu_jammy
+    dependencies:
+      - openstack-helm-images-build-openstack-base-2024-1-ubuntu_jammy
    vars: &build_vars
      branchname: "stable/2024.1"
      shortbranchname: "2024.1"
      openstack_docker_images:
-        - requirements
+        # - requirements
        - keystone
-        - glance
-        - cinder
-        - designate
-        - nova
-        - ironic
-        - neutron
-        - horizon
-        - heat
-        - barbican
-        - placement
-        - cyborg
-        - magnum
-        - masakari
-        - masakari_monitors
-        - manila
-        - tacker
+        # - glance
+        # - cinder
+        # - designate
+        # - nova
+        # - ironic
+        # - neutron
+        # - horizon
+        # - heat
+        # - barbican
+        # - placement
+        # - cyborg
+        # - magnum
+        # - masakari
+        # - masakari_monitors
+        # - manila
+        # - tacker
    files: &openstack_files
      - ^openstack/loci/.*
      - zuul.d/openstack-2024.1.yaml
@ -85,58 +87,3 @@
    vars: *build_vars
    files: *openstack_files

- job:
-    name: openstack-helm-images-compute-kit-2024-1-ubuntu_jammy
-    parent: openstack-helm-compute-kit-2024-1-ubuntu_jammy
-    dependencies:
-      - name: openstack-helm-images-build-openvswitch
-        soft: true
-      - name: openstack-helm-images-build-openstack-2024-1-ubuntu_jammy
-        soft: true
-      - name: openstack-helm-images-build-mariadb
-        soft: true
-      - name: openstack-helm-images-build-libvirt
-        soft: true
-      - name: openstack-helm-images-upload-openvswitch
-        soft: true
-      - name: openstack-helm-images-upload-openstack-2024-1-ubuntu_jammy
-        soft: true
-      - name: openstack-helm-images-upload-mariadb
-        soft: true
-      - name: openstack-helm-images-upload-libvirt
-        soft: true
-    files:
-      - openvswitch/.*
-      - zuul.d/openvswitch.yaml
-      - mariadb/.*
-      - zuul.d/mariadb.yaml
-      - libvirt/.*
-      - zuul.d/libvirt.yaml
-      - ^openstack/loci/.*
-      - zuul.d/openstack-2024.1.yaml
-      - zuul.d/openstack-base.yaml
-      - zuul.d/playbooks/build-openstack.yml
-      - zuul.d/playbooks/vars-openstack.yml
-      - zuul.d/playbooks/Dockerfile_base
-
- job:
-    name: openstack-helm-images-cinder-2024-1-ubuntu_jammy
-    parent: openstack-helm-cinder-2024-1-ubuntu_jammy
-    dependencies:
-      - name: openstack-helm-images-build-openstack-2024-1-ubuntu_jammy
-        soft: true
-      - name: openstack-helm-images-build-mariadb
-        soft: true
-      - name: openstack-helm-images-upload-openstack-2024-1-ubuntu_jammy
-        soft: true
-      - name: openstack-helm-images-upload-mariadb
-        soft: true
-    files:
-      - mariadb/.*
-      - zuul.d/mariadb.yaml
-      - ^openstack/loci/.*
-      - zuul.d/openstack-2024.1.yaml
-      - zuul.d/openstack-base.yaml
-      - zuul.d/playbooks/build-openstack.yml
-      - zuul.d/playbooks/vars-openstack.yml
-      - zuul.d/playbooks/Dockerfile_base
--- a/zuul.d/openstack-base.yaml
+++ b/zuul.d/openstack-base.yaml
@ -28,7 +28,7 @@
      ceph_repo: ""
      ceph_key: ""
      openstack_docker_images:
-        - requirements
+        # - requirements
        - keystone
        - glance
        - cinder
--- a/zuul.d/playbooks/build-openstack-base.yml
+++ b/zuul.d/playbooks/build-openstack-base.yml
@ -0,0 +1,37 @@
+---
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: all[0]
+  gather_facts: true
+  vars_files:
+    - vars-openstack.yml
+
+  pre_tasks:
+    - name: Use buildset registry
+      include_role:
+        name: use-buildset-registry
+
+  tasks:
+    - name: Build base image
+      include_role:
+        name: build-docker-image
+      vars:
+        zuul_work_dir: "{{ zuul.project.src_dir }}"
+        docker_images: "{{ base_docker_images }}"
+
+    - name: Build requirements image
+      include_role:
+        name: build-docker-image
+      vars:
+        zuul_work_dir: "src/opendev.org/openstack/loci"
+        docker_images: "{{ requirements_docker_images }}"
--- a/zuul.d/playbooks/build-openstack.yml
+++ b/zuul.d/playbooks/build-openstack.yml
@ -17,60 +17,62 @@
    - vars-openstack.yml
  vars:
    docker_images: "{{ openstack_docker_images | map('extract', openstack_docker_images_dict) | list }}"
+    wheels_location: "{{ buildset_registry_alias }}/requirements:{{ shortbranchname }}-{{ distroname }}"
+
  pre_tasks:
    - name: Use buildset registry
      include_role:
        name: use-buildset-registry
+
  tasks:
-    - name: Export the right vars
-      shell: |
-        source /etc/ci/mirror_info.sh
-        echo ${NODEPOOL_OPENSUSE_MIRROR}/
-        echo ${NODEPOOL_PYPI_MIRROR}
-        echo ${NODEPOOL_MIRROR_HOST}
-      args:
-        executable: /bin/bash
-      changed_when: False
-      register: _zuul_mirrors
-    - name: Set fact with the exported variables
-      set_fact:
-        opensuse_package_mirror: "{{ _zuul_mirrors.stdout_lines[0] }}"
-        pip_index_url: "{{ _zuul_mirrors.stdout_lines[1] | default('https://pypi.python.org/simple/') }}"
-        pip_trusted_host: "{{ _zuul_mirrors.stdout_lines[2] | default('pypi.python.org') }}"
+    # - name: Export the right vars
+    #   shell: |
+    #     source /etc/ci/mirror_info.sh
+    #     echo ${NODEPOOL_PYPI_MIRROR}
+    #     echo ${NODEPOOL_MIRROR_HOST}
+    #   args:
+    #     executable: /bin/bash
+    #   changed_when: False
+    #   register: _zuul_mirrors

-    - name: Build base image
-      include_role:
-        name: build-docker-image
-      vars:
-        zuul_work_dir: "{{ zuul.project.src_dir }}"
-        docker_images: "{{ base_docker_images }}"
+    # - name: Set fact with the exported variables
+    #   set_fact:
+    #     pip_index_url: "{{ _zuul_mirrors.stdout_lines[0] | default('https://pypi.python.org/simple/') }}"
+    #     pip_trusted_host: "{{ _zuul_mirrors.stdout_lines[1] | default('pypi.python.org') }}"

-    - name: Build base requirements image
-      include_role:
-        name: build-docker-image
-      vars:
-        zuul_work_dir: "src/opendev.org/openstack/loci"
-        docker_images: "{{ requirements_docker_images }}"
+    # - name: Build base image
+    #   include_role:
+    #     name: build-docker-image
+    #   vars:
+    #     zuul_work_dir: "{{ zuul.project.src_dir }}"
+    #     docker_images: "{{ base_docker_images }}"

-    - name: Build webserver serving wheels
-      include_role:
-        name: build-docker-image
-      vars:
-        zuul_work_dir: "{{ zuul.project.src_dir }}"
-        docker_images:
-          - context: zuul.d/playbooks/
-            repository: openstackhelm/loci_wheels
+    # - name: Build requirements image
+    #   include_role:
+    #     name: build-docker-image
+    #   vars:
+    #     zuul_work_dir: "src/opendev.org/openstack/loci"
+    #     docker_images: "{{ requirements_docker_images }}"

-    - name: Allow access to wheels server
-      iptables:
-        action: insert
-        chain: INPUT
-        in_interface: docker0
-        jump: ACCEPT
-      become: yes
+    # - name: Build webserver serving wheels
+    #   include_role:
+    #     name: build-docker-image
+    #   vars:
+    #     zuul_work_dir: "{{ zuul.project.src_dir }}"
+    #     docker_images:
+    #       - context: zuul.d/playbooks/
+    #         repository: openstackhelm/loci_wheels

-    - name: Run http server containing wheels
-      command: docker run -d -p 0.0.0.0:8080:80 openstackhelm/loci_wheels:latest
+    # - name: Allow access to wheels server
+    #   iptables:
+    #     action: insert
+    #     chain: INPUT
+    #     in_interface: docker0
+    #     jump: ACCEPT
+    #   become: yes
+
+    # - name: Run http server containing wheels
+    #   command: docker run -d -p 0.0.0.0:8080:80 openstackhelm/loci_wheels:latest

    - name: Build project images
      include_role:
--- a/zuul.d/playbooks/vars-openstack.yml
+++ b/zuul.d/playbooks/vars-openstack.yml
@ -14,8 +14,13 @@
 currentdate: "{{ now(utc=True,fmt='%Y%m%d') }}"
 python3: "yes"
 #172.17.0.1 is default docker0 ip.
-wheels_location: "http://172.17.0.1:8080/wheels.tar.gz"
+# wheels_location: "http://172.17.0.1:8080/wheels.tar.gz"
+# wheels_location: "{{ buildset_registry_alias }}/requirements:{{ shortbranchname }}-{{ distroname }}"
 base_repository: "base"
+requirements_repository: "requirements"
+tags: &imagetag
+  - "{{ shortbranchname }}-{{ distroname }}"
+  - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}"

 base_docker_images:
  - context: "zuul.d/playbooks/"
@ -25,6 +30,7 @@ base_docker_images:
      - "FROM='{{ from }}'"
      - "CEPH_REPO='{{ ceph_repo }}'"
      - "CEPH_KEY='{{ ceph_key }}'"
+    tags: *imagetag

 requirements_docker_images:
  - context: "."
@ -34,24 +40,25 @@ requirements_docker_images:
      - "PROJECT='requirements'"
      - "PROJECT_REF={{ branchname }}"
      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag

 openstack_docker_images_dict:
-  requirements:
-    context: "."
-    repository: openstackhelm/requirements
-    build_args:
-      - "FROM='{{ base_repository }}'"
-      - "PROJECT='requirements'"
-      - "PROJECT_REF={{ branchname }}"
-      - "KEEP_ALL_WHEELS=yes"
-    tags: &imagetag
-      - "{{ shortbranchname }}-{{ distroname }}"
-      - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}" # Frozen tag
+  # requirements:
+  #   context: "."
+  #   repository: openstackhelm/requirements
+  #   build_args:
+  #     - "FROM='{{ base_repository }}'"
+  #     - "PROJECT='requirements'"
+  #     - "PROJECT_REF={{ branchname }}"
+  #     - "KEEP_ALL_WHEELS=yes"
+  #   tags: &imagetag
+  #     - "{{ shortbranchname }}-{{ distroname }}"
+  #     - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}" # Frozen tag
  keystone:
    context: "."
    repository: openstackhelm/keystone
    build_args:
-      - "FROM='{{ base_repository }}'"
+      - "FROM='{{ base_repository }}:{{ shortbranchname }}-{{ distroname }}'"
      - "PROJECT='keystone'"
      - "PROJECT_REF={{ branchname }}"
      - "PROFILES='fluent apache ldap {{ shortbranchname }}'"