[WIP] Refactor Openstack image build

Change-Id: I7c63ee98125bf0988b3926e40dd916087cbb9476

zuul.d/base.yaml

@@ -56,9 +56,9 @@
         - openstack-helm-images-compute-kit-2023-1-ubuntu_jammy
         - openstack-helm-images-cinder-2023-1-ubuntu_jammy
         - openstack-helm-images-horizon-2023-1-ubuntu_jammy
-    templates:
+    # templates:
-      - publish-openstack-docs-pti
+    #   - publish-openstack-docs-pti
-      - release-notes-jobs-python3
+    #   - release-notes-jobs-python3
 
 # This takes the same structure as other infra jobs.
 # This dict keyname matter. See also job definition for more explanations.
@@ -92,7 +92,7 @@
     name: openstack-helm-images-buildset-registry
     parent: opendev-buildset-registry
     timeout: 10800
-    voting: false
+    # voting: false
 
 # In docker_images:
 # `context` is the folder in osh-images

zuul.d/openstack-2023.2.yaml

@@ -11,29 +11,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- project:
+# - project:
-    check:
+#     check:
-      jobs:
+#       jobs:
-        - openstack-helm-images-build-openstack-2023-2-ubuntu_jammy
+#         - openstack-helm-images-build-openstack-2023-2-ubuntu_jammy
-        - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+#         - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
-        - openstack-helm-images-cinder-2023-2-ubuntu_jammy
+#         - openstack-helm-images-cinder-2023-2-ubuntu_jammy
-    gate:
+#     gate:
-      jobs:
+#       jobs:
-        - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
+#         - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
-        - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+#         - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
-        - openstack-helm-images-cinder-2023-2-ubuntu_jammy
+#         - openstack-helm-images-cinder-2023-2-ubuntu_jammy
-    promote:
+#     promote:
-      jobs:
+#       jobs:
-        - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy
+#         - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy
-    periodic-weekly:
+#     periodic-weekly:
-      jobs:
+#       jobs:
-        - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
+#         - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
-        - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy:
+#         - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy:
-            dependencies:
+#             dependencies:
-              - name: openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+#               - name: openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
-              - name: openstack-helm-images-cinder-2023-2-ubuntu_jammy
+#               - name: openstack-helm-images-cinder-2023-2-ubuntu_jammy
-        - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+#         - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
-        - openstack-helm-images-cinder-2023-2-ubuntu_jammy
+#         - openstack-helm-images-cinder-2023-2-ubuntu_jammy
 
 - job:
     name: openstack-helm-images-build-openstack-2023-2-ubuntu_jammy

zuul.d/openstack-2024.1.yaml

@@ -15,14 +15,11 @@
 - project:
     check:
       jobs:
-        - openstack-helm-images-build-openstack-2024-1-ubuntu_jammy
+        - openstack-helm-images-build-openstack-base-2024-1-ubuntu_jammy
-        # - openstack-helm-images-compute-kit-2024-1-ubuntu_jammy
+        # - openstack-helm-images-build-openstack-2024-1-ubuntu_jammy
-        # - openstack-helm-images-cinder-2024-1-ubuntu_jammy
     gate:
       jobs:
         - openstack-helm-images-upload-openstack-2024-1-ubuntu_jammy
-        # - openstack-helm-images-compute-kit-2024-1-ubuntu_jammy
-        # - openstack-helm-images-cinder-2024-1-ubuntu_jammy
     promote:
       jobs:
         - openstack-helm-images-promote-openstack-2024-1-ubuntu_jammy
@@ -32,37 +29,46 @@
         - openstack-helm-images-promote-openstack-2024-1-ubuntu_jammy:
             dependencies:
               - name: openstack-helm-images-upload-openstack-2024-1-ubuntu_jammy
-              # - name: openstack-helm-images-compute-kit-2024-1-ubuntu_jammy
+
-              # - name: openstack-helm-images-cinder-2024-1-ubuntu_jammy
+- job:
-        # - openstack-helm-images-compute-kit-2024-1-ubuntu_jammy
+    name: openstack-helm-images-build-openstack-base-2024-1-ubuntu_jammy
-        # - openstack-helm-images-cinder-2024-1-ubuntu_jammy
+    description: Build base 2024.1 Jammy images
+    parent: openstack-helm-images-build-openstack-ubuntu_jammy
+    vars:
+      branchname: "stable/2024.1"
+      shortbranchname: "2024.1"
+      openstack_docker_images: []
+    # run:
+    #   - zuul.d/playbooks/build-openstack-base.yml
 
 - job:
     name: openstack-helm-images-build-openstack-2024-1-ubuntu_jammy
     description: Build 2024.1 Jammy images
     parent: openstack-helm-images-build-openstack-ubuntu_jammy
+    dependencies:
+      - openstack-helm-images-build-openstack-base-2024-1-ubuntu_jammy
     vars: &build_vars
       branchname: "stable/2024.1"
       shortbranchname: "2024.1"
       openstack_docker_images:
-        - requirements
+        # - requirements
         - keystone
-        - glance
+        # - glance
-        - cinder
+        # - cinder
-        - designate
+        # - designate
-        - nova
+        # - nova
-        - ironic
+        # - ironic
-        - neutron
+        # - neutron
-        - horizon
+        # - horizon
-        - heat
+        # - heat
-        - barbican
+        # - barbican
-        - placement
+        # - placement
-        - cyborg
+        # - cyborg
-        - magnum
+        # - magnum
-        - masakari
+        # - masakari
-        - masakari_monitors
+        # - masakari_monitors
-        - manila
+        # - manila
-        - tacker
+        # - tacker
     files: &openstack_files
       - ^openstack/loci/.*
       - zuul.d/openstack-2024.1.yaml
@@ -85,58 +91,3 @@
     vars: *build_vars
     files: *openstack_files
 
-- job:
-    name: openstack-helm-images-compute-kit-2024-1-ubuntu_jammy
-    parent: openstack-helm-compute-kit-2024-1-ubuntu_jammy
-    dependencies:
-      - name: openstack-helm-images-build-openvswitch
-        soft: true
-      - name: openstack-helm-images-build-openstack-2024-1-ubuntu_jammy
-        soft: true
-      - name: openstack-helm-images-build-mariadb
-        soft: true
-      - name: openstack-helm-images-build-libvirt
-        soft: true
-      - name: openstack-helm-images-upload-openvswitch
-        soft: true
-      - name: openstack-helm-images-upload-openstack-2024-1-ubuntu_jammy
-        soft: true
-      - name: openstack-helm-images-upload-mariadb
-        soft: true
-      - name: openstack-helm-images-upload-libvirt
-        soft: true
-    files:
-      - openvswitch/.*
-      - zuul.d/openvswitch.yaml
-      - mariadb/.*
-      - zuul.d/mariadb.yaml
-      - libvirt/.*
-      - zuul.d/libvirt.yaml
-      - ^openstack/loci/.*
-      - zuul.d/openstack-2024.1.yaml
-      - zuul.d/openstack-base.yaml
-      - zuul.d/playbooks/build-openstack.yml
-      - zuul.d/playbooks/vars-openstack.yml
-      - zuul.d/playbooks/Dockerfile_base
-
-- job:
-    name: openstack-helm-images-cinder-2024-1-ubuntu_jammy
-    parent: openstack-helm-cinder-2024-1-ubuntu_jammy
-    dependencies:
-      - name: openstack-helm-images-build-openstack-2024-1-ubuntu_jammy
-        soft: true
-      - name: openstack-helm-images-build-mariadb
-        soft: true
-      - name: openstack-helm-images-upload-openstack-2024-1-ubuntu_jammy
-        soft: true
-      - name: openstack-helm-images-upload-mariadb
-        soft: true
-    files:
-      - mariadb/.*
-      - zuul.d/mariadb.yaml
-      - ^openstack/loci/.*
-      - zuul.d/openstack-2024.1.yaml
-      - zuul.d/openstack-base.yaml
-      - zuul.d/playbooks/build-openstack.yml
-      - zuul.d/playbooks/vars-openstack.yml
-      - zuul.d/playbooks/Dockerfile_base

zuul.d/openstack-base.yaml

@@ -28,7 +28,7 @@
       ceph_repo: ""
       ceph_key: ""
       openstack_docker_images:
-        - requirements
+        # - requirements
         - keystone
         - glance
         - cinder

zuul.d/playbooks/build-openstack-base.yml

@@ -0,0 +1,52 @@
+---
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: all[0]
+  gather_facts: true
+  vars_files:
+    - vars-openstack.yml
+
+  pre_tasks:
+    - name: Use buildset registry
+      include_role:
+        name: use-buildset-registry
+
+  tasks:
+    - name: Export the right vars
+      shell: |
+        source /etc/ci/mirror_info.sh
+        echo ${NODEPOOL_PYPI_MIRROR}
+        echo ${NODEPOOL_MIRROR_HOST}
+      args:
+        executable: /bin/bash
+      changed_when: False
+      register: _zuul_mirrors
+
+    - name: Set fact with the exported variables
+      set_fact:
+        pip_index_url: "{{ _zuul_mirrors.stdout_lines[0] | default('https://pypi.python.org/simple/') }}"
+        pip_trusted_host: "{{ _zuul_mirrors.stdout_lines[1] | default('pypi.python.org') }}"
+
+    - name: Build base image
+      include_role:
+        name: build-docker-image
+      vars:
+        zuul_work_dir: "{{ zuul.project.src_dir }}"
+        docker_images: "{{ base_docker_images }}"
+
+    - name: Build requirements image
+      include_role:
+        name: build-docker-image
+      vars:
+        zuul_work_dir: "src/opendev.org/openstack/loci"
+        docker_images: "{{ requirements_docker_images }}"

zuul.d/playbooks/build-openstack.yml

@@ -17,26 +17,28 @@
     - vars-openstack.yml
   vars:
     docker_images: "{{ openstack_docker_images | map('extract', openstack_docker_images_dict) | list }}"
+    wheels_location: "{{ buildset_registry_alias }}/requirements:{{ shortbranchname }}-{{ distroname }}"
+
   pre_tasks:
     - name: Use buildset registry
       include_role:
         name: use-buildset-registry
+
   tasks:
     - name: Export the right vars
       shell: |
         source /etc/ci/mirror_info.sh
-        echo ${NODEPOOL_OPENSUSE_MIRROR}/
         echo ${NODEPOOL_PYPI_MIRROR}
         echo ${NODEPOOL_MIRROR_HOST}
       args:
         executable: /bin/bash
       changed_when: False
       register: _zuul_mirrors
+
     - name: Set fact with the exported variables
       set_fact:
-        opensuse_package_mirror: "{{ _zuul_mirrors.stdout_lines[0] }}"
+        pip_index_url: "{{ _zuul_mirrors.stdout_lines[0] | default('https://pypi.python.org/simple/') }}"
-        pip_index_url: "{{ _zuul_mirrors.stdout_lines[1] | default('https://pypi.python.org/simple/') }}"
+        pip_trusted_host: "{{ _zuul_mirrors.stdout_lines[1] | default('pypi.python.org') }}"
-        pip_trusted_host: "{{ _zuul_mirrors.stdout_lines[2] | default('pypi.python.org') }}"
 
     - name: Build base image
       include_role:
@@ -45,7 +47,7 @@
         zuul_work_dir: "{{ zuul.project.src_dir }}"
         docker_images: "{{ base_docker_images }}"
 
-    - name: Build base requirements image
+    - name: Build requirements image
       include_role:
         name: build-docker-image
       vars:

zuul.d/playbooks/vars-openstack.yml

@@ -14,8 +14,13 @@
 currentdate: "{{ now(utc=True,fmt='%Y%m%d') }}"
 python3: "yes"
 #172.17.0.1 is default docker0 ip.
-wheels_location: "http://172.17.0.1:8080/wheels.tar.gz"
+# wheels_location: "http://172.17.0.1:8080/wheels.tar.gz"
+# wheels_location: "{{ buildset_registry_alias }}/requirements:{{ shortbranchname }}-{{ distroname }}"
 base_repository: "base"
+requirements_repository: "requirements"
+tags: &imagetag
+  - "{{ shortbranchname }}-{{ distroname }}"
+  - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}"
 
 base_docker_images:
   - context: "zuul.d/playbooks/"
@@ -25,6 +30,7 @@ base_docker_images:
       - "FROM='{{ from }}'"
       - "CEPH_REPO='{{ ceph_repo }}'"
       - "CEPH_KEY='{{ ceph_key }}'"
+    tags: *imagetag
 
 requirements_docker_images:
   - context: "."
@@ -34,24 +40,25 @@ requirements_docker_images:
       - "PROJECT='requirements'"
       - "PROJECT_REF={{ branchname }}"
       - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
 
 openstack_docker_images_dict:
-  requirements:
+  # requirements:
-    context: "."
+  #   context: "."
-    repository: openstackhelm/requirements
+  #   repository: openstackhelm/requirements
-    build_args:
+  #   build_args:
-      - "FROM='{{ base_repository }}'"
+  #     - "FROM='{{ base_repository }}'"
-      - "PROJECT='requirements'"
+  #     - "PROJECT='requirements'"
-      - "PROJECT_REF={{ branchname }}"
+  #     - "PROJECT_REF={{ branchname }}"
-      - "KEEP_ALL_WHEELS=yes"
+  #     - "KEEP_ALL_WHEELS=yes"
-    tags: &imagetag
+  #   tags: &imagetag
-      - "{{ shortbranchname }}-{{ distroname }}"
+  #     - "{{ shortbranchname }}-{{ distroname }}"
-      - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}" # Frozen tag
+  #     - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}" # Frozen tag
   keystone:
     context: "."
     repository: openstackhelm/keystone
     build_args:
-      - "FROM='{{ base_repository }}'"
+      - "FROM='{{ base_repository }}:{{ shortbranchname }}-{{ distroname }}'"
       - "PROJECT='keystone'"
       - "PROJECT_REF={{ branchname }}"
       - "PROFILES='fluent apache ldap {{ shortbranchname }}'"