[WIP] Refactor Openstack image build

Change-Id: I7c63ee98125bf0988b3926e40dd916087cbb9476

zuul.d/base.yaml

@@ -56,9 +56,9 @@
         - openstack-helm-images-compute-kit-2023-1-ubuntu_jammy
         - openstack-helm-images-cinder-2023-1-ubuntu_jammy
         - openstack-helm-images-horizon-2023-1-ubuntu_jammy
-    templates:
+    # templates:
-      - publish-openstack-docs-pti
+    #   - publish-openstack-docs-pti
-      - release-notes-jobs-python3
+    #   - release-notes-jobs-python3
 
 # This takes the same structure as other infra jobs.
 # This dict keyname matter. See also job definition for more explanations.
@@ -92,7 +92,7 @@
     name: openstack-helm-images-buildset-registry
     parent: opendev-buildset-registry
     timeout: 10800
-    voting: false
+    # voting: false
 
 # In docker_images:
 # `context` is the folder in osh-images

zuul.d/openstack-2023.2.yaml

@@ -11,29 +11,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- project:
+# - project:
-    check:
+#     check:
-      jobs:
+#       jobs:
-        - openstack-helm-images-build-openstack-2023-2-ubuntu_jammy
+#         - openstack-helm-images-build-openstack-2023-2-ubuntu_jammy
-        - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+#         - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
-        - openstack-helm-images-cinder-2023-2-ubuntu_jammy
+#         - openstack-helm-images-cinder-2023-2-ubuntu_jammy
-    gate:
+#     gate:
-      jobs:
+#       jobs:
-        - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
+#         - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
-        - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+#         - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
-        - openstack-helm-images-cinder-2023-2-ubuntu_jammy
+#         - openstack-helm-images-cinder-2023-2-ubuntu_jammy
-    promote:
+#     promote:
-      jobs:
+#       jobs:
-        - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy
+#         - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy
-    periodic-weekly:
+#     periodic-weekly:
-      jobs:
+#       jobs:
-        - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
+#         - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
-        - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy:
+#         - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy:
-            dependencies:
+#             dependencies:
-              - name: openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+#               - name: openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
-              - name: openstack-helm-images-cinder-2023-2-ubuntu_jammy
+#               - name: openstack-helm-images-cinder-2023-2-ubuntu_jammy
-        - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+#         - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
-        - openstack-helm-images-cinder-2023-2-ubuntu_jammy
+#         - openstack-helm-images-cinder-2023-2-ubuntu_jammy
 
 - job:
     name: openstack-helm-images-build-openstack-2023-2-ubuntu_jammy

zuul.d/openstack-2024.1.yaml

@@ -45,24 +45,24 @@
       branchname: "stable/2024.1"
       shortbranchname: "2024.1"
       openstack_docker_images:
-        - requirements
+        # - requirements
         - keystone
-        - glance
+        # - glance
-        - cinder
+        # - cinder
-        - designate
+        # - designate
-        - nova
+        # - nova
-        - ironic
+        # - ironic
-        - neutron
+        # - neutron
-        - horizon
+        # - horizon
-        - heat
+        # - heat
-        - barbican
+        # - barbican
-        - placement
+        # - placement
-        - cyborg
+        # - cyborg
-        - magnum
+        # - magnum
-        - masakari
+        # - masakari
-        - masakari_monitors
+        # - masakari_monitors
-        - manila
+        # - manila
-        - tacker
+        # - tacker
     files: &openstack_files
       - ^openstack/loci/.*
       - zuul.d/openstack-2024.1.yaml

zuul.d/playbooks/build-openstack.yml

@@ -22,21 +22,21 @@
       include_role:
         name: use-buildset-registry
   tasks:
-    - name: Export the right vars
+    # - name: Export the right vars
-      shell: |
+    #   shell: |
-        source /etc/ci/mirror_info.sh
+    #     source /etc/ci/mirror_info.sh
-        echo ${NODEPOOL_OPENSUSE_MIRROR}/
+    #     echo ${NODEPOOL_OPENSUSE_MIRROR}/
-        echo ${NODEPOOL_PYPI_MIRROR}
+    #     echo ${NODEPOOL_PYPI_MIRROR}
-        echo ${NODEPOOL_MIRROR_HOST}
+    #     echo ${NODEPOOL_MIRROR_HOST}
-      args:
+    #   args:
-        executable: /bin/bash
+    #     executable: /bin/bash
-      changed_when: False
+    #   changed_when: False
-      register: _zuul_mirrors
+    #   register: _zuul_mirrors
-    - name: Set fact with the exported variables
+    # - name: Set fact with the exported variables
-      set_fact:
+    #   set_fact:
-        opensuse_package_mirror: "{{ _zuul_mirrors.stdout_lines[0] }}"
+    #     opensuse_package_mirror: "{{ _zuul_mirrors.stdout_lines[0] }}"
-        pip_index_url: "{{ _zuul_mirrors.stdout_lines[1] | default('https://pypi.python.org/simple/') }}"
+    #     pip_index_url: "{{ _zuul_mirrors.stdout_lines[1] | default('https://pypi.python.org/simple/') }}"
-        pip_trusted_host: "{{ _zuul_mirrors.stdout_lines[2] | default('pypi.python.org') }}"
+    #     pip_trusted_host: "{{ _zuul_mirrors.stdout_lines[2] | default('pypi.python.org') }}"
 
     - name: Build base image
       include_role:
@@ -52,34 +52,34 @@
         zuul_work_dir: "src/opendev.org/openstack/loci"
         docker_images: "{{ requirements_docker_images }}"
 
-    - name: Build webserver serving wheels
+    # - name: Build webserver serving wheels
-      include_role:
+    #   include_role:
-        name: build-docker-image
+    #     name: build-docker-image
-      vars:
+    #   vars:
-        zuul_work_dir: "{{ zuul.project.src_dir }}"
+    #     zuul_work_dir: "{{ zuul.project.src_dir }}"
-        docker_images:
+    #     docker_images:
-          - context: zuul.d/playbooks/
+    #       - context: zuul.d/playbooks/
-            repository: openstackhelm/loci_wheels
+    #         repository: openstackhelm/loci_wheels
 
-    - name: Allow access to wheels server
+    # - name: Allow access to wheels server
-      iptables:
+    #   iptables:
-        action: insert
+    #     action: insert
-        chain: INPUT
+    #     chain: INPUT
-        in_interface: docker0
+    #     in_interface: docker0
-        jump: ACCEPT
+    #     jump: ACCEPT
-      become: yes
+    #   become: yes
 
-    - name: Run http server containing wheels
+    # - name: Run http server containing wheels
-      command: docker run -d -p 0.0.0.0:8080:80 openstackhelm/loci_wheels:latest
+    #   command: docker run -d -p 0.0.0.0:8080:80 openstackhelm/loci_wheels:latest
 
-    - name: Build project images
+    # - name: Build project images
-      include_role:
+    #   include_role:
-        name: build-docker-image
+    #     name: build-docker-image
-      vars:
+    #   vars:
-        zuul_work_dir: "src/opendev.org/openstack/loci"
+    #     zuul_work_dir: "src/opendev.org/openstack/loci"
 
-    - name: Return images to zuul
+    # - name: Return images to zuul
-      include_tasks: _return-image.yml
+    #   include_tasks: _return-image.yml
-      loop: "{{ docker_images }}"
+    #   loop: "{{ docker_images }}"
-      loop_control:
+    #   loop_control:
-        loop_var: zj_image
+    #     loop_var: zj_image

zuul.d/playbooks/vars-openstack.yml

@@ -16,6 +16,7 @@ python3: "yes"
 #172.17.0.1 is default docker0 ip.
 wheels_location: "http://172.17.0.1:8080/wheels.tar.gz"
 base_repository: "base"
+requirements_repository: "requirements"
 
 base_docker_images:
   - context: "zuul.d/playbooks/"
@@ -25,38 +26,42 @@ base_docker_images:
       - "FROM='{{ from }}'"
       - "CEPH_REPO='{{ ceph_repo }}'"
       - "CEPH_KEY='{{ ceph_key }}'"
+    tags: &imagetag
+      - "{{ shortbranchname }}-{{ distroname }}"
+      - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}"
 
 requirements_docker_images:
   - context: "."
-    repository: openstackhelm/requirements
+    repository: "{{ requirements_repository }}"
     build_args:
-      - "FROM='{{ base_repository }}'"
+      - "FROM='{{ base_repository }}:{{ shortbranchname }}-{{ distroname }}'"
       - "PROJECT='requirements'"
       - "PROJECT_REF={{ branchname }}"
       - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
 
 openstack_docker_images_dict:
-  requirements:
+  # requirements:
-    context: "."
+  #   context: "."
-    repository: openstackhelm/requirements
+  #   repository: openstackhelm/requirements
-    build_args:
+  #   build_args:
-      - "FROM='{{ base_repository }}'"
+  #     - "FROM='{{ base_repository }}'"
-      - "PROJECT='requirements'"
+  #     - "PROJECT='requirements'"
-      - "PROJECT_REF={{ branchname }}"
+  #     - "PROJECT_REF={{ branchname }}"
-      - "KEEP_ALL_WHEELS=yes"
+  #     - "KEEP_ALL_WHEELS=yes"
-    tags: &imagetag
+  #   tags: &imagetag
-      - "{{ shortbranchname }}-{{ distroname }}"
+  #     - "{{ shortbranchname }}-{{ distroname }}"
-      - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}" # Frozen tag
+  #     - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}" # Frozen tag
   keystone:
     context: "."
     repository: openstackhelm/keystone
     build_args:
-      - "FROM='{{ base_repository }}'"
+      - "FROM='{{ base_repository }}:{{ shortbranchname }}-{{ distroname }}'"
       - "PROJECT='keystone'"
       - "PROJECT_REF={{ branchname }}"
       - "PROFILES='fluent apache ldap {{ shortbranchname }}'"
       - "PIP_PACKAGES='python-openstackclient'"
-      - "WHEELS='{{ wheels_location }}'"
+      - "WHEELS='{{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ requirements_repository }}:{{ shortbranchname }}-{{ distroname }}'"
       - "KEEP_ALL_WHEELS=yes"
     tags: *imagetag
   heat: