wheel builds : move to individual releases

The publish jobs build and copy the wheels to the AFS r/w partitions,
but they are not released to the mirrors until a final "vos release"
of their volume.

Previously this happened in a final job that would not trigger if any
of the wheel builds failed.  This has meant thing like out-of-sync
centos mirrors or arm64 node failures have stopped all wheel
publishing previously.  There's really no need to block other
platforms publishing if one of these jobs fails.

This converts the jobs to have a release job syncrhonized by a
semaphore.  We don't want many "vos release" processes running all at
once, as it has been a source of failure with openafs before.  Since
releasing requires the admin key, we keep it in a separate job from
the building.

Change-Id: I10c307c2d46c0e5b86732943208c3167da28a694

playbooks/wheel/release.yaml

@@ -2,50 +2,7 @@
   roles:
     - role: create-afs-token
 
-    # CentOS
     - role: release-afs-volume
-      afs_volume: mirror.wheel.cent7x64
+      afs_volume: '{{ wheel_volume }}'
 
-    # CentOS 8 Stream
-    - role: release-afs-volume
-      afs_volume: mirror.wheel.cent8sx64
-    - role: release-afs-volume
-      afs_volume: mirror.wheel.cent8sa64
-
-    # CentOS 9 Stream
-    - role: release-afs-volume
-      afs_volume: mirror.wheel.cent9sx64
-    - role: release-afs-volume
-      afs_volume: mirror.wheel.cent9sa64
-
-    # Debian
-    - role: release-afs-volume
-      afs_volume: mirror.wheel.busterx64
-    - role: release-afs-volume
-      afs_volume: mirror.wheel.bustera64
-    - role: release-afs-volume
-      afs_volume: mirror.wheel.deb11x64
-    - role: release-afs-volume
-      afs_volume: mirror.wheel.deb11a64
-
-    # Ubuntu
-    - role: release-afs-volume
-      afs_volume: mirror.wheel.xenialx64
-
-    - role: release-afs-volume
-      afs_volume: mirror.wheel.bionicx64
-    - role: release-afs-volume
-      afs_volume: mirror.wheel.bionica64
-
-    - role: release-afs-volume
-      afs_volume: mirror.wheel.focalx64
-    - role: release-afs-volume
-      afs_volume: mirror.wheel.focala64
-
-    - role: release-afs-volume
-      afs_volume: mirror.wheel.jammyx64
-    - role: release-afs-volume
-      afs_volume: mirror.wheel.jammya64
-
-    # Fin
     - role: destroy-afs-token

zuul.d/jobs.yaml

@@ -853,6 +853,27 @@
     run: playbooks/proposal/propose-project-config-update.yaml
     protected: true
 
+# The wheel builds can finish at similar times, so to avoid many
+# overlapping "vos release" processes we restrict it with a semaphore.
+- semaphore:
+    name: wheel-release
+    max: 1
+
+- job:
+    name: release-wheel-cache
+    abstract: true
+    protected: true
+    description: |
+        Release a wheel cache volume
+    run:
+      - name: playbooks/release/wheel.yaml
+        semaphores: wheel-release
+    nodeset:
+      nodes: []
+    secrets:
+      - name: afs
+        secret: afsadmin_keytab
+
 - job:
     name: publish-wheel-cache-centos-7
     description: |
@@ -867,6 +888,13 @@
       - name: afs
         secret: wheel_keytab
 
+- job:
+    name: release-wheel-cache-centos-7
+    parent: release-wheel-cache
+    final: true
+    vars:
+      wheel_volume: 'mirror.wheel.cent7x64'
+
 - job:
     name: publish-wheel-cache-centos-8-stream
     description: |
@@ -881,6 +909,13 @@
       - name: afs
         secret: wheel_keytab
 
+- job:
+    name: release-wheel-cache-centos-8-stream
+    parent: release-wheel-cache
+    final: true
+    vars:
+      wheel_volume: 'mirror.wheel.cent8x64'
+
 - job:
     name: publish-wheel-cache-centos-8-stream-arm64
     description: |
@@ -895,6 +930,13 @@
       - name: afs
         secret: wheel_keytab
 
+- job:
+    name: release-wheel-cache-centos-8-stream-arm64
+    parent: release-wheel-cache
+    final: true
+    vars:
+      wheel_volume: 'mirror.wheel.cent8a64'
+
 - job:
     name: publish-wheel-cache-centos-9-stream
     description: |
@@ -909,6 +951,13 @@
       - name: afs
         secret: wheel_keytab
 
+- job:
+    name: release-wheel-cache-centos-9-stream
+    parent: release-wheel-cache
+    final: true
+    vars:
+      wheel_volume: 'mirror.wheel.cent9sx64'
+
 - job:
     name: publish-wheel-cache-centos-9-stream-arm64
     description: |
@@ -923,6 +972,13 @@
       - name: afs
         secret: wheel_keytab
 
+- job:
+    name: release-wheel-cache-centos-9-stream-arm64
+    parent: release-wheel-cache
+    final: true
+    vars:
+      wheel_volume: 'mirror.wheel.cent9sx64'
+
 - job:
     name: publish-wheel-cache-debian-bullseye
     description: |
@@ -937,6 +993,13 @@
       - name: afs
         secret: wheel_keytab
 
+- job:
+    name: release-wheel-cache-debian-bullseye
+    parent: release-wheel-cache
+    final: true
+    vars:
+      wheel_volume: 'mirror.wheel.deb11x64'
+
 - job:
     name: publish-wheel-cache-debian-bullseye-arm64
     description: |
@@ -951,6 +1014,13 @@
       - name: afs
         secret: wheel_keytab
 
+- job:
+    name: release-wheel-cache-debian-bullseye-arm64
+    parent: release-wheel-cache
+    final: true
+    vars:
+      wheel_volume: 'mirror.wheel.deb11a64'
+
 - job:
     name: publish-wheel-cache-debian-buster
     description: |
@@ -965,6 +1035,13 @@
       - name: afs
         secret: wheel_keytab
 
+- job:
+    name: release-wheel-cache-debian-buster
+    parent: release-wheel-cache
+    final: true
+    vars:
+      wheel_volume: 'mirror.wheel.busterx64'
+
 - job:
     name: publish-wheel-cache-debian-buster-arm64
     description: |
@@ -979,6 +1056,13 @@
       - name: afs
         secret: wheel_keytab
 
+- job:
+    name: release-wheel-cache-debian-buster-arm64
+    parent: release-wheel-cache
+    final: true
+    vars:
+      wheel_volume: 'mirror.wheel.bustera64'
+
 - job:
     name: publish-wheel-cache-ubuntu-jammy
     parent: build-wheel-cache-ubuntu-jammy
@@ -993,6 +1077,13 @@
       - name: afs
         secret: wheel_keytab
 
+- job:
+    name: release-wheel-cache-ubuntu-jammy
+    parent: release-wheel-cache
+    final: true
+    vars:
+      wheel_volume: 'mirror.wheel.jammyx64'
+
 - job:
     name: publish-wheel-cache-ubuntu-jammy-arm64
     parent: build-wheel-cache-ubuntu-jammy-arm64
@@ -1007,6 +1098,13 @@
       - name: afs
         secret: wheel_keytab
 
+- job:
+    name: release-wheel-cache-ubuntu-jammy-arm64
+    parent: release-wheel-cache
+    final: true
+    vars:
+      wheel_volume: 'mirror.wheel.jammya64'
+
 - job:
     name: publish-wheel-cache-ubuntu-focal
     parent: build-wheel-cache-ubuntu-focal
@@ -1021,6 +1119,13 @@
       - name: afs
         secret: wheel_keytab
 
+- job:
+    name: release-wheel-cache-ubuntu-focal
+    parent: release-wheel-cache
+    final: true
+    vars:
+      wheel_volume: 'mirror.wheel.focalx64'
+
 - job:
     name: publish-wheel-cache-ubuntu-focal-arm64
     parent: build-wheel-cache-ubuntu-focal-arm64
@@ -1035,6 +1140,13 @@
       - name: afs
         secret: wheel_keytab
 
+- job:
+    name: release-wheel-cache-ubuntu-focal-arm64
+    parent: release-wheel-cache
+    final: true
+    vars:
+      wheel_volume: 'mirror.wheel.focala64'
+
 - job:
     name: publish-wheel-cache-ubuntu-bionic
     parent: build-wheel-cache-ubuntu-bionic
@@ -1049,6 +1161,13 @@
       - name: afs
         secret: wheel_keytab
 
+- job:
+    name: release-wheel-cache-ubuntu-bionic
+    parent: release-wheel-cache
+    final: true
+    vars:
+      wheel_volume: 'mirror.wheel.bionicx64'
+
 - job:
     name: publish-wheel-cache-ubuntu-bionic-arm64
     parent: build-wheel-cache-ubuntu-bionic-arm64
@@ -1063,6 +1182,13 @@
       - name: afs
         secret: wheel_keytab
 
+- job:
+    name: release-wheel-cache-ubuntu-bionic-arm64
+    parent: release-wheel-cache
+    final: true
+    vars:
+      wheel_volume: 'mirror.wheel.bionica64'
+
 - job:
     name: publish-wheel-cache-ubuntu-xenial
     parent: build-wheel-cache-ubuntu-xenial
@@ -1078,20 +1204,11 @@
         secret: wheel_keytab
 
 - job:
-    name: release-wheel-cache
+    name: release-wheel-cache-ubuntu-xenial
-    description: |
+    parent: release-wheel-cache
-      Release published wheels to OpenStack CI mirrors.
+    final: true
-    run: playbooks/wheel/release.yaml
+    vars:
-    nodeset:
+      wheel_volume: 'mirror.wheel.xenialx64'
-      nodes: []
-    # Build and publish jobs for the wheel mirror only run against
-    # master. Restrict the release job to master as well to avoid zuul
-    # config errors due to stable branches not being able to run their
-    # job depenendencies.
-    branches: master
-    secrets:
-      - name: afs
-        secret: afsadmin_keytab
 
 - job:
     name: check-release-approval

zuul.d/projects.yaml

@@ -4584,39 +4584,69 @@
     periodic:
       jobs:
         - publish-wheel-cache-centos-7
-        - publish-wheel-cache-centos-8-stream
+        - release-wheel-cache-centos-7:
-        - publish-wheel-cache-centos-9-stream
-        - publish-wheel-cache-debian-bullseye
-        - publish-wheel-cache-debian-buster
-        - publish-wheel-cache-ubuntu-jammy
-        - publish-wheel-cache-ubuntu-focal
-        - publish-wheel-cache-ubuntu-bionic
-        - publish-wheel-cache-ubuntu-xenial
-        - publish-wheel-cache-centos-8-stream-arm64
-        - publish-wheel-cache-centos-9-stream-arm64
-        - publish-wheel-cache-debian-bullseye-arm64
-        - publish-wheel-cache-debian-buster-arm64
-        - publish-wheel-cache-ubuntu-jammy-arm64
-        - publish-wheel-cache-ubuntu-focal-arm64
-        - publish-wheel-cache-ubuntu-bionic-arm64
-        - release-wheel-cache:
             dependencies:
-              - publish-wheel-cache-centos-7
+              - name: publish-wheel-cache-centos-7
-              - publish-wheel-cache-centos-8-stream
+        - publish-wheel-cache-centos-8-stream
-              - publish-wheel-cache-centos-9-stream
+        - release-wheel-cache-centos-8-stream:
-              - publish-wheel-cache-debian-bullseye
+            dependencies:
-              - publish-wheel-cache-debian-buster
+              - name: publish-wheel-cache-centos-8-stream
-              - publish-wheel-cache-ubuntu-jammy
+        - publish-wheel-cache-centos-9-stream
-              - publish-wheel-cache-ubuntu-focal
+        - release-wheel-cache-centos-9-stream:
-              - publish-wheel-cache-ubuntu-bionic
+            dependencies:
-              - publish-wheel-cache-ubuntu-xenial
+              - name: publish-wheel-cache-centos-9-stream
-              - publish-wheel-cache-centos-8-stream-arm64
+        - publish-wheel-cache-debian-bullseye
-              - publish-wheel-cache-centos-9-stream-arm64
+        - release-wheel-cache-debian-bullseye:
-              - publish-wheel-cache-debian-bullseye-arm64
+            dependencies:
-              - publish-wheel-cache-debian-buster-arm64
+              - name: publish-wheel-cache-debian-bullseye
-              - publish-wheel-cache-ubuntu-jammy-arm64
+        - publish-wheel-cache-debian-buster
-              - publish-wheel-cache-ubuntu-focal-arm64
+        - release-wheel-cache-debian-buster:
-              - publish-wheel-cache-ubuntu-bionic-arm64
+            dependencies:
+              - name: publish-wheel-cache-debian-buster
+        - publish-wheel-cache-ubuntu-jammy
+        - release-wheel-cache-ubuntu-jammy:
+            dependencies:
+              - name: publish-wheel-cache-ubuntu-jammy
+        - publish-wheel-cache-ubuntu-focal
+        - release-wheel-cache-ubuntu-focal:
+            dependencies:
+              - name: publish-wheel-cache-ubuntu-focal
+        - publish-wheel-cache-ubuntu-bionic
+        - release-wheel-cache-ubuntu-bionic:
+            dependencies:
+              - name: publish-wheel-cache-ubuntu-bionic
+        - publish-wheel-cache-ubuntu-xenial
+        - release-wheel-cache-ubuntu-xenial:
+            dependencies:
+              - name: publish-wheel-cache-ubuntu-xenial
+        - publish-wheel-cache-centos-8-stream-arm64
+        - release-wheel-cache-centos-8-stream-arm64:
+            dependencies:
+              - name: publish-wheel-cache-centos-8-stream-arm64
+        - publish-wheel-cache-centos-9-stream-arm64
+        - release-wheel-cache-centos-9-stream-arm64:
+            dependencies:
+              - name: publish-wheel-cache-centos-9-stream-arm64
+        - publish-wheel-cache-debian-bullseye-arm64
+        - release-wheel-cache-debian-bullseye-arm64:
+            dependencies:
+              - name: publish-wheel-cache-debian-bullseye-arm64
+        - publish-wheel-cache-debian-buster-arm64
+        - release-wheel-cache-debian-buster-arm64:
+            dependencies:
+              - name: publish-wheel-cache-debian-buster-arm64
+        - publish-wheel-cache-ubuntu-jammy-arm64
+        - release-wheel-cache-ubuntu-jammy-arm64:
+            dependencies:
+              - name: publish-wheel-cache-ubuntu-jammy-arm64
+        - publish-wheel-cache-ubuntu-focal-arm64
+        - release-wheel-cache-ubuntu-focal-arm64:
+            dependencies:
+              - name: publish-wheel-cache-ubuntu-focal-arm64
+        - publish-wheel-cache-ubuntu-bionic-arm64
+        - release-wheel-cache-ubuntu-bionic-arm64:
+            dependencies:
+              - name: publish-wheel-cache-ubuntu-bionic-arm64
     periodic-weekly:
       jobs:
         - propose-updates: