Merge "Treat wheel builders more like build nodes"

playbooks/wheel/build.yaml

@@ -24,7 +24,7 @@
         wheel_dir: /opt/wheel/workspace
 
   roles:
-    - role: build-wheels
-      wheel_python: python2
-    - role: build-wheels
-      wheel_python: python3
+    - build-wheels
+    - create-afs-token
+    - copy-wheels
+    - destroy-afs-token

playbooks/wheel/release.yaml

@@ -1,10 +1,10 @@
 - hosts: localhost
   roles:
-    - create-afs-token
+    - role: create-afs-token
     - role: release-afs
       afs_volume: trustyx64
     - role: release-afs
       afs_volume: cent7x64
     - role: release-afs
       afs_volume: xenialx64
-    - destroy-afs-token
+    - role: destroy-afs-token

roles/build-wheels/README.rst

@@ -0,0 +1 @@
+Run the build scripts on the wheel mirror builders

roles/build-wheels/tasks/main.yaml

@@ -10,17 +10,3 @@
 
 - name: Build the wheel mirror
   shell: bash -x scripts/wheel-build.sh {{ wheel_dir }} {{ wheel_python }}
-
-- name: Get an afs token and copy the wheels to AFS
-  command: <
-    k5start -t -f /etc/wheel.keytab
-        service/wheel
-        -- timeout -k 2m 30m
-        scripts/wheel-copy.sh {{ wheel_dir }} {{ afs_dir }}
-
-- name: Obtaining token and rebuilding mirror index
-  command: <
-    k5start -t -f /etc/wheel.keytab
-        service/wheel
-        -- timeout -k 2m 30m
-        scripts/wheel-index.sh {{ afs_dir }}

roles/copy-wheels/README.rst

@@ -0,0 +1 @@
+Run the copy scripts on the wheel mirror builders

roles/copy-wheels/tasks/main.yaml

@@ -0,0 +1,5 @@
+- name: Copy the wheels to AFS
+  command: bash -x scripts/wheel-copy.sh {{ wheel_dir }} {{ afs_dir }}
+
+- name: Rebuild mirror index
+  command: bash -x scripts/wheel-index.sh {{ afs_dir }}

zuul.yaml

@@ -323,13 +323,12 @@
 
 - job:
     name: build-wheel-mirror
+    pre-run: playbooks/wheel/pre
     run: playbooks/wheel/build
     post-run: playbooks/wheel/release
     nodes: []
-    final: True
-    # 4.5 hours - double the v2 runtime, since we're running python2 and
-    # python3 in series
-    timeout: 16200
+    # 2.5 hours
+    timeout: 9000
     allowed-projects:
       - openstack/requirements
     semaphore: wheel-mirror
@@ -337,9 +336,29 @@
       - wheel_mirror_centos_7_key
       - wheel_mirror_ubuntu_trusty_key
       - wheel_mirror_ubuntu_xenial_key
+      - name: afs
+        secret: wheel_keytab
+
+- job:
+    name: release-wheel-mirror
+    run: playbooks/wheel/release
+    nodes: []
+    secrets:
       - name: afs
         secret: afsadmin_keytab
 
+- job:
+    name: build-wheel-mirror-python2
+    parent: build-wheel-mirror
+    vars:
+      wheel_python: python2
+
+- job:
+    name: build-wheel-mirror-python3
+    parent: build-wheel-mirror
+    vars:
+      wheel_python: python3
+
 - project:
     name: openstack-infra/project-config
     check:
@@ -620,6 +639,22 @@
           gcFWp3Xx4jIQHQdx7I5ZHTLSojvChAf2b8/Ps4IvYTiBKxt7wLtyfV9/kj3BAQ=
       service_name: service/afsadmin@OPENSTACK.ORG
 
+- secret:
+    name: wheel_keytab
+    data:
+      keytab: !encrypted/pkcs1-oaep
+        - B12EKQPl+E46VynbYHYNOyzbpIYkxLjia3uDsYjah39m2e/VtrnfprDAk/xOzxqZiC4/u
+          uG9bnvzHaUZE6BpoUNja1Tr5Mlzk9UY9OGxnb7KGEzIxxyS3s2H6r50FhlSSwK9VZc8fB
+          Qn2Gp1Vo+Z5VgJoi/W0S7HSPcc9RW+nLdO6S0yX4TLc7EQUlwgQCWowZg5F9V7iouYccz
+          hJ/ZpM3aAK/77vF70gkX2mXKAtu6/AyCUBnEaEG13KrSiONw95xW7+re1WIuHQxDNGbOH
+          FInMT0NxDm7hJp8NycY5F5Otwtx1rVOoAK3ReWYKxiTz6JsaM70jpnwc6Ps3eTXwk2B12
+          mxBsOtjNXJVVF+Np8lKQ/lk7DweHC0uqM+qbX40er1mcYfT9X+Z8W6lqP3FFgWj3cYqBf
+          CsTmqIk+W7PaBG6BCe2635L2Ai7gtlAqavo/J+Cry89zsuaj+AkcVzA6impXhNR9bu0Vm
+          ptwYcxQLEjC+AMUAYwLGhtNGG1wpz1s3kjHyq4ggYUep7C58M3Sk8K58xjVnzAKOldAL1
+          EN/khW6Dz+h7qOktic0Nia2AXRG3U8EQY8G/ZL6GGhvlldABqTi7uyLwtUHq+edirlO+5
+          rk1ilszKDQOr+gBfweJGrJVcTCTOv+uKf4V8jeSTn6d7Wlu5k3tchI41A3pWeA=
+      service_name: service/wheel@OPENSTACK.ORG
+
 - secret:
     name: wheel_mirror_ubuntu_xenial_key
     data:
@@ -797,7 +832,10 @@
     periodic:
       jobs:
         # This is a periodic job to ensure that our wheels are reasonably
-        # up-to-date. Adding it to the requirements project seems to be the
-        # best place to put it, though it's not strictly a requirement for
-        # requirements.
-        - build-wheel-mirror
+        # up-to-date. It uses the upper-constraints files from requirements.
+        - build-wheel-mirror-python2
+        - build-wheel-mirror-python3
+        - release-wheel-mirror:
+            dependencies:
+              - build-wheel-mirror-python2
+              - build-wheel-mirror-python3