diff --git a/playbooks/base/cleanup.yaml b/playbooks/base/cleanup.yaml
new file mode 100644
index 0000000..d793838
--- /dev/null
+++ b/playbooks/base/cleanup.yaml
@@ -0,0 +1,3 @@
+- hosts: all
+  roles:
+    - remove-build-sshkey
diff --git a/playbooks/base/post.yaml b/playbooks/base/post.yaml
index 0318870..c810eab 100644
--- a/playbooks/base/post.yaml
+++ b/playbooks/base/post.yaml
@@ -1,10 +1,3 @@
 - hosts: all
   roles:
     - fetch-output
-
-- hosts: all
-  # NOTE(pabelanger): We ignore_errors for the following tasks as not to fail
-  # successful jobs.
-  ignore_errors: yes
-  roles:
-    - remove-build-sshkey
diff --git a/zuul.yaml b/zuul.yaml
index 86c85b9..e2b33a5 100644
--- a/zuul.yaml
+++ b/zuul.yaml
@@ -11,6 +11,7 @@
       It also sets a default timeout value (which may be overidden).
     pre-run: playbooks/base/pre.yaml
     post-run: playbooks/base/post.yaml
+    cleanup-run: playbooks/base/cleanup.yaml
     roles:
       - zuul: opendev.org/zuul/zuul-jobs
     timeout: 1800