Merge "Use async tasks for long running common tasks" into stable/train

common/deploy-steps-tasks.yaml

@@ -17,6 +17,8 @@
         - host_config
 
     - name: Run puppet host configuration for step {{ step }}
+      async: 3600
+      poll: 0
       when: enable_puppet|bool
       shell: >-
         set -o pipefail;
@@ -29,21 +31,32 @@
         {{ ansible_check_mode | bool | ternary('--hiera_config /etc/puppet/check-mode/hiera.yaml', '') }}
         /var/lib/tripleo-config/{{ ansible_check_mode | bool | ternary('check-mode/', '') }}puppet_step_config.pp
         2>&1 | logger -s -t puppet-user
-      changed_when: outputs.rc == 2
-      register: outputs
-      failed_when: false
+      register: puppet_host_async_result
       no_log: true
       tags:
         - host_config
       check_mode: no
 
+    - name: Wait for puppet host configuration to finish
+      async_status:
+        jid: "{{ puppet_host_async_result.ansible_job_id }}"
+      register: puppet_host_outputs
+      until: puppet_host_outputs.finished
+      retries: 1200
+      delay: 3
+      failed_when:
+        - (not puppet_host_outputs.finished) or (puppet_host_outputs.rc is defined and puppet_host_outputs.rc not in [0, 2])
+      tags:
+        - host_config
+
     - name: "Debug output for task: Run puppet host configuration for step {{ step }}"
       debug:
-        var: outputs.stdout_lines | default([]) | union(outputs.stderr_lines | default([]))
+        var: puppet_host_outputs.stdout_lines | default([]) | union(puppet_host_outputs.stderr_lines | default([]))
       when:
         - enable_puppet | bool
-        - outputs.rc is defined
-      failed_when: outputs.rc not in [0, 2]
+        - puppet_host_outputs.rc is defined
+      changed_when: puppet_host_outputs.rc == 2
+      failed_when: puppet_host_outputs.rc not in [0, 2]
       tags:
         - host_config
       ignore_errors: "{{ ansible_check_mode|bool }}"
@@ -127,6 +140,8 @@
     ######################################
 
     - name: Run container-puppet tasks (generate config) during step {{ step }}
+      async: 3600
+      poll: 0
       shell: "{{ python_cmd }} /var/lib/container-puppet/container-puppet.py"
       environment:
         NET_HOST: 'true'
@@ -141,19 +156,27 @@
         CONTAINER_LOG_STDOUT_PATH: "{{ container_log_stdout_path }}"
         CONTAINER_HEALTHCHECK_DISABLED: "{{ container_healthcheck_disabled }}"
       when: step|int == 1
-      changed_when: false
       check_mode: no
-      register: outputs
-      failed_when: false
-      no_log: true
+      register: generate_config_async_result
+      tags:
+        - container_config
+
+    - name: Wait for container-puppet tasks (generate config) to finish
+      async_status:
+        jid: "{{ generate_config_async_result.ansible_job_id }}"
+      register: generate_config_outputs
+      until: generate_config_outputs.finished
+      retries: 1200
+      delay: 3
+      when: step|int == 1
       tags:
         - container_config
 
     - name: "Debug output for task: Run container-puppet tasks (generate config) during step {{ step }}"
       debug:
-        var: outputs.stdout_lines | default([]) | union(outputs.stderr_lines | default([]))
-      when: outputs.rc is defined
-      failed_when: outputs.rc != 0
+        var: generate_config_outputs.stdout_lines | default([]) | union(generate_config_outputs.stderr_lines | default([]))
+      when: generate_config_outputs.rc is defined
+      failed_when: generate_config_outputs.rc != 0
       tags:
         - container_config
       ignore_errors: "{{ ansible_check_mode|bool }}"
@@ -192,6 +215,8 @@
     # the *step_n.json with a hash of the generated external config added
     # This acts as a salt to enable restarting the container if config changes
     - name: Start containers for step {{ step }} using paunch
+      async: 3600
+      poll: 0
       environment:
         TRIPLEO_MINOR_UPDATE: '{{ tripleo_minor_update | default(false) }}'
       paunch:
@@ -206,16 +231,25 @@
       register: outputs
       failed_when: false
       when: enable_paunch|default(true)
+      register: start_containers_async_result
+      tags:
+        - container_startup_configs
+
+    - name: Wait for containers to start for step {{ step }} using paunch
+      async_status:
+        jid: "{{ start_containers_async_result.ansible_job_id }}"
+      register: start_containers_outputs
+      until: start_containers_outputs.finished
+      retries: 1200
+      delay: 3
       tags:
         - container_startup_configs
 
     - name: "Debug output for task: Start containers for step {{ step }}"
       debug:
-        var: outputs.stdout_lines | default([]) | union(outputs.stderr_lines | default([]))
-      when:
-        - enable_paunch|default(true)
-        - outputs.rc is defined
-      failed_when: outputs.rc != 0
+        var: start_containers_outputs.stdout_lines | default([]) | union(start_containers_outputs.stderr_lines | default([]))
+      when: start_containers_outputs.rc is defined
+      failed_when: start_containers_outputs.rc != 0
       tags:
         - container_startup_configs
 
@@ -252,6 +286,8 @@
       when: host_container_puppet_tasks is defined
 
     - name: Run container-puppet tasks (bootstrap tasks) for step {{ step }}
+      async: 3600
+      poll: 0
       shell: "{{ python_cmd }} /var/lib/container-puppet/container-puppet.py"
       environment:
         CONFIG: /var/lib/container-puppet/{{ ansible_check_mode | bool | ternary('check-mode/', '') }}container-puppet-tasks{{ step }}.json
@@ -263,17 +299,28 @@
         DEBUG: "{{ docker_puppet_debug }}"
         MOUNT_HOST_PUPPET: '{{docker_puppet_mount_host_puppet}}'
       when: host_container_puppet_tasks is defined
-      changed_when: false
-      register: outputs
-      failed_when: false
+      register: bootstrap_tasks_async_result
       no_log: true
       tags:
         - container_config_tasks
 
+    - name: Wait for container-puppet tasks (bootstrap tasks) for step {{ step }} to finish
+      async_status:
+        jid: "{{ bootstrap_tasks_async_result.ansible_job_id }}"
+      register: bootstrap_tasks_outputs
+      until: bootstrap_tasks_outputs.finished
+      retries: 1200
+      delay: 3
+      when: host_container_puppet_tasks is defined
+      tags:
+        - container_config_tasks
+
     - name: "Debug output for task: Run container-puppet tasks (bootstrap tasks) for step {{ step }}"
       debug:
-        var: outputs.stdout_lines | default([]) | union(outputs.stderr_lines | default([]))
-      when: outputs.rc is defined
-      failed_when: outputs.rc != 0
+        var: bootstrap_tasks_outputs.stdout_lines | default([]) | union(bootstrap_tasks_outputs.stderr_lines | default([]))
+      when:
+        - host_container_puppet_tasks is defined
+        - bootstrap_tasks_outputs.rc is defined
+      failed_when: bootstrap_tasks_outputs.rc != 0
       tags:
         - container_config_tasks