Fix dstat time penalty for failed runs

dstat async task with poll=0 should fire and forget, with 1 sec
max limit for the shell to run and send its process to bg.

The change includes:
 - Ignore errors on dstat play.
 - Enable dstat only when tempest runs.
 - Kills dstat before parsing logs and creating charts.
 - Remove dstat_count.

Closes-Bug: #1760917

Change-Id: I9ee407e8274f3ab6c3aea3650a25cd31772a81d6

playbooks/baremetal-full-overcloud-validate.yml

@@ -5,7 +5,9 @@
   tags:
     - overcloud-validate
   roles:
-    - { role: validate-perf, when: validate_performance|bool }
+    - { role: validate-perf,
+        when: validate_performance|bool and run_tempest|bool }
+  ignore_errors: true
 
 - name: Sanity check the overcloud services
   hosts: undercloud

playbooks/multinode-validate.yml

@@ -4,7 +4,9 @@
   tags:
     - overcloud-validate
   roles:
-    - { role: validate-perf, when: validate_performance|bool }
+    - { role: validate-perf,
+        when: validate_performance|bool and run_tempest|bool }
+  ignore_errors: true
 
 # Validate the deployment
 - name:  validate the overcloud

playbooks/quickstart-extras-validate.yml

@@ -4,7 +4,9 @@
   tags:
     - overcloud-validate
   roles:
-    - { role: validate-perf, when: validate_performance|bool }
+    - { role: validate-perf,
+        when: validate_performance|bool and run_tempest|bool }
+  ignore_errors: true
 
 - name:  validate the overcloud
   hosts: undercloud

roles/collect-logs/tasks/collect.yml

@@ -106,6 +106,11 @@
       stat: path=/var/log/extra/dstat-csv.log
       register: dstat_logfile
 
+    - name: kill dstat
+      shell: "pkill dstat"
+      become: true
+      when: dstat_logfile.stat.exists
+
     - name: Get dstat_graph tool
       git:
         repo: "https://github.com/Dabz/dstat_graph.git"

roles/validate-perf/defaults/main.yml

@@ -1,4 +1,3 @@
 dstat_log_name: dstat-csv.log
 dstat_options: tcmndrylpg
 dstat_delay: 1
-dstat_count: 7200

roles/validate-perf/tasks/main.yml

@@ -1,27 +1,29 @@
 ---
-- name: install the latest version of dstat on overcloud
-  package:
-    name: dstat
-    state: present
+- name: Validate performance (dstat) on overcloud
+  block:
+  - name: Install the latest version of dstat on overcloud
+    package:
+      name: dstat
+      state: present
+  - name: Ensure /var/log/extra exists
+    file:
+      path: /var/log/extra
+      state: directory
+      mode: 0777
+  - name: Run dstat for collecting metrics on overcloud
+    shell:
+      cmd: >
+        dstat
+        -{{ dstat_options }}
+        --nocolor
+        --output /var/log/extra/{{ dstat_log_name }}
+        {{ dstat_delay }}
+        > /dev/null &
+    async: 1
+    poll: 0
+    executable: /bin/bash
+    changed_when: false
+  rescue:
+  - debug:
+      msg: "Warning: Performance cannot be validated."
   become: true
-
-- name: ensure /var/log/extra exists
-  file:
-    path: /var/log/extra
-    state: directory
-    mode: 0755
-  become: true
-
-- name: Run dstat for collecting metrics on overcloud
-  command: >-
-    dstat
-    -{{ dstat_options }}
-    --nocolor
-    --output /var/log/extra/{{ dstat_log_name }}
-    {{ dstat_delay }}
-    {{ dstat_count }}
-    > /dev/null
-  async: 7200
-  poll: 0
-  become: true
-  changed_when: false