From df408ef9d39d923d69bbf5b014d5cc6ef00a9914 Mon Sep 17 00:00:00 2001
From: Rafael Folco <rfolco@redhat.com>
Date: Wed, 4 Apr 2018 16:39:22 -0300
Subject: [PATCH] Fix dstat time penalty for failed runs

dstat async task with poll=0 should fire and forget, with 1 sec
max limit for the shell to run and send its process to bg.

The change includes:
 - Ignore errors on dstat play.
 - Enable dstat only when tempest runs.
 - Kills dstat before parsing logs and creating charts.
 - Remove dstat_count.

Closes-Bug: #1760917

Change-Id: I9ee407e8274f3ab6c3aea3650a25cd31772a81d6
---
 .../baremetal-full-overcloud-validate.yml     |  4 +-
 playbooks/multinode-validate.yml              |  4 +-
 playbooks/quickstart-extras-validate.yml      |  4 +-
 roles/collect-logs/tasks/collect.yml          |  5 ++
 roles/validate-perf/defaults/main.yml         |  1 -
 roles/validate-perf/tasks/main.yml            | 52 ++++++++++---------
 6 files changed, 41 insertions(+), 29 deletions(-)

diff --git a/playbooks/baremetal-full-overcloud-validate.yml b/playbooks/baremetal-full-overcloud-validate.yml
index 471d0802b..7610016a7 100644
--- a/playbooks/baremetal-full-overcloud-validate.yml
+++ b/playbooks/baremetal-full-overcloud-validate.yml
@@ -5,7 +5,9 @@
   tags:
     - overcloud-validate
   roles:
-    - { role: validate-perf, when: validate_performance|bool }
+    - { role: validate-perf,
+        when: validate_performance|bool and run_tempest|bool }
+  ignore_errors: true
 
 - name: Sanity check the overcloud services
   hosts: undercloud
diff --git a/playbooks/multinode-validate.yml b/playbooks/multinode-validate.yml
index 1cfcd2cad..24ca9554b 100644
--- a/playbooks/multinode-validate.yml
+++ b/playbooks/multinode-validate.yml
@@ -4,7 +4,9 @@
   tags:
     - overcloud-validate
   roles:
-    - { role: validate-perf, when: validate_performance|bool }
+    - { role: validate-perf,
+        when: validate_performance|bool and run_tempest|bool }
+  ignore_errors: true
 
 # Validate the deployment
 - name:  validate the overcloud
diff --git a/playbooks/quickstart-extras-validate.yml b/playbooks/quickstart-extras-validate.yml
index 4f80a3756..8dd64976b 100644
--- a/playbooks/quickstart-extras-validate.yml
+++ b/playbooks/quickstart-extras-validate.yml
@@ -4,7 +4,9 @@
   tags:
     - overcloud-validate
   roles:
-    - { role: validate-perf, when: validate_performance|bool }
+    - { role: validate-perf,
+        when: validate_performance|bool and run_tempest|bool }
+  ignore_errors: true
 
 - name:  validate the overcloud
   hosts: undercloud
diff --git a/roles/collect-logs/tasks/collect.yml b/roles/collect-logs/tasks/collect.yml
index 94ac84fe9..bec759580 100644
--- a/roles/collect-logs/tasks/collect.yml
+++ b/roles/collect-logs/tasks/collect.yml
@@ -106,6 +106,11 @@
       stat: path=/var/log/extra/dstat-csv.log
       register: dstat_logfile
 
+    - name: kill dstat
+      shell: "pkill dstat"
+      become: true
+      when: dstat_logfile.stat.exists
+
     - name: Get dstat_graph tool
       git:
         repo: "https://github.com/Dabz/dstat_graph.git"
diff --git a/roles/validate-perf/defaults/main.yml b/roles/validate-perf/defaults/main.yml
index f0f6aed76..211103e57 100644
--- a/roles/validate-perf/defaults/main.yml
+++ b/roles/validate-perf/defaults/main.yml
@@ -1,4 +1,3 @@
 dstat_log_name: dstat-csv.log
 dstat_options: tcmndrylpg
 dstat_delay: 1
-dstat_count: 7200
diff --git a/roles/validate-perf/tasks/main.yml b/roles/validate-perf/tasks/main.yml
index fef24582f..97d3762c3 100644
--- a/roles/validate-perf/tasks/main.yml
+++ b/roles/validate-perf/tasks/main.yml
@@ -1,27 +1,29 @@
 ---
-- name: install the latest version of dstat on overcloud
-  package:
-    name: dstat
-    state: present
+- name: Validate performance (dstat) on overcloud
+  block:
+  - name: Install the latest version of dstat on overcloud
+    package:
+      name: dstat
+      state: present
+  - name: Ensure /var/log/extra exists
+    file:
+      path: /var/log/extra
+      state: directory
+      mode: 0777
+  - name: Run dstat for collecting metrics on overcloud
+    shell:
+      cmd: >
+        dstat
+        -{{ dstat_options }}
+        --nocolor
+        --output /var/log/extra/{{ dstat_log_name }}
+        {{ dstat_delay }}
+        > /dev/null &
+    async: 1
+    poll: 0
+    executable: /bin/bash
+    changed_when: false
+  rescue:
+  - debug:
+      msg: "Warning: Performance cannot be validated."
   become: true
-
-- name: ensure /var/log/extra exists
-  file:
-    path: /var/log/extra
-    state: directory
-    mode: 0755
-  become: true
-
-- name: Run dstat for collecting metrics on overcloud
-  command: >-
-    dstat
-    -{{ dstat_options }}
-    --nocolor
-    --output /var/log/extra/{{ dstat_log_name }}
-    {{ dstat_delay }}
-    {{ dstat_count }}
-    > /dev/null
-  async: 7200
-  poll: 0
-  become: true
-  changed_when: false