From 3ca8cfb48affeed8fec6b104caeeaa2ddda2c163 Mon Sep 17 00:00:00 2001
From: Sawan Choudhary <sawchoud@cisco.com>
Date: Wed, 19 Jun 2019 08:20:01 -0700
Subject: [PATCH] docker_check to identify SSH failure on nodes

Change-Id: I8284fce683303a1b86350ebf41ac9c0b8480846e
---
 .../plugins/operator_tests/operator.py        | 83 ++++++++++++-------
 1 file changed, 54 insertions(+), 29 deletions(-)

diff --git a/cloudpulse/scenario/plugins/operator_tests/operator.py b/cloudpulse/scenario/plugins/operator_tests/operator.py
index c44fb4d..b753c56 100644
--- a/cloudpulse/scenario/plugins/operator_tests/operator.py
+++ b/cloudpulse/scenario/plugins/operator_tests/operator.py
@@ -115,6 +115,15 @@ def get_container_name(name):
     return None
 
 
+def create_error_msg(error_msg, reason_msg, error_hosts):
+    if error_msg:
+        error_msg = "{}; {}: {}".format(error_msg, ', '.join(error_hosts),
+                                        reason_msg)
+    else:
+        error_msg = "{}: {}".format(', '.join(error_hosts), reason_msg)
+    return error_msg
+
+
 class operator_scenario(base.Scenario):
 
     def _get_keystone_session_creds(self):
@@ -225,42 +234,58 @@ class operator_scenario(base.Scenario):
         cmd = anscmd + cmd + ' -u root'
 
         res = execute(cmd)
-        docker_failed = None
+        docker_down = []
+        docker_down_msg = "Docker daemon down"
+        ssh_failed = []
+        ssh_failed_msg = "Host(s) unreachable via SSH"
+        unknown_status = []
+        unknown_status_msg = "Failure reason unknown"
+        container_exited = ""
+        docker_failed = ""
 
-        if not res['status']:
-            res['output'] = res['output'].split('\n')
-            output = filter(lambda x: not re.match(r'^\s*$', x), res['output'])
+        res['output'] = res['output'].split('\n')
+        output = filter(lambda x: not re.match(r'^\s*$', x), res['output'])
 
-            for line in output:
-                line = line.split('|')
+        for line in output:
+            line = line.split('|')
+            if 'FAILED' in line[1]:
+                docker_down.append(line[0].strip())
+            elif 'UNREACHABLE' in line[1]:
+                ssh_failed.append(line[0].strip())
+            elif 'SUCCESS' in line[1]:
                 if len(line) < 3:
                     continue
-                if 'SUCCESS' not in line[1]:
-                    if docker_failed:
-                        docker_failed = "{}, {}".format(docker_failed, line[0])
+                line[3] = line[3].replace(' ', '')
+                line[3] = line[3].replace('(stdout)', '')
+                if not re.match(r'^\s*$', line[3]):
+                    line[3] = line[3].replace('\\n', ', ')
+                    if container_exited:
+                        container_exited = "{}; {}: {}"\
+                            .format(container_exited,
+                                    line[0].strip(),
+                                    line[3].strip())
                     else:
-                        docker_failed = line[0]
-                else:
-                    line[3] = line[3].replace(' ', '')
-                    line[3] = line[3].replace('(stdout)', '')
-                    if not re.match(r'^\s*$', line[3]):
-                        line[3] = line[3].replace('\\n', ', ')
-                        if docker_failed:
-                            docker_failed = "{}; {}: {}"\
-                                .format(docker_failed,
-                                        line[0].strip(),
-                                        line[3].strip())
-                        else:
-                            docker_failed = "{}: {}".format(line[0].strip(),
-                                                            line[3].strip())
-            if docker_failed:
-                return (404, docker_failed, [])
+                        container_exited = "{}: {}".format(line[0].strip(),
+                                                           line[3].strip())
             else:
-                return (200, "All docker containers are up",
-                        ['Docker container Test Passed'])
+                unknown_status.append(line[0].strip())
+
+        if container_exited:
+            docker_failed = container_exited
+        if docker_down:
+            docker_failed = create_error_msg(docker_failed, docker_down_msg,
+                                             docker_down)
+        if ssh_failed:
+            docker_failed = create_error_msg(docker_failed, ssh_failed_msg,
+                                             ssh_failed)
+        if unknown_status:
+            docker_failed = create_error_msg(docker_failed, unknown_status_msg,
+                                             unknown_status)
+        if docker_failed:
+            return (404, docker_failed, [])
         else:
-            return (404, ("Docker Check Failed: %s" %
-                          "docker daemon not accessible", []))
+            return (200, "All docker containers are up",
+                    ['Docker container Test Passed'])
 
     @base.scenario(admin_only=False, operator=True)
     def ceph_check(self):