From 94cd0f876ba02253ff5df38fe15a644c052a33f2 Mon Sep 17 00:00:00 2001
From: Flavio Percoco <flaper87@gmail.com>
Date: Fri, 3 Mar 2017 11:46:17 +0100
Subject: [PATCH] Collect more info about docker
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Include volumes list and the content of some of the internal dirs in the
containers. This will provide more info about the configs mounted in the
container and their logs.

Co-Authored-By: Martin André <m.andre@redhat.com>
Change-Id: Ie5872d31dc6caf566dda7aa78b14502708babe4c
---
 scripts/common_functions.sh | 16 +++++++++++--
 scripts/get_docker_logs.sh  | 48 +++++++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+), 2 deletions(-)
 create mode 100755 scripts/get_docker_logs.sh

diff --git a/scripts/common_functions.sh b/scripts/common_functions.sh
index 4e34a3a13..6a6492bfd 100755
--- a/scripts/common_functions.sh
+++ b/scripts/common_functions.sh
@@ -178,8 +178,14 @@ function canusecache(){
 function extract_logs(){
     local name=$1
     mkdir -p $WORKSPACE/logs/$name
-    # Exclude journal files because they're large and not useful in a browser
-    tar -C $WORKSPACE/logs/$name -xf $WORKSPACE/logs/$name.tar.xz var --exclude=journal
+    local logs_tar="$WORKSPACE/logs/$name.tar.xz"
+
+    if [[ -f $logs_tar ]]; then
+        # Exclude journal files because they're large and not useful in a browser
+        tar -C $WORKSPACE/logs/$name -xf $logs_tar var --exclude=journal
+    else
+        echo "$logs_tar doesn't exist. Nothing to untar"
+    fi
 }
 
 function postci(){
@@ -205,6 +211,7 @@ function postci(){
 
     # Generate extra state information from the running undercloud
     sudo -E $TRIPLEO_ROOT/tripleo-ci/scripts/get_host_info.sh
+    sudo -E $TRIPLEO_ROOT/tripleo-ci/scripts/get_docker_logs.sh
     eval $JLOGCMD
 
     if [ "$OVB" == "1" ] ; then
@@ -224,7 +231,9 @@ function postci(){
             NAME=${INSTANCE//=*}
             (
                 scp $SSH_OPTIONS $TRIPLEO_ROOT/tripleo-ci/scripts/get_host_info.sh heat-admin@${SANITIZED_ADDRESS}:/tmp
+                scp $SSH_OPTIONS $TRIPLEO_ROOT/tripleo-ci/scripts/get_docker_logs.sh heat-admin@${SANITIZED_ADDRESS}:/tmp
                 timeout -s 15 -k 600 300 ssh $SSH_OPTIONS heat-admin@$IP sudo /tmp/get_host_info.sh
+                timeout -s 15 -k 600 300 ssh $SSH_OPTIONS heat-admin@$IP sudo /tmp/get_docker_logs.sh
                 ssh $SSH_OPTIONS heat-admin@$IP $JLOGCMD
                 ssh $SSH_OPTIONS heat-admin@$IP $TARCMD > $WORKSPACE/logs/${NAME}.tar.xz
                 extract_logs $NAME
@@ -248,6 +257,8 @@ function postci(){
             mkdir $WORKSPACE/logs/subnode-$i/
             ssh $SSH_OPTIONS -i /etc/nodepool/id_rsa $ip \
                 sudo $TRIPLEO_ROOT/tripleo-ci/scripts/get_host_info.sh
+            ssh $SSH_OPTIONS -i /etc/nodepool/id_rsa $ip \
+                sudo $TRIPLEO_ROOT/tripleo-ci/scripts/get_docker_logs.sh
             ssh $SSH_OPTIONS -i /etc/nodepool/id_rsa $ip $JLOGCMD
             ssh $SSH_OPTIONS -i /etc/nodepool/id_rsa $ip \
                 $TARCMD > $WORKSPACE/logs/subnode-$i/subnode-$i.tar.xz
@@ -262,6 +273,7 @@ function postci(){
                 sudo rm -f /etc/sahara/rootwrap.d/sahara.filters
             ssh $SSH_OPTIONS -i /etc/nodepool/id_rsa $ip \
                 sudo rm -f /etc/cinder/rootwrap.d/os-brick.filters
+
             let i+=1
         done
     fi
diff --git a/scripts/get_docker_logs.sh b/scripts/get_docker_logs.sh
new file mode 100755
index 000000000..426d9cdda
--- /dev/null
+++ b/scripts/get_docker_logs.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+BASE_DOCKER_EXTRA="/var/log/extra/docker"
+
+if command -v docker && systemctl is-active docker; then
+    mkdir -p $BASE_DOCKER_EXTRA
+    ALL_FILE=$BASE_DOCKER_EXTRA/docker_allinfo.log
+    DOCKER_INFO_CMDS=(
+        "docker ps --all --size"
+        "docker images"
+        "docker volume ls"
+        "docker stats --all --no-stream"
+        "docker info"
+    )
+    for cmd in "${DOCKER_INFO_CMDS[@]}"; do
+        echo "+ $cmd" >> $ALL_FILE
+        $cmd >> $ALL_FILE
+    done
+
+    for cont in $(docker ps | awk {'print $NF'} | grep -v NAMES); do
+        INFO_DIR=$BASE_DOCKER_EXTRA/containers/${cont}
+        mkdir -p $INFO_DIR
+        INFO_FILE=$INFO_DIR/docker_info.log
+        DOCKER_CONTAINER_INFO_CMDS=(
+            "docker top $cont auxw"
+            "docker exec $cont top -bwn1"
+            "docker inspect $cont"
+        )
+        for cmd in "${DOCKER_CONTAINER_INFO_CMDS[@]}"; do
+            echo "+ $cmd" >> $INFO_FILE
+            $cmd >> $INFO_FILE
+        done
+        docker logs $cont > $INFO_DIR/stdout.log
+        docker cp $cont:/etc $INFO_DIR/etc
+        docker cp $cont:/var/lib/kolla/config_files/config.json $INFO_DIR/config.json
+        # NOTE(flaper87): This should go away. Services should be
+        # using a `logs` volume
+        docker cp $cont:/var/log $INFO_DIR/log
+
+        # Delete symlinks because they break log collection and are generally
+        # not useful
+        find $INFO_DIR -type l -delete
+    done
+
+    if [[ -d /var/lib/docker/volumes/logs/_data ]]; then
+        cp -r /var/lib/docker/volumes/logs/_data $BASE_DOCKER_EXTRA/logs
+    fi
+fi