From 07509721e8dbb2555eb9c78db7e7c297d18209ef Mon Sep 17 00:00:00 2001
From: Jesse Pretorius <jesse.pretorius@rackspace.co.uk>
Date: Fri, 9 Sep 2016 10:38:13 +0100
Subject: [PATCH] Compress test execution logs

The volume of logs we collect in CI jobs are extensive. This
patch ensures that the logs collected are compressed to reduce
the space taken in CI systems.

PYTHONUNBUFFERED is also set to ensure that the console log
from the CI jobs is recorded in the exact order of execution.

Change-Id: Ia37c4fe369dd8ce9d49c21a46c575707b1826d46
Related-Bug: #1620849
---
 bindep.txt |  3 +++
 tox.ini    | 23 ++++++++++++++---------
 2 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/bindep.txt b/bindep.txt
index 931e963d..fe589259 100644
--- a/bindep.txt
+++ b/bindep.txt
@@ -36,3 +36,6 @@ python-ndg-httpsclient      [platform:ubuntu !platform:ubuntu-trusty]
 python2-pyasn1              [platform:rpm]
 pyOpenSSL                   [platform:rpm]
 python-ndg_httpsclient      [platform:rpm]
+
+# Required for compressing collected log files in CI
+gzip
diff --git a/tox.ini b/tox.ini
index c0b86582..622d0b89 100644
--- a/tox.ini
+++ b/tox.ini
@@ -27,6 +27,7 @@ whitelist_externals =
     wget
 setenv =
     VIRTUAL_ENV={envdir}
+    PYTHONUNBUFFERED=1
 
 
 [testenv:docs]
@@ -155,10 +156,17 @@ commands =
 install_command =
     pip install -U --force-reinstall {opts} {packages}
 
+[testenv:func_logs]
+commands =
+    bash -c 'mkdir -p {toxinidir}/logs'
+    bash -c 'rsync --archive --verbose --ignore-errors /var/log/ /openstack/log/ {toxinidir}/logs/ || true'
+    bash -c 'find "{toxinidir}/logs/" -type f | sed "p;s|$|.txt|" | xargs -n2 mv'
+    bash -c 'command gzip --best --recursive "{toxinidir}/logs/"'
+
 
 [testenv:functional]
 # Ignore_errors is set to true so that the logs are collected at the
-#  end of the run. This will not produce a failse positive. Any
+#  end of the run. This will not produce a false positive. Any
 #  exception will be mark the run as "failed" and exit 1 after all of
 #  the commands have been iterated through.
 ignore_errors = True
@@ -175,13 +183,12 @@ commands =
                      -e @{toxinidir}/tests/keystone-overrides.yml \
                      -e "keystone_rolename={toxinidir}" \
                      {toxinidir}/tests/test.yml -vvvv
-    bash -c 'mkdir -p {toxinidir}/logs'
-    bash -c 'rsync -av --ignore-errors /var/log/ /openstack/log/ {toxinidir}/logs/ || true'
+    {[testenv:func_logs]commands}
 
 
 [testenv:uwsgi_apache]
 # Ignore_errors is set to true so that the logs are collected at the
-#  end of the run. This will not produce a failse positive. Any
+#  end of the run. This will not produce a false positive. Any
 #  exception will be mark the run as "failed" and exit 1 after all of
 #  the commands have been iterated through.
 ignore_errors = True
@@ -199,13 +206,12 @@ commands =
                      -e "keystone_rolename={toxinidir}" \
                      -e "keystone_mod_wsgi_enabled=False" \
                      {toxinidir}/tests/test.yml -vvvv
-    bash -c 'mkdir -p {toxinidir}/logs'
-    bash -c 'rsync -av --ignore-errors /var/log/ /openstack/log/ {toxinidir}/logs/ || true'
+    {[testenv:func_logs]commands}
 
 
 [testenv:uwsgi_nginx]
 # Ignore_errors is set to true so that the logs are collected at the
-#  end of the run. This will not produce a failse positive. Any
+#  end of the run. This will not produce a false positive. Any
 #  exception will be mark the run as "failed" and exit 1 after all of
 #  the commands have been iterated through.
 ignore_errors = True
@@ -224,8 +230,7 @@ commands =
                      -e "keystone_apache_enabled=False" \
                      -e "keystone_mod_wsgi_enabled=False" \
                      {toxinidir}/tests/test.yml -vvvv
-    bash -c 'mkdir -p {toxinidir}/logs'
-    bash -c 'rsync -av --ignore-errors /var/log/ /openstack/log/ {toxinidir}/logs/ || true'
+    {[testenv:func_logs]commands}
 
 
 [testenv:linters]