From 57a9fc5bce838497846bd97b25a7517f7943da96 Mon Sep 17 00:00:00 2001
From: Ilya Tyaptin <ityaptin@mirantis.com>
Date: Fri, 15 May 2015 16:43:36 +0300
Subject: [PATCH] Add running functional scripts for defined backend

This CR allows running "functional" tox job for different backends.
Short description about expected workflow:
1. Gate job run ceilometer/ceilometer/tests/functional/hooks/post_test_hook.py
with <backend> parameter.
2. This script run "tox -efunctional" command with defined variable
CEILOMETER_TEST_BACKEND
3. After in "run-functional-tests.sh" setup script for the backend
and testr are runned.

In this CR running testr with pretty_tox.sh script added because
it allows to use subunit-trace output which developed in tempest-lib
and improve useful of testr output.

Partially implements: blueprint ceilometer-functional-tests
Change-Id: Idb66aca0b46779516db2baec856df8223dbe5c13
---
 .../tests/functional/hooks/post_test_hook.sh  |  3 ++-
 ceilometer/tests/functional/test_empty.py     | 23 +++++++++++++++++++
 run-functional-tests.sh                       | 12 ++++++++++
 ...est-env-elastic.sh => setup-test-env-es.sh |  0
 test-requirements.txt                         |  1 +
 tools/pretty_tox.sh                           | 16 +++++++++++++
 tox.ini                                       |  6 ++---
 7 files changed, 56 insertions(+), 5 deletions(-)
 create mode 100644 ceilometer/tests/functional/test_empty.py
 create mode 100644 run-functional-tests.sh
 rename setup-test-env-elastic.sh => setup-test-env-es.sh (100%)
 create mode 100755 tools/pretty_tox.sh

diff --git a/ceilometer/tests/functional/hooks/post_test_hook.sh b/ceilometer/tests/functional/hooks/post_test_hook.sh
index 6701ac9a..dce23bd5 100644
--- a/ceilometer/tests/functional/hooks/post_test_hook.sh
+++ b/ceilometer/tests/functional/hooks/post_test_hook.sh
@@ -37,7 +37,8 @@ sudo chown -R jenkins:stack $CEILOMETER_DIR
 echo "Running ceilometer functional test suite"
 set +e
 
-sudo -E -H -u jenkins tox -efunctional
+# NOTE(ityaptin) Expected a script param which contains a backend name
+CEILOMETER_TEST_BACKEND="$1" sudo -E -H -u jenkins tox -efunctional
 EXIT_CODE=$?
 set -e
 
diff --git a/ceilometer/tests/functional/test_empty.py b/ceilometer/tests/functional/test_empty.py
new file mode 100644
index 00000000..043eaa6b
--- /dev/null
+++ b/ceilometer/tests/functional/test_empty.py
@@ -0,0 +1,23 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+""" This test is written to avoid failing running the functional test job
+caused by `subunit-trace -f` with empty testr output
+in ./tools/pretty-tox.sh"""
+
+from oslotest import base
+
+
+class TestEmpty(base.BaseTestCase):
+    def test_empty(self):
+        pass
diff --git a/run-functional-tests.sh b/run-functional-tests.sh
new file mode 100644
index 00000000..0ebc5737
--- /dev/null
+++ b/run-functional-tests.sh
@@ -0,0 +1,12 @@
+#!/bin/bash -x
+set -e
+# Use a mongodb backend by default
+
+
+if [ -z $CEILOMETER_TEST_BACKEND ]; then
+    CEILOMETER_TEST_BACKEND="mongodb"
+fi
+echo $CEILOMETER_TEST_BACKEND
+for backend in $CEILOMETER_TEST_BACKEND; do
+    ./setup-test-env-${backend}.sh ./tools/pretty_tox.sh $*
+done
diff --git a/setup-test-env-elastic.sh b/setup-test-env-es.sh
similarity index 100%
rename from setup-test-env-elastic.sh
rename to setup-test-env-es.sh
diff --git a/test-requirements.txt b/test-requirements.txt
index d0aa334e..b33926f9 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -31,3 +31,4 @@ testscenarios>=0.4
 testtools>=0.9.36,!=1.2.0
 gabbi>=0.12.0  # Apache-2.0
 requests-aws>=0.1.4    # BSD License (3 clause)
+tempest-lib>=0.5.0
diff --git a/tools/pretty_tox.sh b/tools/pretty_tox.sh
new file mode 100755
index 00000000..799ac184
--- /dev/null
+++ b/tools/pretty_tox.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -o pipefail
+
+TESTRARGS=$1
+
+# --until-failure is not compatible with --subunit see:
+#
+# https://bugs.launchpad.net/testrepository/+bug/1411804
+#
+# this work around exists until that is addressed
+if [[ "$TESTARGS" =~ "until-failure" ]]; then
+    python setup.py testr --slowest --testr-args="$TESTRARGS"
+else
+    python setup.py testr --slowest --testr-args="--subunit $TESTRARGS" | subunit-trace -f
+fi
diff --git a/tox.ini b/tox.ini
index 070c90e8..725a99cf 100644
--- a/tox.ini
+++ b/tox.ini
@@ -36,14 +36,12 @@ commands =
 commands =
    bash -x {toxinidir}/setup-test-env-elastic.sh python setup.py testr --slowest --testr-args="{posargs}"
 
-# TODO(ityaptin) create a script for running different setup-test-env-*.sh
-# for different backends
 [testenv:functional]
 setenv = VIRTUAL_ENV={envdir}
          EVENTLET_NO_GREENDNS=yes
-         OS_TEST_PATH=ceilometer/tests/functional
+         OS_TEST_PATH=ceilometer/tests/functional/
 commands =
-   bash -x {toxinidir}/setup-test-env-mongodb.sh python setup.py testr --slowest --testr-args="{posargs}"
+   bash -x {toxinidir}/run-functional-tests.sh "{posargs}"
 
 [testenv:py34]
 deps = -r{toxinidir}/requirements.txt