openstack-ansible/playbooks/roles/os_tempest/templates/openstack_tempest_gate.sh.j2

#!/usr/bin/env bash
# Copyright 2014, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Script for running gate tests. Initially very sparse
# additional projects and test types will be added over time.


# -------------------- Shell Options -------------------------
set -x


# -------------------- Parameters -------------------------
# The only parameters this script takes is the names of the test lists
# to use:
# ./$0 <test_list_name> ...
#
# If a name is not supplied scenario will be used.
# If multiple lists are given, the resulting tests are combined and
# duplicates are removed.

test_lists=${*:-scenario}
testr_output_lines=${testr_output_lines:-100}
RUN_TEMPEST_OPTS=${RUN_TEMPEST_OPTS:-''}
TESTR_OPTS=${TESTR_OPTS:-''}


# -------------------- Functions -------------------------
# Test list functions. Each tempest test scenario (eg scenario, api, smoke,
# defcore) should have a function here to generate the list of tests that
# should be run. Each function takes in the full list of tempest tests and
# should output a filtered list.

function gen_test_list_api() {
  # filter test list to produce list of tests to use.
  egrep 'tempest\.api\.(identity|image|volume|network|compute|object_storage)'\
    |grep -vi xml
}

# Run selected scenario tests
function gen_test_list_scenario() {
  # network tests have been removed due to
  # https://bugs.launchpad.net/openstack-ansible/+bug/1425255
  # TODO: add them back once the bug has been fixed
  egrep 'tempest\.scenario\.test_(minimum|swift|server|dashboard)_basic(_ops)?'
}

# Run heat-api tests
function gen_test_list_heat_api() {
  # basic orchestration api tests
  egrep 'tempest\.api\.orchestration\.stacks\.test_non_empty_stack'
}

# Run cinder backup api tests
function gen_test_list_cinder_backup() {
  egrep 'tempest\.api\.volume\.admin\.test_volumes_backup'
}

# Run smoke tests
function gen_test_list_smoke() {
  # this specific test fails frequently and is making our multi node nightly
  # job unstable (see bug in gen_test_list_scenario function)
  # TODO: re-add back in once the specific issue is identified and corrected
  grep smoke | grep -v tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_update_router_admin_state
}

# Run all tests
function gen_test_list_all() {
  cat
}

# Generate test list from official defcore/refstack spec
# Note that unlike the other test list functions, this one isn't a filter
# it ignores its stdin and pulls the test list from github.
function gen_test_list_defcore(){
  branch=${1:-master}
  string=$(python <<END
import json
import requests

branch="$branch"
map = {
    'master': '2015.next',
    'juno': '2015.04',
    'icehouse': '2015.04'
}

url=("https://raw.githubusercontent.com/openstack/"
     "defcore/master/%(version)s.json"
     % {'version': map[branch]}
     )
response = requests.get(url)
for capability in response.json()['capabilities'].values():
  for test in capability['tests']:
    print test
END)

  read -a test_list <<<$string
  parse_cmd="grep "
  for item in ${test_list[@]}; do parse_cmd+="-e $item "; done

  $parse_cmd
}

# defcore tests for juno
function gen_test_list_defcore_juno(){
  gen_test_list_defcore juno
}

# defcore tests for icehouse
function gen_test_list_defcore_icehouse(){
  gen_test_list_defcore icehouse
}

exit_msg(){
  echo $1
  exit $2
}


# -------------------- Main -------------------------

available_test_lists=$(compgen -A function|sed -n '/^gen_test_list_/s/gen_test_list_//p')

for test_list_name in $test_lists; do
  grep $test_list_name <<<$available_test_lists ||\
    exit_msg "$test_list_name is not a valid test list, available test lists: $available_test_lists" 1
done

# work in tempest directory
pushd {{ tempest_git_dest }}

# Load the tempest venv for a tempest run
source {{ tempest_git_dest }}/bin/activate

# read creds into environment
source /root/openrc

# create testr repo - required for testr to do anything
testr init &>/dev/null ||:

# Get list of available tests.
# lines 1-$testr_output_lines are output to stdout, all lines are written to
# full_test_list.
set -o pipefail
testr list-tests |tee >(sed -n 1,${testr_output_lines}p) >full_test_list ||\
  exit_msg "Failed to generate test list" $?
set +o pipefail

# Check the full test list is not empty
[[ -s full_test_list ]] || exit_msg "No tests found" 1

# Write filter test list using selected function. The full test list is
# pre-filtered to only include test lines, this saves adding that filter to
# every test list function.
truncate --size 0 tmp_test_list
for test_list_name in ${test_lists}; do
  grep '^tempest\.' < full_test_list \
    | gen_test_list_${test_list_name} >> tmp_test_list \
    || exit_msg "Filter $test_list_name failed. Output: $(cat test_list)" 1
done

# Eliminate duplicate tests
awk '!seen[$0]++' tmp_test_list > test_list
rm tmp_test_list

# Check the filtered test list is not empty
[[ -s test_list ]] || exit_msg "No tests remain after filtering" 1

test_list_summary="${test_lists} ($(wc -l <test_list) tests)"

echo "Using test list $test_list_summary"

# execute chosen tests with pretty output
./run_tempest.sh --no-virtual-env ${RUN_TEMPEST_OPTS} -- --load-list test_list ${TESTR_OPTS};
result=$?
popd

if [[ $result == 0 ]]; then
  echo "TEMPEST PASS $test_list_summary"
else
  echo "TEMPEST FAIL $test_list_summary"
fi

# Deactivate the venv after run
deactivate || true

exit $result