openinfra
/
interop
Code Issues Proposed changes
interop/tools/cross_check/cross_check.sh

255 lines
8.3 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
# This script is useful when it is time to find
# new candidate tests for refstack. It lists
# all API tests from tempest, designate-tempest-plugin,
# heat-tempest-plugin and manila tempest-plugin except
# those that:
# * are/were part of interop repository or
# * are mentioned in exclude file (see option --exclude-file) or
# * are scenario tests or
# * require admin access.
#
# The list of tests that is produced by this script is
# good start to find new tests but not all tests listed
# by the script must necessarily be added to the refstack.
function print_help() {
SCRIPT_NAME="basename ${BASH_SOURCE[0]}"
echo "Usage: ${SCRIPT_NAME} [OPTION] ..."
echo "Find missing test in interop repository."
echo ""
echo "--interop-url URL to download interop git repository"
echo "--interop-commit ID of interop commit that should be used"
echo "--tempest-url URL to download tempest git repository"
echo "--tempest-commit ID of tempest commit that should be used"
echo "--designate-url URL to download designate-tempest-plugin git repository"
echo "--designate-commit ID of designate-tempest-plugin commit that should be used"
echo "--heat-url URL to download heat-tempest-plugin"
echo "--heat-commit ID of heat-tempest-plugin commit that should be used"
echo "--manila-url URL to download manila-tempest-plugin"
echo "--manila-commit ID of manila-tempest-plugin commit that should be used"
echo "--exclude-file File that contains tests that should be ignored"
echo " (Default: ./exclude_file.txt)"
}
TEMPEST_GIT_URL=https://opendev.org/openstack/tempest.git
TEMPEST_COMMIT=1d500e79156ada6bc6fdb628ed1da0efd4121f6a
DESIGNATE_GIT_URL=https://opendev.org/openstack/designate-tempest-plugin.git
DESIGNATE_COMMIT=master
HEAT_GIT_URL=https://opendev.org/openstack/heat-tempest-plugin.git
HEAT_COMMIT=master
MANILA_GIT_URL=https://opendev.org/openstack/manila-tempest-plugin.git
MANILA_COMMIT=master
INTEROP_GIT_URL=https://opendev.org/osf/interop.git
INTEROP_COMMIT=master
TMP_DIR=$(mktemp -d)
SCRIPTDIR=$(dirname $0)
EXCLUDE_FILE="${SCRIPTDIR}/exclude_file.txt"
while [[ ! -z "$1" ]]; do
case "$1" in
--interop-url)
INTEROP_GIT_URL="$2";
shift 2;;
--interop-commit)
INTEROP_COMMIT="$2";
shift 2;;
--tempest-url)
TEMPEST_GIT_URL="$2";
shift 2;;
--tempest-commit)
TEMPEST_COMMIT="$2";
shift 2;;
--designate-url)
DESIGNATE_GIT_URL="$2";
shift 2;;
--designate-commit)
DESIGNATE_COMMIT="$2";
shift 2;;
--heat-url)
HEAT_URL="$2";
shift 2;;
--heat-commit)
HEAT_COMMIT="$2";
shift 2;;
--manila-url)
MANILA_URL="$2";
shift 2;;
--manila-commit)
MANILA_COMMIT="$2"
shift 2;;
--exclude-file)
EXCLUDE_FILE="$2";
shift 2;;
--help)
print_help
exit;;
*)
echo "ERROR: Not valid option used ($1)";
exit;;
esac
done
if [[ ! -z ${EXCLUDE_FILE} ]] && [[ -f ${EXCLUDE_FILE} ]]; then
EXCLUDE_FILE=$(realpath ${EXCLUDE_FILE})
elif [[ ! -z ${EXCLUDE_FILE} ]] && [[ ! -f ${EXCLUDE_FILE} ]]; then
echo "ERROR: Not valid exclude file (${EXCLUDE_FILE})"
exit
else
touch ${TMP_DIR}/exclude_file.txt
EXCLUDE_FILE="${TMP_DIR}/exclude_file.txt"
fi
echo "Temporary directory used by cross_check.sh: ${TMP_DIR}"
# Clone repositories
echo "Cloning interop repository ..."
git clone --quiet ${INTEROP_GIT_URL} ${TMP_DIR}/interop
cd ${TMP_DIR}/interop
git checkout --quiet ${INTEROP_COMMIT}
cd - &> /dev/null
echo "Cloning tempest repository ..."
git clone --quiet ${TEMPEST_GIT_URL} ${TMP_DIR}/tempest
cd ${TMP_DIR}/tempest
git checkout --quiet ${TEMPEST_COMMIT}
cd - &> /dev/null
echo "Cloning designate-tempest-plugin repository ..."
git clone --quiet ${DESIGNATE_GIT_URL} ${TMP_DIR}/designate-tempest-plugin
cd ${TMP_DIR}/designate-tempest-plugin
git checkout --quiet ${DESIGNATE_COMMIT}
cd - &> /dev/null
echo "Cloning heat-tempest-plugin repository ..."
git clone --quiet ${HEAT_GIT_URL} ${TMP_DIR}/heat-tempest-plugin
cd ${TMP_DIR}/heat-tempest-plugin
git checkout --quiet ${HEAT_COMMIT}
cd - &> /dev/null
echo "Cloning manila-tempest-plugin repository ..."
git clone --quiet ${MANILA_GIT_URL} ${TMP_DIR}/manila-tempest-plugin
cd ${TMP_DIR}/manila-tempest-plugin
git checkout --quiet ${MANILA_COMMIT}
cd - &> /dev/null
########################################################
# Find and print missing tests from interop repository.
#
# GLOBALS:
# TMP_DIR, EXCLUDE_FILE
# ARGUMENTS:
# 1 - Path to the repository in which the missing
# tests should be found.
# OUTPUTS:
# List of tests that are part of passed repository
# but not part of interop and not mentioned in
# exclude file.
#
#######################################################
function cross_check() {
REPO_DIR=$1
# Get every idempotent id from the repository
grep -hr --exclude-dir="doc" "@decorators.idempotent_id('" ${REPO_DIR} \
| sed --expression="s/@decorators.idempotent_id('//g" \
| sed --expression="s/')//g" \
| sed --expression="s/ *//g" \
| sort > ${TMP_DIR}/repo_idempotent_ids.txt
# Get every idempotent id from interop
cd ${TMP_DIR}/interop/guidelines
cat $(ls -1 | grep -E "[0-9]{4}\.[0-9]{2}\.json|next.json") ${TMP_DIR}/interop/add-ons/guidelines/* \
| grep -oE "id-[[:alnum:]]{8}-[[:alnum:]]{4}-[[:alnum:]]{4}-[[:alnum:]]{4}-[[:alnum:]]{12}" \
| sed --expression='s/id-//g' \
| sort > ${TMP_DIR}/interop_idempotent_ids.txt
cd - &> /dev/null
# Get ids of missing tests in interop
comm -23 ${TMP_DIR}/repo_idempotent_ids.txt ${TMP_DIR}/interop_idempotent_ids.txt \
| sort > ${TMP_DIR}/missing_idempotent_ids.txt
# Print missing tests to stdout
cd ${REPO_DIR}
while read ID; do
FILE=$(grep -rl "${ID}" && cd - > /dev/null)
FUNC_NAME=$(sed -e "1,/${ID}/d" ${FILE} \
| grep -m1 "def" \
| sed "s/def//g; s/(.*//g; s/ //g")
CLASS_NAME=$(sed "/${ID}/q" ${FILE} \
| grep "^class " \
| tail -1 \
| sed "s/class//g; s/(.*//g; s/ //g")
FILE_ID=$(echo ${FILE} | sed "s/\.py//g; s/\.\///g; s/\//\./g")
TEST_NAME="${FILE_ID}.${CLASS_NAME}.${FUNC_NAME}"
# Check if test:
# * requires admin access or
# * is already present in interop next files
# (necessary for older next files as they do not
# use idempotent ids) or
# * is scenario test or
# * is present in exclude file.
ADMIN_TEST=$(echo ${FILE_ID} | grep "\.admin\.")
SCENARIO_TEST=$(echo ${FILE_ID} | grep "\.scenario\.")
IN_INTEROP=$(grep -r ${TEST_NAME} ${TMP_DIR}/interop)
IN_EXCLUDE=$(cat ${EXCLUDE_FILE} | grep "${TEST_NAME}")
if [[ ! ${IN_INTEROP} ]] && \
[[ ! ${ADMIN_TEST} ]] && \
[[ ! ${IN_EXCLUDE} ]] && \
[[ ! ${SCENARIO_TEST} ]]
then
echo "${TEST_NAME}[${ID}]"
fi
done < ${TMP_DIR}/missing_idempotent_ids.txt
cd - &> /dev/null
}
echo "Searching tempest repository ..."
cross_check ${TMP_DIR}/tempest > ${TMP_DIR}/tempest_tests.txt
echo "Searching designate-tempest-plugin repository ..."
cross_check ${TMP_DIR}/designate-tempest-plugin > ${TMP_DIR}/designate_tests.txt
echo "Searching heat-tempest-plugin repository ..."
cross_check ${TMP_DIR}/heat-tempest-plugin > ${TMP_DIR}/heat_tests.txt
echo "Searching manila-tempest-plugin repository ..."
cross_check ${TMP_DIR}/manila-tempest-plugin > ${TMP_DIR}/manila_tests.txt
echo "Tempest tests:"
echo "--------------"
sort ${TMP_DIR}/tempest_tests.txt
echo ""
echo "Designate-tempest-plugin tests:"
echo "-------------------------------"
sort ${TMP_DIR}/designate_tests.txt
echo ""
echo "Heat-tempest-plugin tests:"
echo "--------------------------"
sort ${TMP_DIR}/heat_tests.txt
echo ""
echo "Manila-tempest-plugin tests:"
echo "----------------------------"
sort ${TMP_DIR}/manila_tests.txt
# Cleanup
rm -rf ${TMP_DIR}
View Git Blame Copy Permalink