From 91d463b9b54ed777738350b0540d0502bfb5ee20 Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Fri, 3 Nov 2017 18:28:47 +0000 Subject: [PATCH] Used cached git sources and enable depends-on Many, many patches have gone into the master branch to enable the use of depends-on for role tests and to make use of the git clones put into place by zuul. This patch ensures that this functionality is now available for this branch too. The primary benefit for doing this would be to make use of the cached git clones provided by zuul to improve test reliability. The secondary benefit would be to have the ability to use depends-on to test patches together. However, this will come at the cost of requiring a tox configuration change, and the implementation of the tests/tests-repo-clone.sh and run_tests.sh scripts across all repositories. Change-Id: Ief9e0ace5a31dc44fb3fce2cf509606181ecb232 Depends-On: I2abda86df19980c963e3cb176db2f877ba9a312b --- run_tests.sh | 26 ++++------ tests/tests-repo-clone.sh | 103 ++++++++++++++++++++++++++++++++++++++ tox.ini | 39 ++++++--------- 3 files changed, 128 insertions(+), 40 deletions(-) create mode 100755 tests/tests-repo-clone.sh diff --git a/run_tests.sh b/run_tests.sh index 0da3d1d8..d9dacfb8 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -35,8 +35,12 @@ source /etc/os-release || source /usr/lib/os-release install_pkg_deps() { pkg_deps="git" + # Prefer dnf over yum for CentOS. + which dnf &>/dev/null && RHT_PKG_MGR='dnf' || RHT_PKG_MGR='yum' + case ${ID,,} in - centos|rhel) pkg_mgr_cmd="yum install -y" ;; + *suse*) pkg_mgr_cmd="zypper -n in" ;; + centos|rhel) pkg_mgr_cmd="${RHT_PKG_MGR} install -y" ;; fedora) pkg_mgr_cmd="dnf -y install" ;; ubuntu|debian) pkg_mgr_cmd="apt-get install -y" ;; *) echo "unsupported distribution: ${ID,,}"; exit 1 ;; @@ -45,24 +49,12 @@ install_pkg_deps() { eval sudo $pkg_mgr_cmd $pkg_deps } -git_clone_repo() { - if [[ ! -d tests/common ]]; then - # The tests repo doesn't need a clone, we can just - # symlink it. - if [[ "$(basename ${WORKING_DIR})" == "openstack-ansible-tests" ]]; then - ln -s ${WORKING_DIR} ${WORKING_DIR}/tests/common - else - git clone -b stable/newton \ - https://git.openstack.org/openstack/openstack-ansible-tests \ - tests/common - fi - fi -} - +# Install the host distro package dependencies install_pkg_deps -git_clone_repo +# Clone the tests repo for access to the common test script +source tests/tests-repo-clone.sh -# start executing the main test script +# Execute the common test script source tests/common/run_tests_common.sh diff --git a/tests/tests-repo-clone.sh b/tests/tests-repo-clone.sh new file mode 100755 index 00000000..9e3bd8ea --- /dev/null +++ b/tests/tests-repo-clone.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# Copyright 2017, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# PURPOSE: +# This script clones the openstack-ansible-tests repository to the +# tests/common folder in order to be able to re-use test components +# for role testing. + +# WARNING: +# This file is maintained in the openstack-ansible-tests repository: +# https://git.openstack.org/cgit/openstack/openstack-ansible-tests +# If you need to change this script, then propose the change there. +# Once it merges, the change will be replicated to the other repositories. + +## Shell Opts ---------------------------------------------------------------- + +set -e + +## Vars ---------------------------------------------------------------------- + +export TESTING_HOME=${TESTING_HOME:-$HOME} +export WORKING_DIR=${WORKING_DIR:-$(pwd)} +export CLONE_UPGRADE_TESTS=${CLONE_UPGRADE_TESTS:-no} +export ZUUL_TESTS_CLONE_LOCATION="/home/zuul/src/git.openstack.org/openstack/openstack-ansible-tests" + +## Functions ----------------------------------------------------------------- + +function create_tests_clonemap { + +# Prepare the clonemap for zuul-cloner to use +cat > ${TESTING_HOME}/tests-clonemap.yaml << EOF +clonemap: + - name: openstack/openstack-ansible-tests + dest: ${WORKING_DIR}/tests/common +EOF + +} + +## Main ---------------------------------------------------------------------- + +# If zuul-cloner is present, use it so that we +# also include any dependent patches from the +# tests repo noted in the commit message. +# We only want to use zuul-cloner if we detect +# zuul v2 running, so we check for the presence +# of the ZUUL_REF environment variable. +# ref: http://git.openstack.org/cgit/openstack-infra/zuul/tree/zuul/ansible/filter/zuul_filters.py?h=feature/zuulv3#n17 +if [[ -x /usr/zuul-env/bin/zuul-cloner ]] && [[ "${ZUUL_REF:-none}" != "none" ]]; then + + # Prepare the clonemap for zuul-cloner to use + create_tests_clonemap + + # Execute the clone + /usr/zuul-env/bin/zuul-cloner \ + --cache-dir /opt/git \ + --map ${TESTING_HOME}/tests-clonemap.yaml \ + git://git.openstack.org \ + openstack/openstack-ansible-tests + + # Clean up the clonemap. + rm -f ${TESTING_HOME}/tests-clonemap.yaml + +# Alternatively, use a simple git-clone. We do +# not re-clone if the directory exists already +# to prevent overwriting any local changes which +# may have been made. +elif [[ ! -d tests/common ]]; then + + # The tests repo doesn't need a clone, we can just + # symlink it. As zuul v3 clones into a folder called + # 'workspace' we have to use one of its environment + # variables to determine the project name. + if [[ "${ZUUL_SHORT_PROJECT_NAME:-none}" == "openstack-ansible-tests" ]] ||\ + [[ "$(basename ${WORKING_DIR})" == "openstack-ansible-tests" ]]; then + ln -s ${WORKING_DIR} ${WORKING_DIR}/tests/common + + # In zuul v3 any dependent repository is placed into + # /home/zuul/src/git.openstack.org, so we check to see + # if there is a tests checkout there already. If so, we + # symlink that and use it. + elif [[ -d "${ZUUL_TESTS_CLONE_LOCATION}" ]]; then + ln -s "${ZUUL_TESTS_CLONE_LOCATION}" ${WORKING_DIR}/tests/common + + # Otherwise we're clearly not in zuul or using a previously setup + # repo in some way, so just clone it from upstream. + else + git clone -b stable/newton \ + https://git.openstack.org/openstack/openstack-ansible-tests \ + ${WORKING_DIR}/tests/common + fi +fi diff --git a/tox.ini b/tox.ini index 6b618254..9da81fe6 100644 --- a/tox.ini +++ b/tox.ini @@ -53,16 +53,9 @@ commands = {posargs} -[testenv:tests_clone] -commands = - bash -c "if [ ! -d "{toxinidir}/tests/common" ]; then \ - git clone -b stable/newton https://git.openstack.org/openstack/openstack-ansible-tests {toxinidir}/tests/common; \ - fi" - - [testenv:pep8] commands = - {[testenv:tests_clone]commands} + bash -c "{toxinidir}/tests/tests-repo-clone.sh" bash -c "{toxinidir}/tests/common/test-pep8.sh" @@ -70,19 +63,28 @@ commands = # Ignores the following rules due to how ansible modules work in general # F403 'from ansible.module_utils.basic import *' used; # unable to detect undefined names -# H303 No wildcard (*) import. -ignore=F403,H303 +ignore=F403 [testenv:bashate] commands = - {[testenv:tests_clone]commands} + bash -c "{toxinidir}/tests/tests-repo-clone.sh" bash -c "{toxinidir}/tests/common/test-bashate.sh" +# The deps URL should be set to the appropriate git URL. +# In the tests repo itself, the variable is uniquely set to +# the toxinidir so that the role is able to test itself, but +# the tox config is exactly the same as other repositories. +# +# The value for other repositories must be: +# http://git.openstack.org/cgit/openstack/openstack-ansible-tests/plain/test-ansible-deps.txt +# or for a stable branch: +# http://git.openstack.org/cgit/openstack/openstack-ansible-tests/plain/test-ansible-deps.txt?h=stable/newton [testenv:ansible] deps = {[testenv]deps} + -rhttps://git.openstack.org/cgit/openstack/openstack-ansible/plain/global-requirement-pins.txt?h=stable/newton -rhttp://git.openstack.org/cgit/openstack/openstack-ansible-tests/plain/test-ansible-deps.txt?h=stable/newton @@ -90,7 +92,7 @@ deps = deps = {[testenv:ansible]deps} commands = - {[testenv:tests_clone]commands} + bash -c "{toxinidir}/tests/tests-repo-clone.sh" bash -c "{toxinidir}/tests/common/test-ansible-syntax.sh" @@ -98,7 +100,7 @@ commands = deps = {[testenv:ansible]deps} commands = - {[testenv:tests_clone]commands} + bash -c "{toxinidir}/tests/tests-repo-clone.sh" bash -c "{toxinidir}/tests/common/test-ansible-lint.sh" @@ -106,18 +108,9 @@ commands = deps = {[testenv:ansible]deps} commands = - {[testenv:tests_clone]commands} + bash -c "{toxinidir}/tests/tests-repo-clone.sh" bash -c "{toxinidir}/tests/common/test-ansible-functional.sh" -[testenv:ssl] -deps = - {[testenv:ansible]deps} -setenv = - {[testenv]setenv} - ANSIBLE_PARAMETERS=-vvv -e galera_use_ssl=True -e galera_user_ssl_ca_cert="/tmp/self-signed-certs/galera-ca.crt" -e galera_user_ssl_cert="/tmp/self-signed-certs/galera.crt" -e galera_user_ssl_key="/tmp/self-signed-certs/galera.key" -commands = - {[testenv:tests_clone]commands} - bash -c "{toxinidir}/tests/common/test-ansible-functional.sh" [testenv:linters] deps =