From 0a4ac2a1c797122069e2cbaa514c3e1de4aa008e Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Mon, 26 Mar 2018 13:52:26 +0100 Subject: [PATCH] Have run_tests clone the test repo Current run_tests.sh executes another script (tests-repo-clone.sh) to clone the tests repo. That script therefore has to be replicated out to all the other repositories too. In order to try to reduce the footprint of things that need to be replicated out to the repositories, we fold the tests repo clone into run_tests.sh and move the clone of the previous tests repo into the common test script. We also remove the conditional for the previous test repo clone as it does no harm to always have it. To reduce the maintenance when creating new branches, we key the branch to be cloned by run_tests.sh to the content in the .gitreview file. This way as soon as a branch is created and that file is updated, the right branch is cloned and the system becomes self maintaining. Some extra comments are added to explain the purpose of variables/ commands and the install_pkg_deps function is changed to just be in line script as the function is no re-used anywhere else. We change the set vars in the common script to add x and remove v to reduce the output verbosity and only output the useful stuff. Finally, the linters job definition is changed so that it uses run_tests.sh to execute the lint tests. This simplifies the job definition and also ensures that zuul runs tests in the same way that humans do. Change-Id: I9f26f4f438715ce6361cc9960f58f2d256c5a839 --- .gitignore | 1 + run_tests.sh | 73 ++++++++++++++++++++++++++++++++------------- run_tests_common.sh | 44 +++++++++++++++++++++++++-- tox.ini | 19 ------------ zuul.d/jobs.yaml | 62 ++++++++------------------------------ 5 files changed, 108 insertions(+), 91 deletions(-) diff --git a/.gitignore b/.gitignore index 5421f904..968d8b98 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,4 @@ tests/*.retry # Git clones openstack-ansible-ops +previous diff --git a/run_tests.sh b/run_tests.sh index 4b72d3fa..a31f9464 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -12,8 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Note: + +# PURPOSE: +# This script clones the openstack-ansible-tests repository to the +# tests/common folder in order to be able to re-use test components +# for role testing. This is intended to be the thinnest possible +# shim for test execution outside of OpenStack CI. + +# WARNING: # This file is maintained in the openstack-ansible-tests repository. # https://git.openstack.org/cgit/openstack/openstack-ansible-tests/tree/run_tests.sh # If you need to modify this file, update the one in the openstack-ansible-tests @@ -26,33 +32,60 @@ set -xeu ## Vars ---------------------------------------------------------------------- -export WORKING_DIR=${WORKING_DIR:-$(pwd)} +WORKING_DIR="$(readlink -f $(dirname $0))" + +COMMON_TESTS_PATH="${WORKING_DIR}/tests/common" +TESTING_HOME=${TESTING_HOME:-$HOME} +ZUUL_TESTS_CLONE_LOCATION="/home/zuul/src/git.openstack.org/openstack/openstack-ansible-tests" + +# Use .gitreview as the key to determine the appropriate +# branch to clone for tests. +TESTING_BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' "${WORKING_DIR}/.gitreview") +if [[ "${TESTING_BRANCH}" == "" ]]; then + TESTING_BRANCH="master" +fi ## Main ---------------------------------------------------------------------- +# Source distribution information source /etc/os-release || source /usr/lib/os-release -install_pkg_deps() { - pkg_deps="git" +# Prefer dnf over yum for CentOS. +which dnf &>/dev/null && RHT_PKG_MGR='dnf' || RHT_PKG_MGR='yum' - # Prefer dnf over yum for CentOS. - which dnf &>/dev/null && RHT_PKG_MGR='dnf' || RHT_PKG_MGR='yum' +# Figure out the appropriate package install command +case ${ID,,} in + *suse*) pkg_mgr_cmd="zypper -n in" ;; + centos|rhel|fedora) pkg_mgr_cmd="${RHT_PKG_MGR} install -y" ;; + ubuntu|debian) pkg_mgr_cmd="apt-get install -y" ;; + *) echo "unsupported distribution: ${ID,,}"; exit 1 ;; +esac - case ${ID,,} in - *suse*) pkg_mgr_cmd="zypper -n in" ;; - centos|rhel|fedora) pkg_mgr_cmd="${RHT_PKG_MGR} install -y" ;; - ubuntu|debian) pkg_mgr_cmd="apt-get install -y" ;; - *) echo "unsupported distribution: ${ID,,}"; exit 1 ;; - esac - - eval sudo $pkg_mgr_cmd $pkg_deps -} - -# Install the host distro package dependencies -install_pkg_deps +# Install git so that we can clone the tests repo +eval sudo $pkg_mgr_cmd git # Clone the tests repo for access to the common test script -source tests/tests-repo-clone.sh +if [[ ! -d ${COMMON_TESTS_PATH} ]]; then + # The tests repo doesn't need a clone, we can just + # symlink it. + if [[ "$(basename ${WORKING_DIR})" == "openstack-ansible-tests" ]]; then + ln -s ${WORKING_DIR} ${COMMON_TESTS_PATH} + + # In zuul v3 any dependent repository is placed into + # /home/zuul/src/git.openstack.org, so we check to see + # if there is a tests checkout there already. If so, we + # symlink that and use it. + elif [[ -d "${ZUUL_TESTS_CLONE_LOCATION}" ]]; then + ln -s "${ZUUL_TESTS_CLONE_LOCATION}" ${COMMON_TESTS_PATH} + + # Otherwise we're clearly not in zuul or using a previously setup + # repo in some way, so just clone it from upstream. + else + git clone -b ${TESTING_BRANCH} \ + https://git.openstack.org/openstack/openstack-ansible-tests \ + ${COMMON_TESTS_PATH} + fi +fi # Execute the common test script source tests/common/run_tests_common.sh diff --git a/run_tests_common.sh b/run_tests_common.sh index 2470206a..cb536d44 100755 --- a/run_tests_common.sh +++ b/run_tests_common.sh @@ -13,16 +13,56 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -o pipefail -set -euov +## Shell Opts ---------------------------------------------------------------- +set -o pipefail +set -xeuo + +## Prerequisite check -------------------------------------------------------- + +# Check whether the require environment variables are set. +if [[ -z ${WORKING_DIR+x} ]] ||\ + [[ -z ${COMMON_TESTS_PATH+x} ]] ||\ + [[ -z ${TESTING_HOME+x} ]] ||\ + [[ -z ${TESTING_BRANCH+x} ]]; then + echo "Required environment variables are not set." + echo "Please ensure that run_tests.sh is used to execute tests." + exit 1 +fi + +## Vars ---------------------------------------------------------------------- + +# Set the source branch for upgrade tests +# Be sure to change this whenever a new stable branch +# is created. The checkout must always be N-1. +UPGRADE_SOURCE_BRANCH=${UPGRADE_SOURCE_BRANCH:-'stable/queens'} + +# The bindep file contains the basic distribution packages +# required in order to install pip, and ansible via pip. BINDEP_FILE=${BINDEP_FILE:-bindep.txt} +## Main ---------------------------------------------------------------------- + +# If this test set includes an upgrade test, the +# previous stable release tests repo must also be +# cloned. +# Note: +# Dependent patches to the previous stable release +# tests repo are not supported. +if [[ ! -d "${COMMON_TESTS_PATH}/previous" ]]; then + git clone -b ${UPGRADE_SOURCE_BRANCH} \ + https://git.openstack.org/openstack/openstack-ansible-tests \ + ${COMMON_TESTS_PATH}/previous +fi + +# Source distribution information source /etc/os-release || source /usr/lib/os-release # Prefer dnf over yum for CentOS. which dnf &>/dev/null && RHT_PKG_MGR='dnf' || RHT_PKG_MGR='yum' +# Perform the initial distribution package install +# to allow pip and bindep to work. case "${ID,,}" in *suse*) # Need to pull libffi and python-pyOpenSSL early diff --git a/tox.ini b/tox.ini index 29c6ec2b..1dd00c8f 100644 --- a/tox.ini +++ b/tox.ini @@ -56,7 +56,6 @@ commands = [testenv:pep8] commands = - bash -c "{toxinidir}/tests/tests-repo-clone.sh" bash -c "{toxinidir}/tests/common/test-pep8.sh" @@ -69,7 +68,6 @@ ignore=F403 [testenv:bashate] commands = - bash -c "{toxinidir}/tests/tests-repo-clone.sh" bash -c "{toxinidir}/tests/common/test-bashate.sh" @@ -93,7 +91,6 @@ deps = deps = {[testenv:ansible]deps} commands = - bash -c "{toxinidir}/tests/tests-repo-clone.sh" bash -c "{toxinidir}/tests/common/test-ansible-syntax.sh" @@ -101,29 +98,13 @@ commands = deps = {[testenv:ansible]deps} commands = - bash -c "{toxinidir}/tests/tests-repo-clone.sh" bash -c "{toxinidir}/tests/common/test-ansible-lint.sh" -# This is here due to a bug in the ubuntu-xenial -# job which does not do the translation from -# 'func' to 'functional' for us. -# TODO(odyssey4me): -# Remove this once https://review.openstack.org/512351 -# has been merged. -[testenv:func] -deps = - {[testenv:ansible]deps} -commands = - bash -c "{toxinidir}/tests/tests-repo-clone.sh" - bash -c "{toxinidir}/tests/common/test-ansible-functional.sh" - - [testenv:functional] deps = {[testenv:ansible]deps} commands = - bash -c "{toxinidir}/tests/tests-repo-clone.sh" bash -c "{toxinidir}/tests/common/test-ansible-functional.sh" diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 186960fa..62777261 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -13,56 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -- job: - name: openstack-ansible-linters - parent: openstack-tox-linters - description: | - Run lint tests for an OpenStack-Ansible project. - Uses tox with the ``linters`` environment. - required-projects: - - name: openstack/ansible-hardening - - name: openstack/ansible-role-python_venv_build - - name: openstack/ansible-role-systemd_mount - - name: openstack/ansible-role-systemd_networkd - - name: openstack/ansible-role-systemd_service - - name: openstack/openstack-ansible-apt_package_pinning - - name: openstack/openstack-ansible-ceph_client - - name: openstack/openstack-ansible-galera_client - - name: openstack/openstack-ansible-galera_server - - name: openstack/openstack-ansible-haproxy_server - - name: openstack/openstack-ansible-lxc_container_create - - name: openstack/openstack-ansible-lxc_hosts - - name: openstack/openstack-ansible-memcached_server - - name: openstack/openstack-ansible-openstack_hosts - - name: openstack/openstack-ansible-openstack_openrc - - name: openstack/openstack-ansible-ops - - name: openstack/openstack-ansible-os_aodh - - name: openstack/openstack-ansible-os_barbican - - name: openstack/openstack-ansible-os_ceilometer - - name: openstack/openstack-ansible-os_cinder - - name: openstack/openstack-ansible-os_designate - - name: openstack/openstack-ansible-os_glance - - name: openstack/openstack-ansible-os_gnocchi - - name: openstack/openstack-ansible-os_heat - - name: openstack/openstack-ansible-os_keystone - - name: openstack/openstack-ansible-os_magnum - - name: openstack/openstack-ansible-os_neutron - - name: openstack/openstack-ansible-os_nova - - name: openstack/openstack-ansible-os_octavia - - name: openstack/openstack-ansible-os_swift - - name: openstack/openstack-ansible-os_tempest - - name: openstack/openstack-ansible-os_trove - - name: openstack/openstack-ansible-pip_install - - name: openstack/openstack-ansible-plugins - - name: openstack/openstack-ansible-rabbitmq_server - - name: openstack/openstack-ansible-repo_build - - name: openstack/openstack-ansible-repo_server - - name: openstack/openstack-ansible-rsyslog_client - - name: openstack/openstack-ansible-rsyslog_server - - name: openstack/openstack-ansible-tests - - name: openstack/requirements - timeout: 600 # 5 mins - - job: name: openstack-ansible-functional parent: base @@ -127,6 +77,18 @@ vars: tox_env: functional +- job: + name: openstack-ansible-linters + parent: openstack-ansible-functional + description: | + Run lint tests for an OpenStack-Ansible project. + Uses tox with the ``linters`` environment. + irrelevant-files: [] + timeout: 600 # 5 mins + nodeset: ubuntu-xenial + vars: + tox_env: linters + - job: name: openstack-ansible-upgrade parent: openstack-ansible-functional