Have run_tests clone the test repo

Current run_tests.sh executes another script (tests-repo-clone.sh)
to clone the tests repo. That script therefore has to be replicated
out to all the other repositories too.

In order to try to reduce the footprint of things that need to be
replicated out to the repositories, we fold the tests repo clone
into run_tests.sh and move the clone of the previous tests repo
into the common test script. We also remove the conditional for
the previous test repo clone as it does no harm to always have it.

To reduce the maintenance when creating new branches, we key the
branch to be cloned by run_tests.sh to the content in the .gitreview
file. This way as soon as a branch is created and that file is
updated, the right branch is cloned and the system becomes self
maintaining.

Some extra comments are added to explain the purpose of variables/
commands and the install_pkg_deps function is changed to just be
in line script as the function is no re-used anywhere else.

We change the set vars in the common script to add x and remove v
to reduce the output verbosity and only output the useful stuff.

Finally, the linters job definition is changed so that it uses
run_tests.sh to execute the lint tests. This simplifies the job
definition and also ensures that zuul runs tests in the same way
that humans do.

Change-Id: I9f26f4f438715ce6361cc9960f58f2d256c5a839

.gitignore

@@ -69,3 +69,4 @@ tests/*.retry
 
 # Git clones
 openstack-ansible-ops
+previous

run_tests.sh

@@ -12,8 +12,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-# Note:
+
+# PURPOSE:
+# This script clones the openstack-ansible-tests repository to the
+# tests/common folder in order to be able to re-use test components
+# for role testing. This is intended to be the thinnest possible
+# shim for test execution outside of OpenStack CI.
+
+# WARNING:
 # This file is maintained in the openstack-ansible-tests repository.
 # https://git.openstack.org/cgit/openstack/openstack-ansible-tests/tree/run_tests.sh
 # If you need to modify this file, update the one in the openstack-ansible-tests
@@ -26,33 +32,60 @@ set -xeu
 
 ## Vars ----------------------------------------------------------------------
 
-export WORKING_DIR=${WORKING_DIR:-$(pwd)}
+WORKING_DIR="$(readlink -f $(dirname $0))"
+
+COMMON_TESTS_PATH="${WORKING_DIR}/tests/common"
+TESTING_HOME=${TESTING_HOME:-$HOME}
+ZUUL_TESTS_CLONE_LOCATION="/home/zuul/src/git.openstack.org/openstack/openstack-ansible-tests"
+
+# Use .gitreview as the key to determine the appropriate
+# branch to clone for tests.
+TESTING_BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' "${WORKING_DIR}/.gitreview")
+if [[ "${TESTING_BRANCH}" == "" ]]; then
+  TESTING_BRANCH="master"
+fi
 
 ## Main ----------------------------------------------------------------------
 
+# Source distribution information
 source /etc/os-release || source /usr/lib/os-release
 
-install_pkg_deps() {
-    pkg_deps="git"
+# Prefer dnf over yum for CentOS.
+which dnf &>/dev/null && RHT_PKG_MGR='dnf' || RHT_PKG_MGR='yum'
 
-    # Prefer dnf over yum for CentOS.
-    which dnf &>/dev/null && RHT_PKG_MGR='dnf' || RHT_PKG_MGR='yum'
+# Figure out the appropriate package install command
+case ${ID,,} in
+    *suse*) pkg_mgr_cmd="zypper -n in" ;;
+    centos|rhel|fedora) pkg_mgr_cmd="${RHT_PKG_MGR} install -y" ;;
+    ubuntu|debian) pkg_mgr_cmd="apt-get install -y" ;;
+    *) echo "unsupported distribution: ${ID,,}"; exit 1 ;;
+esac
 
-    case ${ID,,} in
-        *suse*) pkg_mgr_cmd="zypper -n in" ;;
-        centos|rhel|fedora) pkg_mgr_cmd="${RHT_PKG_MGR} install -y" ;;
-        ubuntu|debian) pkg_mgr_cmd="apt-get install -y" ;;
-        *) echo "unsupported distribution: ${ID,,}"; exit 1 ;;
-    esac
-
-    eval sudo $pkg_mgr_cmd $pkg_deps
-}
-
-# Install the host distro package dependencies
-install_pkg_deps
+# Install git so that we can clone the tests repo
+eval sudo $pkg_mgr_cmd git
 
 # Clone the tests repo for access to the common test script
-source tests/tests-repo-clone.sh
+if [[ ! -d ${COMMON_TESTS_PATH} ]]; then
+    # The tests repo doesn't need a clone, we can just
+    # symlink it.
+    if [[ "$(basename ${WORKING_DIR})" == "openstack-ansible-tests" ]]; then
+        ln -s ${WORKING_DIR} ${COMMON_TESTS_PATH}
+
+    # In zuul v3 any dependent repository is placed into
+    # /home/zuul/src/git.openstack.org, so we check to see
+    # if there is a tests checkout there already. If so, we
+    # symlink that and use it.
+    elif [[ -d "${ZUUL_TESTS_CLONE_LOCATION}" ]]; then
+        ln -s "${ZUUL_TESTS_CLONE_LOCATION}" ${COMMON_TESTS_PATH}
+
+    # Otherwise we're clearly not in zuul or using a previously setup
+    # repo in some way, so just clone it from upstream.
+    else
+        git clone -b ${TESTING_BRANCH} \
+            https://git.openstack.org/openstack/openstack-ansible-tests \
+            ${COMMON_TESTS_PATH}
+    fi
+fi
 
 # Execute the common test script
 source tests/common/run_tests_common.sh

run_tests_common.sh

@@ -13,16 +13,56 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -o pipefail
-set -euov
+## Shell Opts ----------------------------------------------------------------
 
+set -o pipefail
+set -xeuo
+
+## Prerequisite check --------------------------------------------------------
+
+# Check whether the require environment variables are set.
+if [[ -z ${WORKING_DIR+x} ]] ||\
+   [[ -z ${COMMON_TESTS_PATH+x} ]] ||\
+   [[ -z ${TESTING_HOME+x} ]] ||\
+   [[ -z ${TESTING_BRANCH+x} ]]; then
+     echo "Required environment variables are not set."
+     echo "Please ensure that run_tests.sh is used to execute tests."
+     exit 1
+fi
+
+## Vars ----------------------------------------------------------------------
+
+# Set the source branch for upgrade tests
+# Be sure to change this whenever a new stable branch
+# is created. The checkout must always be N-1.
+UPGRADE_SOURCE_BRANCH=${UPGRADE_SOURCE_BRANCH:-'stable/queens'}
+
+# The bindep file contains the basic distribution packages
+# required in order to install pip, and ansible via pip.
 BINDEP_FILE=${BINDEP_FILE:-bindep.txt}
 
+## Main ----------------------------------------------------------------------
+
+# If this test set includes an upgrade test, the
+# previous stable release tests repo must also be
+# cloned.
+# Note:
+# Dependent patches to the previous stable release
+# tests repo are not supported.
+if [[ ! -d "${COMMON_TESTS_PATH}/previous" ]]; then
+  git clone -b ${UPGRADE_SOURCE_BRANCH} \
+      https://git.openstack.org/openstack/openstack-ansible-tests \
+      ${COMMON_TESTS_PATH}/previous
+fi
+
+# Source distribution information
 source /etc/os-release || source /usr/lib/os-release
 
 # Prefer dnf over yum for CentOS.
 which dnf &>/dev/null && RHT_PKG_MGR='dnf' || RHT_PKG_MGR='yum'
 
+# Perform the initial distribution package install
+# to allow pip and bindep to work.
 case "${ID,,}" in
     *suse*)
         # Need to pull libffi and python-pyOpenSSL early

tox.ini

@@ -56,7 +56,6 @@ commands =
 
 [testenv:pep8]
 commands =
-    bash -c "{toxinidir}/tests/tests-repo-clone.sh"
     bash -c "{toxinidir}/tests/common/test-pep8.sh"
 
 
@@ -69,7 +68,6 @@ ignore=F403
 
 [testenv:bashate]
 commands =
-    bash -c "{toxinidir}/tests/tests-repo-clone.sh"
     bash -c "{toxinidir}/tests/common/test-bashate.sh"
 
 
@@ -93,7 +91,6 @@ deps =
 deps =
     {[testenv:ansible]deps}
 commands =
-    bash -c "{toxinidir}/tests/tests-repo-clone.sh"
     bash -c "{toxinidir}/tests/common/test-ansible-syntax.sh"
 
 
@@ -101,29 +98,13 @@ commands =
 deps =
     {[testenv:ansible]deps}
 commands =
-    bash -c "{toxinidir}/tests/tests-repo-clone.sh"
     bash -c "{toxinidir}/tests/common/test-ansible-lint.sh"
 
 
-# This is here due to a bug in the ubuntu-xenial
-# job which does not do the translation from
-# 'func' to 'functional' for us.
-# TODO(odyssey4me):
-# Remove this once https://review.openstack.org/512351
-# has been merged.
-[testenv:func]
-deps =
-    {[testenv:ansible]deps}
-commands =
-    bash -c "{toxinidir}/tests/tests-repo-clone.sh"
-    bash -c "{toxinidir}/tests/common/test-ansible-functional.sh"
-
-
 [testenv:functional]
 deps =
     {[testenv:ansible]deps}
 commands =
-    bash -c "{toxinidir}/tests/tests-repo-clone.sh"
     bash -c "{toxinidir}/tests/common/test-ansible-functional.sh"
 
 

zuul.d/jobs.yaml

@@ -13,56 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- job:
-    name: openstack-ansible-linters
-    parent: openstack-tox-linters
-    description: |
-      Run lint tests for an OpenStack-Ansible project.
-      Uses tox with the ``linters`` environment.
-    required-projects:
-      - name: openstack/ansible-hardening
-      - name: openstack/ansible-role-python_venv_build
-      - name: openstack/ansible-role-systemd_mount
-      - name: openstack/ansible-role-systemd_networkd
-      - name: openstack/ansible-role-systemd_service
-      - name: openstack/openstack-ansible-apt_package_pinning
-      - name: openstack/openstack-ansible-ceph_client
-      - name: openstack/openstack-ansible-galera_client
-      - name: openstack/openstack-ansible-galera_server
-      - name: openstack/openstack-ansible-haproxy_server
-      - name: openstack/openstack-ansible-lxc_container_create
-      - name: openstack/openstack-ansible-lxc_hosts
-      - name: openstack/openstack-ansible-memcached_server
-      - name: openstack/openstack-ansible-openstack_hosts
-      - name: openstack/openstack-ansible-openstack_openrc
-      - name: openstack/openstack-ansible-ops
-      - name: openstack/openstack-ansible-os_aodh
-      - name: openstack/openstack-ansible-os_barbican
-      - name: openstack/openstack-ansible-os_ceilometer
-      - name: openstack/openstack-ansible-os_cinder
-      - name: openstack/openstack-ansible-os_designate
-      - name: openstack/openstack-ansible-os_glance
-      - name: openstack/openstack-ansible-os_gnocchi
-      - name: openstack/openstack-ansible-os_heat
-      - name: openstack/openstack-ansible-os_keystone
-      - name: openstack/openstack-ansible-os_magnum
-      - name: openstack/openstack-ansible-os_neutron
-      - name: openstack/openstack-ansible-os_nova
-      - name: openstack/openstack-ansible-os_octavia
-      - name: openstack/openstack-ansible-os_swift
-      - name: openstack/openstack-ansible-os_tempest
-      - name: openstack/openstack-ansible-os_trove
-      - name: openstack/openstack-ansible-pip_install
-      - name: openstack/openstack-ansible-plugins
-      - name: openstack/openstack-ansible-rabbitmq_server
-      - name: openstack/openstack-ansible-repo_build
-      - name: openstack/openstack-ansible-repo_server
-      - name: openstack/openstack-ansible-rsyslog_client
-      - name: openstack/openstack-ansible-rsyslog_server
-      - name: openstack/openstack-ansible-tests
-      - name: openstack/requirements
-    timeout: 600 # 5 mins
-
 - job:
     name: openstack-ansible-functional
     parent: base
@@ -127,6 +77,18 @@
     vars:
       tox_env: functional
 
+- job:
+    name: openstack-ansible-linters
+    parent: openstack-ansible-functional
+    description: |
+      Run lint tests for an OpenStack-Ansible project.
+      Uses tox with the ``linters`` environment.
+    irrelevant-files: []
+    timeout: 600 # 5 mins
+    nodeset: ubuntu-xenial
+    vars:
+      tox_env: linters
+
 - job:
     name: openstack-ansible-upgrade
     parent: openstack-ansible-functional