diff --git a/run_tests.sh b/run_tests.sh index f271015f..c2d2cb21 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -53,7 +53,6 @@ function usage { # root=`pwd -P` venv=$root/.venv -venv_env_version=$venv/environments with_venv=tools/with_venv.sh included_dirs="sahara_dashboard" @@ -85,6 +84,14 @@ check_only=0 pseudo=0 manage=0 +# NOTE(tonyb): the release team will automatically update tox.ini to point at +# the correct requirements branch when creating stable/* from master. So go to +# a little effort to get the deault from there to avoid dift and having to +# update this when branching +_default_uc=$(sed -n 's/^.*{env:UPPER_CONSTRAINTS_FILE\:\([^}]*\)}.*$/\1/p' \ + tox.ini | head -n1) + + # Jenkins sets a "JOB_NAME" variable, if it's not set, we'll make it "default" [ "$JOB_NAME" ] || JOB_NAME="default" @@ -211,39 +218,6 @@ function destroy_venv { echo "Virtualenv removed." } -function environment_check { - echo "Checking environment." - if [ -f $venv_env_version ]; then - set +o errexit - cat requirements.txt test-requirements.txt | cmp $venv_env_version - > /dev/null - local env_check_result=$? - set -o errexit - if [ $env_check_result -eq 0 ]; then - # If the environment exists and is up-to-date then set our variables - command_wrapper="${root}/${with_venv}" - echo "Environment is up to date." - return 0 - fi - fi - - if [ $always_venv -eq 1 ]; then - install_venv - else - if [ ! -e ${venv} ]; then - echo -e "Environment not found. Install? (Y/n) \c" - else - echo -e "Your environment appears to be out of date. Update? (Y/n) \c" - fi - read update_env - if [ "x$update_env" = "xY" -o "x$update_env" = "x" -o "x$update_env" = "xy" ]; then - install_venv - else - # Set our command wrapper anyway. - command_wrapper="${root}/${with_venv}" - fi - fi -} - function sanity_check { # Anything that should be determined prior to running the tests, server, etc. # Don't sanity-check anything environment-related in -N flag is set @@ -293,6 +267,7 @@ function restore_environment { function install_venv { # Install with install_venv.py + export UPPER_CONSTRAINTS_FILE=${UPPER_CONSTRAINTS_FILE:-$_default_uc} export PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE-/tmp/.pip_download_cache} export PIP_USE_MIRRORS=true if [ $quiet -eq 1 ]; then @@ -305,7 +280,6 @@ function install_venv { # Make sure it worked and record the environment version sanity_check chmod -R 754 $venv - cat requirements.txt test-requirements.txt > $venv_env_version } function run_tests { @@ -473,8 +447,8 @@ if [ $never_venv -eq 0 ]; then destroy_venv fi - # Then check if it's up-to-date - environment_check + # Create or update venv. + install_venv # Create a backup of the up-to-date environment if desired if [ $backup_env -eq 1 ]; then diff --git a/tools/install_venv.py b/tools/install_venv.py index bcb1f93d..45b6a410 100644 --- a/tools/install_venv.py +++ b/tools/install_venv.py @@ -32,7 +32,7 @@ VENV = os.path.join(ROOT, '.venv') WITH_VENV = os.path.join(ROOT, 'tools', 'with_venv.sh') PIP_REQUIRES = os.path.join(ROOT, 'requirements.txt') TEST_REQUIRES = os.path.join(ROOT, 'test-requirements.txt') -PIP_INSTALL_WRAPPER = os.path.join(ROOT, 'tools', 'pip_install.sh') +PIP_INSTALL_WRAPPER = os.path.join(ROOT, 'tools', 'tox_install.sh') def die(message, *args): @@ -110,12 +110,15 @@ def create_virtualenv(venv=VENV): def pip_install(*args): - args = [WITH_VENV, 'pip', 'install', '--upgrade'] + list(args) + constraints_url = os.environ.get('UPPER_CONSTRAINTS_FILE') + args = [WITH_VENV, 'pip', 'install', '-c', constraints_url, + '--upgrade'] + list(args) run_command(args, redirect_output=False) def pip_install_with_horizon(*args): - args = [WITH_VENV, PIP_INSTALL_WRAPPER, 'unconstrained'] + list(args) + constraints_url = os.environ.get('UPPER_CONSTRAINTS_FILE') + args = [WITH_VENV, PIP_INSTALL_WRAPPER, constraints_url] + list(args) run_command(args, redirect_output=False) diff --git a/tools/pip_install.sh b/tools/pip_install.sh deleted file mode 100755 index ad085f40..00000000 --- a/tools/pip_install.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/sh - -# The original script is borrowed from neutron-* repos. - -# Many of horizon's repos suffer from the problem of depending on horizon, -# but it not existing on pypi. - -# This wrapper for tox's package installer will use the existing package -# if it exists, else use zuul-cloner if that program exists, else grab it -# from horizon master via a hard-coded URL. That last case should only -# happen with devs running unit tests locally. - -# From the tox.ini config page: -# install_command=ARGV -# default: -# pip install {opts} {packages} - -ZUUL_CLONER=/usr/zuul-env/bin/zuul-cloner -BRANCH_NAME=master -horizon_installed=$(echo "import horizon" | python 2>/dev/null ; echo $?) - -set -e - -zuul_install () { - if [ -x "$ZUUL_CLONER" ]; then - export ZUUL_BRANCH=${ZUUL_BRANCH-$BRANCH} - echo "ZUUL CLONER" > /tmp/tox_install.txt - cwd=$(/bin/pwd) - cd /tmp - $ZUUL_CLONER --cache-dir \ - /opt/git \ - --branch $BRANCH_NAME \ - git://git.openstack.org \ - openstack/horizon - cd openstack/horizon - $install_cmd -e . - cd "$cwd" - else - return 1 - fi -} - -usual_install () { - echo "PIP HARDCODE" > /tmp/tox_install.txt - if [ -z "$HORIZON_PIP_LOCATION" ]; then - HORIZON_PIP_LOCATION="git+https://git.openstack.org/openstack/horizon@$BRANCH_NAME#egg=horizon" - fi - $install_cmd -U -e ${HORIZON_PIP_LOCATION} -} - -install_cmd="pip install" -if [ "$1" = "constrained" ]; then - install_cmd="$install_cmd $2" - shift -fi -shift - -if [ $horizon_installed -eq 0 ]; then - echo "ALREADY INSTALLED" > /tmp/tox_install.txt - echo "Horizon already installed; using existing package" -else - zuul_install || usual_install -fi - -$install_cmd -U $* -exit $? diff --git a/tools/tox_install.sh b/tools/tox_install.sh new file mode 100755 index 00000000..20759240 --- /dev/null +++ b/tools/tox_install.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +# Client constraint file contains this client version pin that is in conflict +# with installing the client from source. We should remove the version pin in +# the constraints file before applying it for from-source installation. +# The script also has a secondary purpose to install certain special +# dependencies directly from git. + +# Wrapper for pip install that always uses constraints. +function pip_install() { + pip install -c"$localfile" -U "$@" +} + +# Grab the library from git using either zuul-cloner or pip. The former is +# there to a take advantage of the setup done by the gate infrastructure +# and honour any/all Depends-On headers in the commit message +function install_from_git() { + ZUUL_CLONER=/usr/zuul-env/bin/zuul-cloner + GIT_HOST=git.openstack.org + PROJ=$1 + EGG=$2 + + edit-constraints "$localfile" -- "$EGG" + if [ -x "$ZUUL_CLONER" ]; then + SRC_DIR="$VIRTUAL_ENV/src" + mkdir -p "$SRC_DIR" + cd "$SRC_DIR" >/dev/null + ZUUL_CACHE_DIR=${ZUUL_CACHE_DIR:-/opt/git} $ZUUL_CLONER \ + --branch "$BRANCH_NAME" \ + "git://$GIT_HOST" "$PROJ" + pip_install -e "$PROJ/." + cd - >/dev/null + else + pip_install -e"git+https://$GIT_HOST/$PROJ@$BRANCH_NAME#egg=${EGG}" + fi +} + + + +CONSTRAINTS_FILE="$1" +shift 1 + +# This script will either complete with a return code of 0 or the return code +# of whatever failed. +set -e + +# NOTE(tonyb): Place this in the tox environment's log dir so it will get +# published to logs.openstack.org for easy debugging. +mkdir -p "$VIRTUAL_ENV/log/" +localfile="$VIRTUAL_ENV/log/upper-constraints.txt" + +if [[ "$CONSTRAINTS_FILE" != http* ]]; then + CONSTRAINTS_FILE="file://$CONSTRAINTS_FILE" +fi +# NOTE(tonyb): need to add curl to bindep.txt if the project supports bindep +curl "$CONSTRAINTS_FILE" --insecure --progress-bar --output "$localfile" + +pip_install openstack-requirements + +# This is the main purpose of the script: Allow local installation of +# the current repo. It is listed in constraints file and thus any +# install will be constrained and we need to unconstrain it. +edit-constraints "$localfile" -- "$CLIENT_NAME" + +declare -a passthrough_args +while [ $# -gt 0 ] ; do + case "$1" in + # If we have any special os: deps then process them + os:*) + declare -a pkg_spec + IFS=: pkg_spec=($1) + install_from_git "${pkg_spec[1]}" "${pkg_spec[2]}" + ;; + # Otherwise just pass the other deps through to the constrained pip install + *) + passthrough_args+=("$1") + ;; + esac + shift 1 +done + +# If *only* had special args then then isn't any need to run pip. +if [ -n "$passthrough_args" ] ; then + pip_install "${passthrough_args[@]}" +fi diff --git a/tools/with_venv.sh b/tools/with_venv.sh index 7303990b..15e4cc06 100755 --- a/tools/with_venv.sh +++ b/tools/with_venv.sh @@ -4,4 +4,7 @@ VENV_PATH=${VENV_PATH:-${TOOLS_PATH}} VENV_DIR=${VENV_NAME:-/../.venv} TOOLS=${TOOLS_PATH} VENV=${VENV:-${VENV_PATH}/${VENV_DIR}} +# tools/tox_install.sh looks for VIRTUAL_ENV evrythign else looks for VENV +# so for now make them the same. +VIRTUAL_ENV=${VENV:-${VENV_PATH}/${VENV_DIR}} source ${VENV}/bin/activate && "$@" diff --git a/tox.ini b/tox.ini index bf42dd02..ac2db82f 100644 --- a/tox.ini +++ b/tox.ini @@ -5,11 +5,14 @@ skipsdist = True [testenv] usedevelop = True -install_command = {toxinidir}/tools/pip_install.sh unconstrained {opts} {packages} +install_command = {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages} setenv = VIRTUAL_ENV={envdir} + BRANCH_NAME=master + CLIENT_NAME=sahara-dashboard deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt + os:openstack/horizon:horizon whitelist_externals = /bin/bash commands = /bin/bash run_tests.sh -N --no-pep8 {posargs}