From 73b44247f2100e7fe28274fadbe730f8504fbdb1 Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Wed, 22 Nov 2017 19:51:31 +0100 Subject: [PATCH] Align tox_install.sh with other projects The tox_install.sh here behaves a bit differently than the others, which is causing some problems when trying to rework some of the shared gate jobs. Align it to the form used in other repos. Change-Id: I93d14b522b2c55e8988c6586e4b5c32cbd151e56 --- tools/tox_install.sh | 124 ++++++++++++++++++------------------------- tox.ini | 3 +- 2 files changed, 53 insertions(+), 74 deletions(-) diff --git a/tools/tox_install.sh b/tools/tox_install.sh index 20759240..7890956f 100755 --- a/tools/tox_install.sh +++ b/tools/tox_install.sh @@ -1,85 +1,65 @@ #!/usr/bin/env bash -# Client constraint file contains this client version pin that is in conflict -# with installing the client from source. We should remove the version pin in -# the constraints file before applying it for from-source installation. -# The script also has a secondary purpose to install certain special -# dependencies directly from git. +# Many of neutron's repos suffer from the problem of depending on neutron, +# but it not existing on pypi. -# Wrapper for pip install that always uses constraints. -function pip_install() { - pip install -c"$localfile" -U "$@" -} +# This wrapper for tox's package installer will use the existing package +# if it exists, else use zuul-cloner if that program exists, else grab it +# from neutron master via a hard-coded URL. That last case should only +# happen with devs running unit tests locally. -# Grab the library from git using either zuul-cloner or pip. The former is -# there to a take advantage of the setup done by the gate infrastructure -# and honour any/all Depends-On headers in the commit message -function install_from_git() { - ZUUL_CLONER=/usr/zuul-env/bin/zuul-cloner - GIT_HOST=git.openstack.org - PROJ=$1 - EGG=$2 +# From the tox.ini config page: +# install_command=ARGV +# default: +# pip install {opts} {packages} - edit-constraints "$localfile" -- "$EGG" - if [ -x "$ZUUL_CLONER" ]; then - SRC_DIR="$VIRTUAL_ENV/src" - mkdir -p "$SRC_DIR" - cd "$SRC_DIR" >/dev/null - ZUUL_CACHE_DIR=${ZUUL_CACHE_DIR:-/opt/git} $ZUUL_CLONER \ - --branch "$BRANCH_NAME" \ - "git://$GIT_HOST" "$PROJ" - pip_install -e "$PROJ/." - cd - >/dev/null +ZUUL_CLONER=/usr/zuul-env/bin/zuul-cloner +BRANCH_NAME=master +GIT_BASE=${GIT_BASE:-https://git.openstack.org/} + +install_project() { + local project=$1 + local branch=${2:-$BRANCH_NAME} + local module_name=${project//-/_} + + set +e + project_installed=$(echo "import $module_name" | python 2>/dev/null ; echo $?) + set -e + + if [ $project_installed -eq 0 ]; then + echo "ALREADY INSTALLED" > /tmp/tox_install.txt + echo "$project already installed; using existing package" + elif [ -x "$ZUUL_CLONER" ]; then + echo "ZUUL CLONER" > /tmp/tox_install.txt + # Make this relative to current working directory so that + # git clean can remove it. We cannot remove the directory directly + # since it is referenced after $install_cmd -e + mkdir -p .tmp + PROJECT_DIR=$(/bin/mktemp -d -p $(pwd)/.tmp) + pushd $PROJECT_DIR + $ZUUL_CLONER --cache-dir \ + /opt/git \ + --branch $branch \ + http://git.openstack.org \ + openstack/$project + cd openstack/$project + $install_cmd -e . + popd else - pip_install -e"git+https://$GIT_HOST/$PROJ@$BRANCH_NAME#egg=${EGG}" + echo "PIP HARDCODE" > /tmp/tox_install.txt + local GIT_REPO="$GIT_BASE/openstack/$project" + SRC_DIR="$VIRTUAL_ENV/src/$project" + git clone --depth 1 --branch $branch $GIT_REPO $SRC_DIR + $install_cmd -U -e $SRC_DIR fi } - - -CONSTRAINTS_FILE="$1" -shift 1 - -# This script will either complete with a return code of 0 or the return code -# of whatever failed. set -e -# NOTE(tonyb): Place this in the tox environment's log dir so it will get -# published to logs.openstack.org for easy debugging. -mkdir -p "$VIRTUAL_ENV/log/" -localfile="$VIRTUAL_ENV/log/upper-constraints.txt" +install_cmd="pip install -c$1" +shift -if [[ "$CONSTRAINTS_FILE" != http* ]]; then - CONSTRAINTS_FILE="file://$CONSTRAINTS_FILE" -fi -# NOTE(tonyb): need to add curl to bindep.txt if the project supports bindep -curl "$CONSTRAINTS_FILE" --insecure --progress-bar --output "$localfile" +install_project horizon -pip_install openstack-requirements - -# This is the main purpose of the script: Allow local installation of -# the current repo. It is listed in constraints file and thus any -# install will be constrained and we need to unconstrain it. -edit-constraints "$localfile" -- "$CLIENT_NAME" - -declare -a passthrough_args -while [ $# -gt 0 ] ; do - case "$1" in - # If we have any special os: deps then process them - os:*) - declare -a pkg_spec - IFS=: pkg_spec=($1) - install_from_git "${pkg_spec[1]}" "${pkg_spec[2]}" - ;; - # Otherwise just pass the other deps through to the constrained pip install - *) - passthrough_args+=("$1") - ;; - esac - shift 1 -done - -# If *only* had special args then then isn't any need to run pip. -if [ -n "$passthrough_args" ] ; then - pip_install "${passthrough_args[@]}" -fi +$install_cmd -U $* +exit $? diff --git a/tox.ini b/tox.ini index e24e0a79..4f21c939 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,6 @@ setenv = CLIENT_NAME=sahara-dashboard deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt - os:openstack/horizon:horizon whitelist_externals = /bin/bash commands = /bin/bash run_tests.sh -N --no-pep8 {posargs} @@ -71,7 +70,7 @@ show-source = True ignore = E123,E125,H405 enable-extensions = H203,H106 builtins = _ -exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,.ropeproject,tools +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,.ropeproject,tools,.tmp [testenv:bindep] # Do not install any requirements. We want this to be fast and work even if