From d58427d3174f2fabb45fba69842d73133d24e12b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 26 Dec 2013 10:01:38 -0800 Subject: [PATCH] Switch to just using tox Instead of requiring all the complicated venv management logic that tox already provides just use tox directly instead. This maps more closely to what jenkins does, and how most other python development occurs (which does not typically rely on custom venv scripts). Change-Id: Iff6cf0a7eb24ddd2ef65ddd723d0e4cb803b9bb9 --- openstack-common.conf | 1 - run_tests.sh | 115 +++++++++-------------- tools/install_venv.py | 75 --------------- tools/install_venv_common.py | 172 ----------------------------------- tools/with_venv.sh | 5 - 5 files changed, 42 insertions(+), 326 deletions(-) delete mode 100644 tools/install_venv.py delete mode 100644 tools/install_venv_common.py delete mode 100755 tools/with_venv.sh diff --git a/openstack-common.conf b/openstack-common.conf index 9bac7d725..fd948cad2 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -3,7 +3,6 @@ # The list of modules to copy from oslo-incubator.git module=excutils module=importutils -module=install_venv_common module=jsonutils module=py3kcompat module=timeutils diff --git a/run_tests.sh b/run_tests.sh index f96cc107b..1689fb9c6 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -4,116 +4,85 @@ function usage { echo "Usage: $0 [OPTION]..." echo "Run Taskflow's test suite(s)" echo "" - echo " -V, --virtual-env Always use virtualenv. Install automatically if not present" - echo " -N, --no-virtual-env Don't use virtualenv. Run tests in local environment" echo " -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added." - echo " -u, --update Update the virtual environment with any newer package versions" echo " -p, --pep8 Just run pep8" echo " -P, --no-pep8 Don't run static code checks" + echo " -v, --verbose Increase verbosity of reporting output" echo " -h, --help Print this usage message" echo "" - echo "Note: with no options specified, the script will try to run the tests in a virtual environment," - echo " If no virtualenv is found, the script will ask if you would like to create one. If you " - echo " prefer to run tests NOT in a virtual environment, simply pass the -N option." exit } function process_option { case "$1" in -h|--help) usage;; - -V|--virtual-env) let always_venv=1; let never_venv=0;; - -N|--no-virtual-env) let always_venv=0; let never_venv=1;; -p|--pep8) let just_pep8=1;; -P|--no-pep8) let no_pep8=1;; -f|--force) let force=1;; - -u|--update) update=1;; - -c|--coverage) noseopts="$noseopts --with-coverage --cover-package=taskflow";; - -*) noseopts="$noseopts $1";; - *) noseargs="$noseargs $1" + -v|--verbose) let verbose=1;; + *) pos_args="$pos_args $1" esac } -venv=.venv -with_venv=tools/with_venv.sh -always_venv=0 -never_venv=0 +verbose=0 force=0 -noseopts= -noseargs= -wrapper="" +pos_args="" just_pep8=0 no_pep8=0 -update=0 - -export NOSE_WITH_OPENSTACK=1 -export NOSE_OPENSTACK_COLOR=1 -export NOSE_OPENSTACK_RED=0.05 -export NOSE_OPENSTACK_YELLOW=0.025 -export NOSE_OPENSTACK_SHOW_ELAPSED=1 -export NOSE_OPENSTACK_STDOUT=1 +tox_args="" +tox="" for arg in "$@"; do process_option $arg done +py=`which python` +if [ -z "$py" ]; then + echo "Python is required to use $0" + echo "Please install it via your distributions package management system." + exit 1 +fi + +py_envs=`python -c 'import sys; print("py%s%s" % (sys.version_info[0:2]))'` +py_envs=${PY_ENVS:-$py_envs} + function run_tests { - # Cleanup *pyc - ${wrapper} find . -type f -name "*.pyc" -delete - ${wrapper} $NOSETESTS + local tox_cmd="${tox} ${tox_args} -e $py_envs ${pos_args}" + echo "Running tests for environments $py_envs via $tox_cmd" + bash -c "$tox_cmd" } -function run_pep8 { - echo "Running pep8 ..." - PEP8_EXCLUDE=".venv,.tox,dist,doc,openstack" - PEP8_OPTIONS="--exclude=$PEP8_EXCLUDE --repeat" - PEP8_IGNORE="--ignore=E125,E126,E711,E712" - PEP8_INCLUDE="." - - ${wrapper} pep8 $PEP8_OPTIONS $PEP8_INCLUDE $PEP8_IGNORE +function run_flake8 { + local tox_cmd="${tox} ${tox_args} -e pep8 ${pos_args}" + echo "Running flake8 via $tox_cmd" + bash -c "$tox_cmd" } +if [ $force -eq 1 ]; then + tox_args="$tox_args -r" +fi -NOSETESTS="nosetests $noseopts $noseargs" +if [ $verbose -eq 1 ]; then + tox_args="$tox_args -v" +fi -if [ $never_venv -eq 0 ] -then - # Remove the virtual environment if --force used - if [ $force -eq 1 ]; then - echo "Cleaning virtualenv..." - rm -rf ${venv} - fi - if [ $update -eq 1 ]; then - echo "Updating virtualenv..." - python tools/install_venv.py - fi - if [ -e ${venv} ]; then - wrapper="${with_venv}" - else - if [ $always_venv -eq 1 ]; then - # Automatically install the virtualenv - python tools/install_venv.py - wrapper="${with_venv}" - else - echo -e "No virtual environment found...create one? (Y/n) \c" - read use_ve - if [ "x$use_ve" = "xY" -o "x$use_ve" = "x" -o "x$use_ve" = "xy" ]; then - # Install the virtualenv and run the test suite in it - python tools/install_venv.py - wrapper=${with_venv} - fi - fi - fi +tox=`which tox` +if [ -z "$tox" ]; then + echo "Tox is required to use $0" + echo "Please install it via \`pip\` or via your distributions" \ + "package management system." + echo "Visit http://tox.readthedocs.org/ for additional installation" \ + "instructions." + exit 1 fi if [ $just_pep8 -eq 1 ]; then - run_pep8 - exit + run_flake8 + exit fi run_tests || exit -if [ -z "$noseargs" ]; then - if [ $no_pep8 -eq 0 ]; then - run_pep8 - fi +if [ $no_pep8 -eq 0 ]; then + run_flake8 fi diff --git a/tools/install_venv.py b/tools/install_venv.py deleted file mode 100644 index 80df201d7..000000000 --- a/tools/install_venv.py +++ /dev/null @@ -1,75 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Copyright 2010 OpenStack LLC. -# Copyright 2013 IBM Corp. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Installation script for Taskflows's development virtualenv -""" - -import os -import sys - -import install_venv_common as install_venv - - -def print_help(): - help = """ -Taskflow development environment setup is complete. - -Taskflow development uses virtualenv to track and manage Python dependencies -while in development and testing. - -To activate the taskflow virtualenv for the extent of your current shell -session you can run: - -$ source .venv/bin/activate - -Or, if you prefer, you can run commands in the virtualenv on a case by case -basis by running: - -$ tools/with_venv.sh - -Also, make test will automatically use the virtualenv. - """ - for line in help.splitlines(): - print(" %s" % (line)) - - -def main(argv): - root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - venv = os.path.join(root, '.venv') - pip_requires = os.path.join(root, 'requirements.txt') - test_requires = os.path.join(root, 'test-requirements.txt') - py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) - project = 'Taskflow' - install = install_venv.InstallVenv(root, venv, pip_requires, test_requires, - py_version, project) - options = install.parse_args(argv) - install.check_python_version() - install.check_dependencies() - install.create_virtualenv(no_site_packages=options.no_site_packages) - install.install_dependencies() - install.run_command([os.path.join(venv, 'bin/python'), - 'setup.py', 'develop']) - install.post_process() - print_help() - -if __name__ == '__main__': - main(sys.argv) diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py deleted file mode 100644 index 46822e329..000000000 --- a/tools/install_venv_common.py +++ /dev/null @@ -1,172 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# Copyright 2013 IBM Corp. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Provides methods needed by installation script for OpenStack development -virtual environments. - -Since this script is used to bootstrap a virtualenv from the system's Python -environment, it should be kept strictly compatible with Python 2.6. - -Synced in from openstack-common -""" - -from __future__ import print_function - -import optparse -import os -import subprocess -import sys - - -class InstallVenv(object): - - def __init__(self, root, venv, requirements, - test_requirements, py_version, - project): - self.root = root - self.venv = venv - self.requirements = requirements - self.test_requirements = test_requirements - self.py_version = py_version - self.project = project - - def die(self, message, *args): - print(message % args, file=sys.stderr) - sys.exit(1) - - def check_python_version(self): - if sys.version_info < (2, 6): - self.die("Need Python Version >= 2.6") - - def run_command_with_code(self, cmd, redirect_output=True, - check_exit_code=True): - """Runs a command in an out-of-process shell. - - Returns the output of that command. Working directory is self.root. - """ - if redirect_output: - stdout = subprocess.PIPE - else: - stdout = None - - proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout) - output = proc.communicate()[0] - if check_exit_code and proc.returncode != 0: - self.die('Command "%s" failed.\n%s', ' '.join(cmd), output) - return (output, proc.returncode) - - def run_command(self, cmd, redirect_output=True, check_exit_code=True): - return self.run_command_with_code(cmd, redirect_output, - check_exit_code)[0] - - def get_distro(self): - if (os.path.exists('/etc/fedora-release') or - os.path.exists('/etc/redhat-release')): - return Fedora( - self.root, self.venv, self.requirements, - self.test_requirements, self.py_version, self.project) - else: - return Distro( - self.root, self.venv, self.requirements, - self.test_requirements, self.py_version, self.project) - - def check_dependencies(self): - self.get_distro().install_virtualenv() - - def create_virtualenv(self, no_site_packages=True): - """Creates the virtual environment and installs PIP. - - Creates the virtual environment and installs PIP only into the - virtual environment. - """ - if not os.path.isdir(self.venv): - print('Creating venv...', end=' ') - if no_site_packages: - self.run_command(['virtualenv', '-q', '--no-site-packages', - self.venv]) - else: - self.run_command(['virtualenv', '-q', self.venv]) - print('done.') - else: - print("venv already exists...") - pass - - def pip_install(self, *args): - self.run_command(['tools/with_venv.sh', - 'pip', 'install', '--upgrade'] + list(args), - redirect_output=False) - - def install_dependencies(self): - print('Installing dependencies with pip (this can take a while)...') - - # First things first, make sure our venv has the latest pip and - # setuptools and pbr - self.pip_install('pip>=1.4') - self.pip_install('setuptools') - self.pip_install('pbr') - - self.pip_install('-r', self.requirements, '-r', self.test_requirements) - - def parse_args(self, argv): - """Parses command-line arguments.""" - parser = optparse.OptionParser() - parser.add_option('-n', '--no-site-packages', - action='store_true', - help="Do not inherit packages from global Python " - "install") - return parser.parse_args(argv[1:])[0] - - -class Distro(InstallVenv): - - def check_cmd(self, cmd): - return bool(self.run_command(['which', cmd], - check_exit_code=False).strip()) - - def install_virtualenv(self): - if self.check_cmd('virtualenv'): - return - - if self.check_cmd('easy_install'): - print('Installing virtualenv via easy_install...', end=' ') - if self.run_command(['easy_install', 'virtualenv']): - print('Succeeded') - return - else: - print('Failed') - - self.die('ERROR: virtualenv not found.\n\n%s development' - ' requires virtualenv, please install it using your' - ' favorite package management tool' % self.project) - - -class Fedora(Distro): - """This covers all Fedora-based distributions. - - Includes: Fedora, RHEL, CentOS, Scientific Linux - """ - - def check_pkg(self, pkg): - return self.run_command_with_code(['rpm', '-q', pkg], - check_exit_code=False)[1] == 0 - - def install_virtualenv(self): - if self.check_cmd('virtualenv'): - return - - if not self.check_pkg('python-virtualenv'): - self.die("Please install 'python-virtualenv'.") - - super(Fedora, self).install_virtualenv() diff --git a/tools/with_venv.sh b/tools/with_venv.sh deleted file mode 100755 index f34cfc23f..000000000 --- a/tools/with_venv.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -TOOLS=`dirname $0` -VENV=$TOOLS/../.venv -source $VENV/bin/activate && $@