Add openstack-common and test infrastructure.

Fix pep8 errors (project is pep8 clean now).

Update setup.py to use openstack-common style dependencies.

Remove the unused novaclient dependency.

Change the keystoneclient dependency to a git URL.

Add test-requires, and move some pip-requires dependencies
into it.

Remove the test_utils unit test which wasn't testing anything
that is actually present in the project.

Add the test_authors unit test.

Use tox for running tests locally.

See: http://wiki.openstack.org/ProjectTestingInterface

Tox can manage virtualenvs, and is currently doing so for running
tests in Jenkins. It's just as, or more, useful for running tests
locally, so this starts the migration from the run_tests system to
tox. The goal is to reduce duplicate testing infrastructure, and
get what's running locally on developer workstations as close to
what is run by Jenkins as possible.

Run_tests.sh will now call tox to facilitate the transition for
developers used to typing "run_tests.sh".

Developers will need tox installed on their workstations. It can
be installed from PyPI with "pip install tox". run_tests.sh outputs
those instructions if tox is not present.

New facilities are available using tox directly, including:

tox -e py26 # run tests under python 2.6
tox -e py27 # run tests under python 2.7
tox -e pep8 # run pep8 tests
tox # run all of the above
tox -e venv foo # run the command "foo" inside a virtualenv

The OpenStack nose plugin is used when running tox from the
command line, so the enhanced, colorized output is visible to
developers running the test suite locally. However, when Jenkins
runs tox, xunit output will be used instead, which is natively
understood by jenkins and much more readable in that context.

Change-Id: Ib627be3b37b5a09d3795006d412ddcc35f8c6c1e
This commit is contained in:
James E. Blair 2012-04-28 22:21:53 +00:00
parent 9c945bee79
commit 95c2f27fa4
20 changed files with 353 additions and 243 deletions

2
.gitignore vendored
View File

@ -7,3 +7,5 @@
build
dist
python_openstackclient.egg-info
.tox/
ChangeLog

4
.gitreview Normal file
View File

@ -0,0 +1,4 @@
[gerrit]
host=review.openstack.org
port=29418
project=openstack/python-openstackclient.git

View File

@ -1 +1,2 @@
Dean Troyer <dtroyer@gmail.com>
James E. Blair <jeblair@hp.com>

View File

@ -1,5 +1,5 @@
include AUTHORS
include LICENSE
include README.rst
recursive-inlcude docs *
recursive-include docs *
recursive-include tests *

7
openstack-common.conf Normal file
View File

@ -0,0 +1,7 @@
[DEFAULT]
# The list of modules to copy from openstack-common
modules=setup
# The base module to hold the copy of openstack.common
base=openstackclient

View File

@ -29,6 +29,7 @@ def _find_server(cs, server):
"""Get a server by name or ID."""
return utils.find_resource(cs.servers, server)
def _print_server(cs, server):
# By default when searching via name we will do a
# findall(name=blah) and due a REST /details which is not the same
@ -56,6 +57,7 @@ def _print_server(cs, server):
utils.print_dict(info)
class List_Server(command.OpenStackCommand):
"List server command."
@ -74,6 +76,7 @@ class List_Server(command.OpenStackCommand):
def run(self, parsed_args):
self.log.info('v2.List_Server.run(%s)' % parsed_args)
class Show_Server(command.OpenStackCommand):
"Show server command."

View File

View File

@ -0,0 +1,140 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Utilities with minimum-depends for use in setup.py
"""
import os
import re
import subprocess
def parse_mailmap(mailmap='.mailmap'):
mapping = {}
if os.path.exists(mailmap):
fp = open(mailmap, 'r')
for l in fp:
l = l.strip()
if not l.startswith('#') and ' ' in l:
canonical_email, alias = l.split(' ')
mapping[alias] = canonical_email
return mapping
def canonicalize_emails(changelog, mapping):
""" Takes in a string and an email alias mapping and replaces all
instances of the aliases in the string with their real email
"""
for alias, email in mapping.iteritems():
changelog = changelog.replace(alias, email)
return changelog
# Get requirements from the first file that exists
def get_reqs_from_files(requirements_files):
reqs_in = []
for requirements_file in requirements_files:
if os.path.exists(requirements_file):
return open(requirements_file, 'r').read().split('\n')
return []
def parse_requirements(requirements_files=['requirements.txt',
'tools/pip-requires']):
requirements = []
for line in get_reqs_from_files(requirements_files):
if re.match(r'\s*-e\s+', line):
requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1',
line))
elif re.match(r'\s*-f\s+', line):
pass
else:
requirements.append(line)
return requirements
def parse_dependency_links(requirements_files=['requirements.txt',
'tools/pip-requires']):
dependency_links = []
for line in get_reqs_from_files(requirements_files):
if re.match(r'(\s*#)|(\s*$)', line):
continue
if re.match(r'\s*-[ef]\s+', line):
dependency_links.append(re.sub(r'\s*-[ef]\s+', '', line))
return dependency_links
def write_requirements():
venv = os.environ.get('VIRTUAL_ENV', None)
if venv is not None:
with open("requirements.txt", "w") as req_file:
output = subprocess.Popen(["pip", "-E", venv, "freeze", "-l"],
stdout=subprocess.PIPE)
requirements = output.communicate()[0].strip()
req_file.write(requirements)
def _run_shell_command(cmd):
output = subprocess.Popen(["/bin/sh", "-c", cmd],
stdout=subprocess.PIPE)
return output.communicate()[0].strip()
def write_vcsversion(location):
""" Produce a vcsversion dict that mimics the old one produced by bzr
"""
if os.path.isdir('.git'):
branch_nick_cmd = 'git branch | grep -Ei "\* (.*)" | cut -f2 -d" "'
branch_nick = _run_shell_command(branch_nick_cmd)
revid_cmd = "git rev-parse HEAD"
revid = _run_shell_command(revid_cmd).split()[0]
revno_cmd = "git log --oneline | wc -l"
revno = _run_shell_command(revno_cmd)
with open(location, 'w') as version_file:
version_file.write("""
# This file is automatically generated by setup.py, So don't edit it. :)
version_info = {
'branch_nick': '%s',
'revision_id': '%s',
'revno': %s
}
""" % (branch_nick, revid, revno))
def write_git_changelog():
"""Write a changelog based on the git changelog"""
if os.path.isdir('.git'):
git_log_cmd = 'git log --stat'
changelog = _run_shell_command(git_log_cmd)
mailmap = parse_mailmap()
with open("ChangeLog", "w") as changelog_file:
changelog_file.write(canonicalize_emails(changelog, mailmap))
def generate_authors():
"""Create AUTHORS file using git commits"""
jenkins_email = 'jenkins@review.openstack.org'
if os.path.isdir('.git'):
# don't include jenkins email address in AUTHORS file
git_log_cmd = "git log --format='%aN <%aE>' | sort -u | " \
"grep -v " + jenkins_email
changelog = _run_shell_command(git_log_cmd)
mailmap = parse_mailmap()
with open("AUTHORS", "w") as authors_file:
authors_file.write(canonicalize_emails(changelog, mailmap))

View File

@ -95,17 +95,20 @@ class OpenStackShell(App):
parser.add_argument('--os-identity-api-version',
metavar='<identity-api-version>',
default=env('OS_IDENTITY_API_VERSION', default='2.0'),
help='Identity API version, default=2.0 (Env: OS_IDENTITY_API_VERSION)')
help='Identity API version, default=2.0 '\
'(Env: OS_IDENTITY_API_VERSION)')
parser.add_argument('--os-compute-api-version',
metavar='<compute-api-version>',
default=env('OS_COMPUTE_API_VERSION', default='2'),
help='Compute API version, default=2 (Env: OS_COMPUTE_API_VERSION)')
help='Compute API version, default=2 '\
'(Env: OS_COMPUTE_API_VERSION)')
parser.add_argument('--os-image-api-version',
metavar='<image-api-version>',
default=env('OS_IMAGE_API_VERSION', default='1.0'),
help='Image API version, default=1.0 (Env: OS_IMAGE_API_VERSION)')
help='Image API version, default=1.0 '\
'(Env: OS_IMAGE_API_VERSION)')
parser.add_argument('--os-token', metavar='<token>',
default=env('OS_TOKEN'),
@ -130,7 +133,10 @@ class OpenStackShell(App):
}
if self.options.debug:
print "API: Identity=%s Compute=%s Image=%s" % (self.api_version['identity'], self.api_version['compute'], self.api_version['image'])
print "API: Identity=%s Compute=%s Image=%s" % (
self.api_version['identity'],
self.api_version['compute'],
self.api_version['image'])
print "cmd: %s" % cmd
# do checking of os_username, etc here
@ -169,7 +175,8 @@ class OpenStackShell(App):
auth_url=kwargs.get('auth_url'),
)
token = self.auth_client.auth_token
endpoint = self.auth_client.service_catalog.url_for(service_type=cmd.api)
endpoint = self.auth_client.service_catalog.url_for(
service_type=cmd.api)
if self.options.debug:
print "api: %s" % cmd.api

View File

@ -1,152 +1,49 @@
#!/bin/bash
set -eu
function usage {
echo "Usage: $0 [OPTION]..."
echo "Run openstackclient test suite"
echo "Run python-openstackclient's test suite(s)"
echo ""
echo " -V, --virtual-env Always use virtualenv. Install automatically if not present"
echo " -N, --no-virtual-env Don't use virtualenv. Run tests in local environment"
echo " -s, --no-site-packages Isolate the virtualenv from the global Python environment"
echo " -x, --stop Stop running tests after the first error or failure."
echo " -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added."
echo " -p, --pep8 Just run pep8"
echo " -P, --no-pep8 Don't run pep8"
echo " -c, --coverage Generate coverage report"
echo " -h, --help Print this usage message"
echo " --hide-elapsed Don't print the elapsed time for each test along with slow test list"
echo ""
echo "Note: with no options specified, the script will try to run the tests in a virtual environment,"
echo " If no virtualenv is found, the script will ask if you would like to create one. If you "
echo " prefer to run tests NOT in a virtual environment, simply pass the -N option."
echo "This script is deprecated and currently retained for compatibility."
echo 'You can run the full test suite for multiple environments by running "tox".'
echo 'You can run tests for only python 2.7 by running "tox -e py27", or run only'
echo 'the pep8 tests with "tox -e pep8".'
exit
}
command -v tox > /dev/null 2>&1
if [ $? -ne 0 ]; then
echo 'This script requires "tox" to run.'
echo 'You can install it with "pip install tox".'
exit 1;
fi
just_pep8=0
function process_option {
case "$1" in
-h|--help) usage;;
-V|--virtual-env) always_venv=1; never_venv=0;;
-N|--no-virtual-env) always_venv=0; never_venv=1;;
-s|--no-site-packages) no_site_packages=1;;
-f|--force) force=1;;
-p|--pep8) just_pep8=1;;
-P|--no-pep8) no_pep8=1;;
-c|--coverage) coverage=1;;
-*) noseopts="$noseopts $1";;
*) noseargs="$noseargs $1"
-p|--pep8) let just_pep8=1;;
esac
}
venv=.venv
with_venv=tools/with_venv.sh
always_venv=0
never_venv=0
force=0
no_site_packages=0
installvenvopts=
noseargs=
noseopts=
wrapper=""
just_pep8=0
no_pep8=0
coverage=0
for arg in "$@"; do
process_option $arg
done
# If enabled, tell nose to collect coverage data
if [ $coverage -eq 1 ]; then
noseopts="$noseopts --with-coverage --cover-package=openstackclient"
fi
if [ $no_site_packages -eq 1 ]; then
installvenvopts="--no-site-packages"
fi
function run_tests {
# Just run the test suites in current environment
${wrapper} $NOSETESTS
# If we get some short import error right away, print the error log directly
RESULT=$?
return $RESULT
}
function run_pep8 {
echo "Running pep8 ..."
srcfiles="openstackclient tests"
# Just run PEP8 in current environment
#
# NOTE(sirp): W602 (deprecated 3-arg raise) is being ignored for the
# following reasons:
#
# 1. It's needed to preserve traceback information when re-raising
# exceptions; this is needed b/c Eventlet will clear exceptions when
# switching contexts.
#
# 2. There doesn't appear to be an alternative, "pep8-tool" compatible way of doing this
# in Python 2 (in Python 3 `with_traceback` could be used).
#
# 3. Can find no corroborating evidence that this is deprecated in Python 2
# other than what the PEP8 tool claims. It is deprecated in Python 3, so,
# perhaps the mistake was thinking that the deprecation applied to Python 2
# as well.
pep8_opts="--ignore=E202,W602 --repeat"
${wrapper} pep8 ${pep8_opts} ${srcfiles}
}
NOSETESTS="nosetests $noseopts $noseargs"
if [ $never_venv -eq 0 ]
then
# Remove the virtual environment if --force used
if [ $force -eq 1 ]; then
echo "Cleaning virtualenv..."
rm -rf ${venv}
fi
if [ -e ${venv} ]; then
wrapper="${with_venv}"
else
if [ $always_venv -eq 1 ]; then
# Automatically install the virtualenv
python tools/install_venv.py $installvenvopts
wrapper="${with_venv}"
else
echo -e "No virtual environment found...create one? (Y/n) \c"
read use_ve
if [ "x$use_ve" = "xY" -o "x$use_ve" = "x" -o "x$use_ve" = "xy" ]; then
# Install the virtualenv and run the test suite in it
python tools/install_venv.py $installvenvopts
wrapper=${with_venv}
fi
fi
fi
fi
# Delete old coverage data from previous runs
if [ $coverage -eq 1 ]; then
${wrapper} coverage erase
fi
if [ $just_pep8 -eq 1 ]; then
run_pep8
exit
tox -e pep8
exit
fi
run_tests
# NOTE(sirp): we only want to run pep8 when we're running the full-test suite,
# not when we're running tests individually. To handle this, we need to
# distinguish between options (noseopts), which begin with a '-', and
# arguments (noseargs).
if [ -z "$noseargs" ]; then
if [ $no_pep8 -eq 0 ]; then
run_pep8
fi
tox -e py27 $toxargs 2>&1 | tee run_tests.err.log || exit
if [ ${PIPESTATUS[0]} -ne 0 ]; then
exit ${PIPESTATUS[0]}
fi
if [ $coverage -eq 1 ]; then
echo "Generating coverage report in covhtml/"
${wrapper} coverage html -d covhtml -i
if [ -z "$toxargs" ]; then
tox -e pep8
fi

View File

@ -3,6 +3,8 @@ cover-package = openstackclient
cover-html = true
cover-erase = true
cover-inclusive = true
verbosity=2
detailed-errors=1
[build_sphinx]
source-dir = docs/

View File

@ -1,36 +1,30 @@
import os
import sys
from setuptools import setup, find_packages
import setuptools
from openstackclient.openstack.common.setup import parse_requirements
from openstackclient.openstack.common.setup import parse_dependency_links
from openstackclient.openstack.common.setup import write_git_changelog
requires = parse_requirements()
dependency_links = parse_dependency_links()
write_git_changelog()
def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read()
requirements = [
'cliff',
'distribute',
'httplib2',
'prettytable',
"python-keystoneclient >= 2012.1",
"python-novaclient >= 2012.1",
]
if sys.version_info < (2, 6):
requirements.append('simplejson')
if sys.version_info < (2, 7):
requirements.append("argparse")
setup(
name = "python-openstackclient",
version = "2012.0",
description = "OpenStack command-line client",
long_description = read('README.rst'),
setuptools.setup(
name="python-openstackclient",
version="2012.2",
description="OpenStack command-line client",
long_description=read('README.rst'),
url='https://github.com/openstack/python-openstackclient',
license="Apache License, Version 2.0",
author = "Dean Troyer",
author_email = "dtroyer@gmail.com",
packages=find_packages(exclude=['tests', 'tests.*']),
url = "https://github.com/dtroyer/python-openstackclient",
install_requires=requirements,
author='OpenStack Client Contributors',
author_email='openstackclient@example.com',
packages=setuptools.find_packages(exclude=['tests', 'tests.*']),
classifiers=[
'Development Status :: 2 - Pre-Alpha',
'Environment :: Console',
@ -39,18 +33,16 @@ setup(
'License :: OSI Approved :: Apache Software License',
'Operating System :: OS Independent',
'Programming Language :: Python',
],
tests_require = ["nose", "mock", "mox"],
test_suite = "nose.collector",
entry_points = {
'console_scripts': ['stack = openstackclient.shell:main'],
],
install_requires=requires,
dependency_links=dependency_links,
test_suite="nose.collector",
entry_points={
'console_scripts': ['stack=openstackclient.shell:main'],
'openstack.cli': [
'list_server = openstackclient.compute.v2.server:List_Server',
'show_server = openstackclient.compute.v2.server:Show_Server',
'list_service = openstackclient.identity.v2_0.service:List_Service',
'list_server=openstackclient.compute.v2.server:List_Server',
'show_server=openstackclient.compute.v2.server:Show_Server',
'list_service=openstackclient.identity.v2_0.service:List_Service',
]
}
)

63
tests/test_authors.py Normal file
View File

@ -0,0 +1,63 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 OpenStack LLC
# Copyright 2012 Nebula Inc
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import commands
import os
import unittest
def parse_mailmap(mailmap='.mailmap'):
mapping = {}
if os.path.exists(mailmap):
fp = open(mailmap, 'r')
for l in fp:
l = l.strip()
if not l.startswith('#') and ' ' in l:
canonical_email, alias = l.split(' ')
mapping[alias] = canonical_email
return mapping
def str_dict_replace(s, mapping):
for s1, s2 in mapping.iteritems():
s = s.replace(s1, s2)
return s
class AuthorsTestCase(unittest.TestCase):
def test_authors_up_to_date(self):
root = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
contributors = set()
missing = set()
authors_file = open(os.path.join(root, 'AUTHORS'), 'r').read()
if os.path.exists(os.path.join(root, '.git')):
mailmap = parse_mailmap(os.path.join(root, '.mailmap'))
for email in commands.getoutput('git log --format=%ae').split():
if not email:
continue
if "jenkins" in email and "openstack.org" in email:
continue
email = '<' + email + '>'
contributors.add(str_dict_replace(email, mailmap))
for contributor in contributors:
if not contributor in authors_file:
missing.add(contributor)
self.assertTrue(len(missing) == 0,
'%r not listed in AUTHORS file.' % missing)

View File

@ -1,51 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
from openstackclient import utils as os_utils
from tests import utils
OBJ_LIST = [
{
'id': '123',
'name': 'foo',
'extra': {
'desc': 'foo fu',
'status': 'present',
}
},
{
'id': 'abc',
'name': 'bar',
'extra': {
'desc': 'babar',
'status': 'waiting',
}
}
]
class Obj(object):
def __init__(self):
pass
class UtilsTest(utils.TestCase):
def setUp(self):
super(UtilsTest, self).setUp()
self.objs = []
for o in OBJ_LIST:
obj = Obj()
for k in o.keys():
setattr(obj, k, o.get(k))
self.objs.append(obj)
def tearDown(self):
super(UtilsTest, self).tearDown()
self.objs = []
def test_expand_meta(self):
ret = os_utils.expand_meta(self.objs, 'extra')
assert (getattr(ret[0], 'desc') == 'foo fu')
assert (getattr(ret[0], 'status') == 'present')
assert (getattr(ret[0], 'extra', 'qaz') == 'qaz')

View File

@ -1,21 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
import time
import mox
import unittest
class TestCase(unittest.TestCase):
def setUp(self):
super(TestCase, self).setUp()
self.mox = mox.Mox()
self._original_time = time.time
time.time = lambda: 1234
def tearDown(self):
time.time = self._original_time
super(TestCase, self).tearDown()
self.mox.UnsetStubs()
self.mox.VerifyAll()

View File

@ -1,9 +1,8 @@
cliff
argparse
coverage
httplib2
mock
mox
nose
pep8>=0.6.1
prettytable
simplejson
-e git://github.com/openstack/python-keystoneclient.git#egg=python-keystoneclient

9
tools/test-requires Normal file
View File

@ -0,0 +1,9 @@
distribute>=0.6.24
mox
nose
nose-exclude
nosexcover
openstack.nose_plugin
pep8==0.6.1
sphinx>=1.1.2

10
tools/with_venv.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
command -v tox > /dev/null 2>&1
if [ $? -ne 0 ]; then
echo 'This script requires "tox" to run.'
echo 'You can install it with "pip install tox".'
exit 1;
fi
tox -evenv -- $@

46
tox.ini Normal file
View File

@ -0,0 +1,46 @@
[tox]
envlist = py26,py27,pep8
[testenv]
setenv = VIRTUAL_ENV={envdir}
NOSE_WITH_OPENSTACK=1
NOSE_OPENSTACK_COLOR=1
NOSE_OPENSTACK_RED=0.05
NOSE_OPENSTACK_YELLOW=0.025
NOSE_OPENSTACK_SHOW_ELAPSED=1
deps = -r{toxinidir}/tools/pip-requires
-r{toxinidir}/tools/test-requires
commands = nosetests
[testenv:pep8]
deps = pep8
commands = pep8 --repeat --show-source openstackclient setup.py
[testenv:venv]
commands = {posargs}
[testenv:cover]
commands = nosetests --cover-erase --cover-package=openstackclient --with-xcoverage
[tox:jenkins]
downloadcache = ~/cache/pip
[testenv:jenkins26]
basepython = python2.6
setenv = NOSE_WITH_XUNIT=1
deps = file://{toxinidir}/.cache.bundle
[testenv:jenkins27]
basepython = python2.7
setenv = NOSE_WITH_XUNIT=1
deps = file://{toxinidir}/.cache.bundle
[testenv:jenkinscover]
deps = file://{toxinidir}/.cache.bundle
setenv = NOSE_WITH_XUNIT=1
commands = nosetests --cover-erase --cover-package=openstackclient --with-xcoverage
[testenv:jenkinsvenv]
deps = file://{toxinidir}/.cache.bundle
setenv = NOSE_WITH_XUNIT=1
commands = {posargs}