rework tox.ini

separate works for py27, py35
add docs test
some other changes to let tests pass

Change-Id: I94291a5e65fee1e266fbe4e2ec3e14f042654598
This commit is contained in:
tikitavi 2017-10-20 11:46:04 +03:00
parent c5f4ce8eda
commit 0445156611
19 changed files with 48 additions and 538 deletions

View File

@ -1,11 +1,9 @@
from __future__ import print_function
import os
import subprocess
import sys
import os
import fileinput
import fnmatch
import warnings
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
@ -19,8 +17,8 @@ os.environ['DJANGO_SETTINGS_MODULE'] = 'openstack_dashboard.settings'
# -- General configuration ----------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc',
'sphinx.ext.doctest',
'sphinx.ext.todo',
@ -63,9 +61,9 @@ pygments_style = 'sphinx'
# The theme to use for HTML and HTML Help pages. Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
#html_theme_path = ["."]
#html_theme = '_theme'
#html_static_path = ['static']
# html_theme_path = ["."]
# html_theme = '_theme'
# html_static_path = ['static']
html_theme = 'openstackdocs'
html_last_updated_fmt = '%Y-%m-%d %H:%M'

View File

@ -8,11 +8,11 @@ Configuration options
The following options allow configuration that EC2API supports.
.. include:: ./tables/ec2api-clients.rst
.. include:: ./tables/ec2api-database.rst
.. include:: ./tables/ec2api-service.rst
.. include:: ./tables/ec2api-ec2.rst
.. include:: ./tables/ec2api-s3.rst
.. include:: ./tables/ec2api-common.rst
.. include:: ./tables/ec2api-clients.inc
.. include:: ./tables/ec2api-database.inc
.. include:: ./tables/ec2api-service.inc
.. include:: ./tables/ec2api-ec2.inc
.. include:: ./tables/ec2api-s3.inc
.. include:: ./tables/ec2api-common.inc

View File

@ -13,4 +13,4 @@ To configure Metadata Service for ec2api, EC2 API configuration_ common sections
The additional configuration options for EC2 Metadata:
.. include:: ./tables/ec2api-metadata.rst
.. include:: ./tables/ec2api-metadata.inc

View File

@ -8,6 +8,7 @@ fixtures>=3.0.0 # Apache-2.0/BSD
mock>=2.0.0 # BSD
nose>=1.3.7 # LGPL
oslotest>=1.10.0 # Apache-2.0
os-testr>=1.0.0 # Apache-2.0
pylint==1.4.5 # GPLv2
python-subunit>=0.0.18 # Apache-2.0/BSD
sphinx>=1.6.2 # BSD

View File

@ -1,81 +0,0 @@
#!/usr/bin/env python
# Copyright (c) 2012, Cloudscaling
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
'''
find_unused_options.py
Compare the ec2api.conf file with the ec2api.conf.sample file to find any
unused options or default values in ec2api.conf
'''
from __future__ import print_function
import argparse
import os
import sys
from oslo_config import iniparser
sys.path.append(os.getcwd())
class PropertyCollecter(iniparser.BaseParser):
def __init__(self):
super(PropertyCollecter, self).__init__()
self.key_value_pairs = {}
def assignment(self, key, value):
self.key_value_pairs[key] = value
def new_section(self, section):
pass
@classmethod
def collect_properties(cls, lineiter, sample_format=False):
def clean_sample(f):
for line in f:
if line.startswith("#") and not line.startswith("# "):
line = line[1:]
yield line
pc = cls()
if sample_format:
lineiter = clean_sample(lineiter)
pc.parse(lineiter)
return pc.key_value_pairs
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='''Compare the ec2api.conf
file with the ec2api.conf.sample file to find any unused options or
default values in ec2api.conf''')
parser.add_argument('-c', action='store',
default='/etc/ec2api/ec2api.conf',
help='path to ec2api.conf\
(defaults to /etc/ec2api/ec2api.conf)')
parser.add_argument('-s', default='./etc/ec2api/ec2api.conf.sample',
help='path to ec2api.conf.sample\
(defaults to ./etc/ec2api/ec2api.conf.sample')
options = parser.parse_args()
conf_file_options = PropertyCollecter.collect_properties(open(options.c))
sample_conf_file_options = PropertyCollecter.collect_properties(
open(options.s), sample_format=True)
for k, v in sorted(conf_file_options.items()):
if k not in sample_conf_file_options:
print("Unused:", k)
for k, v in sorted(conf_file_options.items()):
if k in sample_conf_file_options and v == sample_conf_file_options[k]:
print("Default valued:", k)

View File

@ -1,42 +0,0 @@
#!/bin/sh
# Copyright 2011 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
PRE_COMMIT_SCRIPT=.git/hooks/pre-commit
make_hook() {
echo "exec ./run_tests.sh -N -p" >> $PRE_COMMIT_SCRIPT
chmod +x $PRE_COMMIT_SCRIPT
if [ -w $PRE_COMMIT_SCRIPT -a -x $PRE_COMMIT_SCRIPT ]; then
echo "pre-commit hook was created successfully"
else
echo "unable to create pre-commit hook"
fi
}
# NOTE(jk0): Make sure we are in ec2-api's root directory before adding the hook.
if [ ! -d ".git" ]; then
echo "unable to find .git; moving up a directory"
cd ..
if [ -d ".git" ]; then
make_hook
else
echo "still unable to find .git; hook not created"
fi
else
make_hook
fi

View File

@ -1,72 +0,0 @@
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Copyright 2010 OpenStack Foundation
# Copyright 2013 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import sys
import install_venv_common as install_venv
def print_help(venv, root):
help = """
EC2 development environment setup is complete.
EC2 development uses virtualenv to track and manage Python dependencies
while in development and testing.
To activate the EC2 virtualenv for the extent of your current shell
session you can run:
$ source %s/bin/activate
Or, if you prefer, you can run commands in the virtualenv on a case by case
basis by running:
$ %s/tools/with_venv.sh <your command>
Also, make test will automatically use the virtualenv.
"""
print help % (venv, root)
def main(argv):
root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
if os.environ.get('tools_path'):
root = os.environ['tools_path']
venv = os.path.join(root, '.venv')
if os.environ.get('venv'):
venv = os.environ['venv']
pip_requires = os.path.join(root, 'requirements.txt')
test_requires = os.path.join(root, 'test-requirements.txt')
py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1])
project = 'ec2api'
install = install_venv.InstallVenv(root, venv, pip_requires, test_requires,
py_version, project)
options = install.parse_args(argv)
install.check_python_version()
install.check_dependencies()
install.create_virtualenv(no_site_packages=options.no_site_packages)
install.install_dependencies()
install.post_process()
print_help(venv, root)
if __name__ == '__main__':
main(sys.argv)

View File

@ -1,211 +0,0 @@
# Copyright 2013 OpenStack Foundation
# Copyright 2013 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Provides methods needed by installation script for OpenStack development
virtual environments.
Since this script is used to bootstrap a virtualenv from the system's Python
environment, it should be kept strictly compatible with Python 2.6.
Synced in from openstack-common
"""
from __future__ import print_function
import optparse
import os
import subprocess
import sys
class InstallVenv(object):
def __init__(self, root, venv, requirements,
test_requirements, py_version,
project):
self.root = root
self.venv = venv
self.requirements = requirements
self.test_requirements = test_requirements
self.py_version = py_version
self.project = project
def die(self, message, *args):
print(message % args, file=sys.stderr)
sys.exit(1)
def check_python_version(self):
if sys.version_info < (2, 6):
self.die("Need Python Version >= 2.6")
def run_command_with_code(self, cmd, redirect_output=True,
check_exit_code=True):
"""Runs a command in an out-of-process shell.
Returns the output of that command. Working directory is self.root.
"""
if redirect_output:
stdout = subprocess.PIPE
else:
stdout = None
proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout)
output = proc.communicate()[0]
if check_exit_code and proc.returncode != 0:
self.die('Command "%s" failed.\n%s', ' '.join(cmd), output)
return (output, proc.returncode)
def run_command(self, cmd, redirect_output=True, check_exit_code=True):
return self.run_command_with_code(cmd, redirect_output,
check_exit_code)[0]
def get_distro(self):
if (os.path.exists('/etc/fedora-release') or
os.path.exists('/etc/redhat-release')):
return Fedora(
self.root, self.venv, self.requirements,
self.test_requirements, self.py_version, self.project)
else:
return Distro(
self.root, self.venv, self.requirements,
self.test_requirements, self.py_version, self.project)
def check_dependencies(self):
self.get_distro().install_virtualenv()
def create_virtualenv(self, no_site_packages=True):
"""Creates the virtual environment and installs PIP.
Creates the virtual environment and installs PIP only into the
virtual environment.
"""
if not os.path.isdir(self.venv):
print('Creating venv...', end=' ')
if no_site_packages:
self.run_command(['virtualenv', '-q', '--no-site-packages',
self.venv])
else:
self.run_command(['virtualenv', '-q', self.venv])
print('done.')
else:
print("venv already exists...")
pass
def pip_install(self, *args):
self.run_command(['tools/with_venv.sh',
'pip', 'install', '--upgrade'] + list(args),
redirect_output=False)
def install_dependencies(self):
print('Installing dependencies with pip (this can take a while)...')
# First things first, make sure our venv has the latest pip and
# setuptools and pbr
self.pip_install('pip>=1.4')
self.pip_install('setuptools')
self.pip_install('pbr')
self.pip_install('-r', self.requirements, '-r', self.test_requirements)
def post_process(self):
self.get_distro().post_process()
def parse_args(self, argv):
"""Parses command-line arguments."""
parser = optparse.OptionParser()
parser.add_option('-n', '--no-site-packages',
action='store_true',
help="Do not inherit packages from global Python "
"install")
return parser.parse_args(argv[1:])[0]
class Distro(InstallVenv):
def check_cmd(self, cmd):
return bool(self.run_command(['which', cmd],
check_exit_code=False).strip())
def install_virtualenv(self):
if self.check_cmd('virtualenv'):
return
if self.check_cmd('easy_install'):
print('Installing virtualenv via easy_install...', end=' ')
if self.run_command(['easy_install', 'virtualenv']):
print('Succeeded')
return
else:
print('Failed')
self.die('ERROR: virtualenv not found.\n\n%s development'
' requires virtualenv, please install it using your'
' favorite package management tool' % self.project)
def post_process(self):
"""Any distribution-specific post-processing gets done here.
In particular, this is useful for applying patches to code inside
the venv.
"""
pass
class Fedora(Distro):
"""This covers all Fedora-based distributions.
Includes: Fedora, RHEL, CentOS, Scientific Linux
"""
def check_pkg(self, pkg):
return self.run_command_with_code(['rpm', '-q', pkg],
check_exit_code=False)[1] == 0
def apply_patch(self, originalfile, patchfile):
self.run_command(['patch', '-N', originalfile, patchfile],
check_exit_code=False)
def install_virtualenv(self):
if self.check_cmd('virtualenv'):
return
if not self.check_pkg('python-virtualenv'):
self.die("Please install 'python-virtualenv'.")
super(Fedora, self).install_virtualenv()
def post_process(self):
"""Workaround for a bug in eventlet.
This currently affects RHEL6.1, but the fix can safely be
applied to all RHEL and Fedora distributions.
This can be removed when the fix is applied upstream.
EC2: https://bugs.launchpad.net/nova/+bug/884915
Upstream: https://bitbucket.org/eventlet/eventlet/issue/89
RHEL: https://bugzilla.redhat.com/958868
"""
if os.path.exists('contrib/redhat-eventlet.patch'):
# Install "patch" program if it's not there
if not self.check_pkg('patch'):
self.die("Please install 'patch'.")
# Apply the eventlet patch
self.apply_patch(os.path.join(self.venv, 'lib', self.py_version,
'site-packages',
'eventlet/green/subprocess.py'),
'contrib/redhat-eventlet.patch')

View File

@ -1,48 +0,0 @@
# Copyright 2013 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import sys
import install_venv_common as install_venv # noqa
def first_file(file_list):
for candidate in file_list:
if os.path.exists(candidate):
return candidate
def main(argv):
root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
venv = os.environ['VIRTUAL_ENV']
pip_requires = first_file([
os.path.join(root, 'requirements.txt'),
os.path.join(root, 'tools', 'pip-requires'),
])
test_requires = first_file([
os.path.join(root, 'test-requirements.txt'),
os.path.join(root, 'tools', 'test-requires'),
])
py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1])
project = 'ec2api'
install = install_venv.InstallVenv(root, venv, pip_requires, test_requires,
py_version, project)
#NOTE(dprince): For Tox we only run post_process (which patches files, etc)
install.post_process()
if __name__ == '__main__':
main(sys.argv)

View File

@ -1,30 +0,0 @@
#!/usr/bin/env bash
# Client constraint file contains this client version pin that is in conflict
# with installing the client from source. We should remove the version pin in
# the constraints file before applying it for from-source installation.
CONSTRAINTS_FILE="$1"
shift 1
set -e
# NOTE(tonyb): Place this in the tox enviroment's log dir so it will get
# published to logs.openstack.org for easy debugging.
localfile="$VIRTUAL_ENV/log/upper-constraints.txt"
if [[ "$CONSTRAINTS_FILE" != http* ]]; then
CONSTRAINTS_FILE="file://$CONSTRAINTS_FILE"
fi
# NOTE(tonyb): need to add curl to bindep.txt if the project supports bindep
curl "$CONSTRAINTS_FILE" --insecure --progress-bar --output "$localfile"
pip install -c"$localfile" openstack-requirements
# This is the main purpose of the script: Allow local installation of
# the current repo. It is listed in constraints file and thus any
# install will be constrained and we need to unconstrain it.
edit-constraints "$localfile" -- "$CLIENT_NAME"
pip install -c"$localfile" -U "$@"
exit $?

View File

@ -1,7 +0,0 @@
#!/bin/bash
tools_path=${tools_path:-$(dirname $0)}
venv_path=${venv_path:-${tools_path}}
venv_dir=${venv_name:-/../.venv}
TOOLS=${tools_path}
VENV=${venv:-${venv_path}/${venv_dir}}
source ${VENV}/bin/activate && "$@"

66
tox.ini
View File

@ -1,50 +1,59 @@
[tox]
minversion = 2.0
envlist = py35,py27,pep8
minversion = 2.1
envlist = py{35,27},pep8,docs,api-ref
skipsdist = True
[testenv]
sitepackages = False
usedevelop = True
install_command = {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
# tox is silly... these need to be separated by a newline....
whitelist_externals = bash
find
rm
env
install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
setenv = VIRTUAL_ENV={envdir}
BRANCH_NAME=master
CLIENT_NAME=ec2-api
LANG=en_US.UTF-8
LANGUAGE=en_US:en
LC_ALL=C
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
deps = -r{toxinidir}/test-requirements.txt
commands =
find . -type f -name "*.pyc" -delete
rm -f .testrepository/times.dbm
python tools/patch_tox_venv.py
python setup.py testr --slowest --testr-args='{posargs}'
whitelist_externals =
rm
[tox:jenkins]
sitepackages = False
[testenv:py27]
commands =
{[testenv]commands}
stestr --test-path 'ec2api/tests/unit' run '{posargs}'
stestr slowest
[testenv:py35]
commands =
{[testenv]commands}
stestr --test-path 'ec2api/tests/unit' run '{posargs}'
stestr slowest
[testenv:pep8]
sitepackages = False
commands =
flake8 {posargs}
[testenv:genconfig]
commands = oslo-config-generator --config-file=etc/ec2api/ec2api-config-generator.conf
[testenv:cover]
# Also do not run test_coverage_ext tests while gathering coverage as those
# tests conflict with coverage.
commands =
coverage erase
python setup.py test --coverage --testr-args='{posargs}'
coverage combine
coverage html --include='ec2api/*' --omit='ec2api/tests/*' -d covhtml -i
[testenv:venv]
commands = {posargs}
[testenv:docs]
# This environment is called from CI scripts to test and publish
# the API Ref and Docs to developer.openstack.org.
commands = python setup.py build_sphinx
[testenv:api-ref]
# This environment is called from CI scripts to test and publish
# the API Ref to developer.openstack.org.
commands =
rm -rf api-ref/build
sphinx-build -W -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html
[flake8]
# E712 is ignored on purpose, since it is normal to use 'column == true'
# in sqlalchemy.
@ -55,15 +64,8 @@ ignore = E121,E122,E123,E124,E126,E127,E128,E711,E712,H102,H303,H404,F403,F811,F
# H106: Dont put vim configuration in source files
# H203: Use assertIs(Not)None to check for None
enable-extensions=H106,H203
exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,tools,ec2api/tests/functional/obsolete
exclude = .venv,.git,.tox,dist,envname,*lib/python*,*egg,build,tools,ec2api/tests/functional/obsolete
max-complexity=25
[hacking]
local-check-factory = ec2api.hacking.checks.factory
[testenv:api-ref]
# This environment is called from CI scripts to test and publish
# the API Ref to developer.openstack.org.
commands =
rm -rf api-ref/build
sphinx-build -W -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html