openstack
/
requirements
Code Issues Proposed changes

Add upper-constraints.txt generator. 

This script when run creates a constraints file with exact pins for
the transitive dependencies of a requirements file.

Change-Id: I1282f8e4010c0ec46c473495bacddf044d06c0af
This commit is contained in:
Robert Collins 2015-06-08 17:10:49 +12:00
parent 9436d30a7d
commit 4273910b78
6 changed files with 246 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.rst
View File

@ -73,7 +73,7 @@ However due to the interactions with transitive dependencies this doesn't
actually deliver what we need.
We are moving to a system where we will define the precise versions of all
dependencies using ``upper-contraints.txt``. This will be overlaid onto all
dependencies using ``upper-constraints.txt``. This will be overlaid onto all
pip commands made during devstack, and by tox, and will provide a single,
atomic, source of truth for what works at any given time. The constraints will
be required to be compatible with the global requirements, and will

160
openstack_requirements/generate.py Normal file
View File

@ -0,0 +1,160 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import optparse
import os.path
import subprocess
import sys
import fixtures
def _parse_freeze(text):
"""Parse a freeze into structured data.
:param text: The output from a pip freeze command.
:return: A list of (package, version) tuples.
"""
result = []
for line in text.splitlines():
line = line.strip()
if line.startswith('-'):
raise Exception("Irregular line: %s" % line)
if line.startswith('#'):
continue
if not line:
continue
package, version = line.split('==')[:2]
result.append((package, version))
return result
def _freeze(requirements, python):
"""Generate a frozen install from requirements.
A constraints file is the result of installing a set of requirements and
then freezing the result. We currently special case pip and setuptools
as pip does, excluding them from the set. We may however want to revisit
this in future if releases of those things break our gate.
In principle we should determine this by introspecting all the packages
transitively, since we need to deal wit environment markers....
but thats reimplementing a large chunk of pip (and since pip doesn't
resolve yet, differently too). For now, we take a list of Python
executables to test under, and then union the results. This is in fact the
key difference between a constraints file and a requirements file: we're
not triggering installation, so we can and will list packages that are
not relevant to e.g. Python3.4 in the constraints output.
:param requirements: The path to a requirements file to use when generating
the constraints.
:param python: A Python binary to use. E.g. /usr/bin/python3.4
:return: A tuple (python_version, list of (package, version)'s)
"""
output = []
try:
version_out = subprocess.check_output(
[python, "--version"], stderr=subprocess.STDOUT)
output.append(version_out)
version_all = version_out.split()[1]
version = '.'.join(version_all.split('.')[:2])
with fixtures.TempDir() as temp:
output.append(subprocess.check_output(
['virtualenv', '-p', python, temp.path]))
pip_bin = os.path.join(temp.path, 'bin', 'pip')
output.append(subprocess.check_output(
[pip_bin, 'install', '-U', 'pip', 'setuptools', 'wheel']))
output.append(subprocess.check_output(
[pip_bin, 'install', '-r', requirements]))
freeze = subprocess.check_output([pip_bin, 'freeze'])
output.append(freeze)
return (version, _parse_freeze(freeze))
except Exception as exc:
if isinstance(exc, subprocess.CalledProcessError):
output.append(exc.output)
raise Exception(
"Failed to generate freeze: %s %s"
% ('\n'.join(output), exc))
def _combine_freezes(freezes):
"""Combine multiple freezes into a single structure.
This deals with the variation between different python versions by
generating environment markers when different pythons need different
versions of a dependency.
:param freezes: A list of (python_version, frozen_requirements) tuples.
:return: A list of '\n' terminated lines for a requirements file.
"""
packages = {} # {package : {version : [py_version]}}
reference_versions = []
for py_version, freeze in freezes:
if py_version in reference_versions:
raise Exception("Duplicate python %s" % py_version)
reference_versions.append(py_version)
for package, version in freeze:
packages.setdefault(
package, {}).setdefault(version, []).append(py_version)
for package, versions in sorted(packages.items()):
if len(versions) != 1 or versions.values()[0] != reference_versions:
# markers
for version, py_versions in sorted(versions.items()):
# Once the ecosystem matures, we can consider using OR.
for py_version in sorted(py_versions):
yield (
"%s===%s;python_version=='%s'\n" %
(package, version, py_version))
else:
# no markers
yield '%s===%s\n' % (package, versions.keys()[0])
# -- untested UI glue from here down.
def _validate_options(options):
"""Check that options are valid.
:param options: The optparse options for this program.
"""
if not options.pythons:
raise Exception("No Pythons given - see -p.")
for python in options.pythons:
if not os.path.exists(python):
raise Exception(
"Python %(python)s not found." % dict(python=python))
if not options.requirements:
raise Exception("No requirements file specified - see -r.")
if not os.path.exists(options.requirements):
raise Exception(
"Requirements file %(req)s not found."
% dict(req=options.requirementes))
def main(argv=None, stdout=None):
parser = optparse.OptionParser()
parser.add_option(
"-p", dest="pythons", action="append",
help="Specify Python versions to use when generating constraints."
"e.g. -p /usr/bin/python3.4")
parser.add_option(
"-r", dest="requirements", help="Requirements file to process.")
options, args = parser.parse_args(argv)
if stdout is None:
stdout = sys.stdout
_validate_options(options)
freezes = [
_freeze(options.requirements, python) for python in options.pythons]
stdout.writelines(_combine_freezes(freezes))
stdout.flush()

82
openstack_requirements/tests/test_generate.py Normal file
View File

@ -0,0 +1,82 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os.path
import fixtures
import testtools
from testtools import matchers
from openstack_requirements import generate
class TestFreeze(testtools.TestCase):
def test_freeze_smoke(self):
# Use an aribtrary python. The installation of virtualenv system wide
# is presumed.
versions = ['/usr/bin/python%(v)s' % dict(v=v) for v in
["2.7", "3.4"]]
found = [v for v in versions if os.path.exists(v)][0]
req = self.useFixture(fixtures.TempDir()).path + '/r.txt'
with open(req, 'wt') as output:
output.write('fixtures==1.2.0')
frozen = generate._freeze(req, found)
expected_version = found[-3:]
self.expectThat(frozen, matchers.HasLength(2))
self.expectThat(frozen[0], matchers.Equals(expected_version))
# There are multiple items in the dependency tree of fixtures.
# Since this is a smoke test, just ensure fixtures is there.
self.expectThat(frozen[1], matchers.Contains(('fixtures', '1.2.0')))
class TestParse(testtools.TestCase):
def test_parse(self):
text = "linecache2==1.0.0\nargparse==1.2\n\n# fred\n"
parsed = generate._parse_freeze(text)
self.assertEqual(
[('linecache2', '1.0.0'), ('argparse', '1.2')], parsed)
def test_editable_banned(self):
text = "-e git:..."
self.assertRaises(Exception, generate._parse_freeze, text) # noqa
class TestCombine(testtools.TestCase):
def test_same_items(self):
fixtures = [('fixtures', '1.2.0')]
freeze_27 = ('2.7', fixtures)
freeze_34 = ('3.4', fixtures)
self.assertEqual(
['fixtures===1.2.0\n'],
list(generate._combine_freezes([freeze_27, freeze_34])))
def test_distinct_items(self):
freeze_27 = ('2.7', [('fixtures', '1.2.0')])
freeze_34 = ('3.4', [('fixtures', '1.2.0'), ('enum', '1.5.0')])
self.assertEqual(
["enum===1.5.0;python_version=='3.4'\n", 'fixtures===1.2.0\n'],
list(generate._combine_freezes([freeze_27, freeze_34])))
def test_different_versions(self):
freeze_27 = ('2.7', [('fixtures', '1.2.0')])
freeze_34 = ('3.4', [('fixtures', '1.5.0')])
self.assertEqual(
["fixtures===1.2.0;python_version=='2.7'\n",
"fixtures===1.5.0;python_version=='3.4'\n"],
list(generate._combine_freezes([freeze_27, freeze_34])))
def test_duplicate_pythons(self):
with testtools.ExpectedException(Exception):
list(generate._combine_freezes([('2.7', []), ('2.7', [])]))

1
requirements.txt
View File

@ -1 +1,2 @@
fixtures>=0.3.14
Parsley

1
setup.cfg
View File

@ -23,4 +23,5 @@ packages =
[entry_points]
console_scripts =
generate-constraints = openstack_requirements.generate:main
update-requirements = openstack_requirements.update:main

2
test-requirements.txt
View File

@ -1,8 +1,8 @@
# NOTE: These are requirements for testing the requirements project only
# See global-requirements for the actual requirements list
hacking>=0.10,<0.11
fixtures>=0.3.14
testrepository>=0.0.18
testscenarios>=0.4
testtools>=0.9.36
toposort>=1.0 # Apache 2.0
virtualenv