openstack
/
requirements
Code Issues Proposed changes

Support multiple marked reqs for the same package 

This is needed when a single specifier set is insufficient to handle
both Python2.7 and 3.x. We're not currently dealing with this, but I
have observed it in other projects and we can be forearmed quite
easily.

Change-Id: If1493625d0d2ebf750159ccb8ed0fd22699766a1
This commit is contained in:
Robert Collins 2015-06-15 14:58:55 +12:00
parent 32fed9acdb
commit a3a3f7f57a
2 changed files with 149 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
openstack_requirements/tests/test_update.py
View File

@ -17,6 +17,7 @@ from __future__ import print_function
import io
import StringIO
import sys
import textwrap
import fixtures
import pkg_resources
@ -348,3 +349,128 @@ class TestParseRequirementFailures(testtools.TestCase):
def test_does_not_parse(self):
with testtools.ExpectedException(pkg_resources.RequirementParseError):
update._parse_requirement(self.line)
class TestSyncRequirementsFile(testtools.TestCase):
def test_multiple_lines_in_global_one_in_project(self):
global_content = textwrap.dedent("""\
foo<2;python_version=='2.7'
foo>1;python_version!='2.7'
""")
project_content = textwrap.dedent("""\
foo
""")
global_reqs = update._parse_reqs(global_content)
actions = update._sync_requirements_file(
global_reqs, project_content, 'f', False, False, 'f', False)
self.assertEqual(update.File('f', textwrap.dedent("""\
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
foo<2;python_version=='2.7'
foo>1;python_version!='2.7'
""")), actions[1])
self.assertEqual(update.StdOut(
" foo "
"-> foo<2;python_version=='2.7'\n"), actions[4])
self.assertEqual(update.StdOut(
" "
"-> foo>1;python_version!='2.7'\n"), actions[5])
self.assertThat(actions, matchers.HasLength(6))
def test_multiple_lines_separated_in_project_nochange(self):
global_content = textwrap.dedent("""\
foo<2;python_version=='2.7'
foo>1;python_version!='2.7'
""")
project_content = textwrap.dedent("""\
foo<2;python_version=='2.7'
# mumbo gumbo
foo>1;python_version!='2.7'
""")
global_reqs = update._parse_reqs(global_content)
actions = update._sync_requirements_file(
global_reqs, project_content, 'f', False, False, 'f', False)
self.assertEqual(update.File('f', textwrap.dedent("""\
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
foo<2;python_version=='2.7'
foo>1;python_version!='2.7'
# mumbo gumbo
""")), actions[1])
self.assertThat(actions, matchers.HasLength(2))
def test_multiple_lines_separated_in_project(self):
global_content = textwrap.dedent("""\
foo<2;python_version=='2.7'
foo>1;python_version!='2.7'
""")
project_content = textwrap.dedent("""\
foo<1.8;python_version=='2.7'
# mumbo gumbo
foo>0.9;python_version!='2.7'
""")
global_reqs = update._parse_reqs(global_content)
actions = update._sync_requirements_file(
global_reqs, project_content, 'f', False, False, 'f', False)
self.assertEqual(update.File('f', textwrap.dedent("""\
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
foo<2;python_version=='2.7'
foo>1;python_version!='2.7'
# mumbo gumbo
""")), actions[1])
self.assertEqual(update.StdOut(
" foo<1.8;python_version=='2.7' -> "
"foo<2;python_version=='2.7'\n"), actions[4])
self.assertEqual(update.StdOut(
" foo>0.9;python_version!='2.7' -> "
"foo>1;python_version!='2.7'\n"), actions[5])
self.assertThat(actions, matchers.HasLength(6))
def test_multiple_lines_nochange(self):
global_content = textwrap.dedent("""\
foo<2;python_version=='2.7'
foo>1;python_version!='2.7'
""")
project_content = textwrap.dedent("""\
foo<2;python_version=='2.7'
foo>1;python_version!='2.7'
""")
global_reqs = update._parse_reqs(global_content)
actions = update._sync_requirements_file(
global_reqs, project_content, 'f', False, False, 'f', False)
self.assertEqual(update.File('f', textwrap.dedent("""\
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
foo<2;python_version=='2.7'
foo>1;python_version!='2.7'
""")), actions[1])
self.assertThat(actions, matchers.HasLength(2))
def test_single_global_multiple_in_project(self):
global_content = textwrap.dedent("""\
foo>1
""")
project_content = textwrap.dedent("""\
foo<2;python_version=='2.7'
foo>1;python_version!='2.7'
""")
global_reqs = update._parse_reqs(global_content)
actions = update._sync_requirements_file(
global_reqs, project_content, 'f', False, False, 'f', False)
self.assertEqual(update.File('f', textwrap.dedent("""\
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
foo>1
""")), actions[1])
self.assertEqual(update.StdOut(
" foo<2;python_version=='2.7' -> foo>1\n"), actions[4])
self.assertEqual(update.StdOut(
" foo>1;python_version!='2.7' -> \n"), actions[5])
self.assertThat(actions, matchers.HasLength(6))

32
openstack_requirements/update.py
View File

@ -161,27 +161,31 @@ def _sync_requirements_file(
source_reqs, content, dest_path, softupdate, hacking, dest_name,
non_std_reqs):
actions = []
dest_reqs = list(_content_to_reqs(content))
dest_sequence = list(_content_to_reqs(content))
dest_reqs = _parse_reqs(content)
changes = []
actions.append(Verbose("Syncing %s" % dest_path))
content_lines = []
processed_packages = set()
# Check the instructions header
if dest_reqs[:len(_REQS_HEADER)] != zip(
if dest_sequence[:len(_REQS_HEADER)] != zip(
itertools.repeat(None), _REQS_HEADER):
content_lines.extend(_REQS_HEADER)
for req, req_line in dest_reqs:
for req, req_line in dest_sequence:
if req is None:
# Unparsable lines.
content_lines.append(req_line)
continue
if not req.package:
elif not req.package:
# Comment-only lines
content_lines.append(req_line)
continue
elif req.package.lower() in processed_packages:
continue
processed_packages.add(req.package.lower())
# Special cases:
# projects need to align hacking version on their own time
if req.package == "hacking" and not hacking:
@ -190,9 +194,19 @@ def _sync_requirements_file(
reference = source_reqs.get(req.package.lower())
if reference:
if reference[0] != req:
changes.append(Change(req.package, req_line, reference[1]))
content_lines.append(reference[1])
actual = dest_reqs.get(req.package.lower())
for req, ref in itertools.izip_longest(actual, reference):
if not req:
# More in globals
changes.append(Change(ref[0].package, '', ref[1]))
elif not ref:
# less in globals
changes.append(Change(req[0].package, req[1], ''))
elif req[0] != ref[0]:
# A change on this entry
changes.append(Change(req[0].package, req[1], ref[1]))
if ref:
content_lines.append(ref[1])
elif softupdate:
# under softupdate we pass through anything unknown packages,
# this is intended for ecosystem projects that want to stay in
@ -272,7 +286,7 @@ def _parse_reqs(content):
req_lines = _content_to_reqs(content)
for req, req_line in req_lines:
if req is not None:
reqs[req.package.lower()] = (req, req_line)
reqs.setdefault(req.package.lower(), []).append((req, req_line))
return reqs