Also update extras in setup.cfg.
This required some care. There doesn't seem to be a sane Python3 ready comment-preserving ini parser around, so I wrote a minimal-for-our-case one in Parsley. Parsley is already in use in infra in bindep, but I need to add it to global-requirements as this is the first use in a managed project of it. Change-Id: I48de3a2f36e945f75b534f689e3af802bbdc5be9 Depends-On: I7d7e91694c9145fac0ddab8a9de5f789d723c641 Depends-On: I16e967356d5c56f1474ee661b954b3db11a608cb
This commit is contained in:
Robert Collins
parent
00356aaf1e
commit
4b22b94752
|
|
@ -92,6 +92,7 @@ os-refresh-config
|
|||
os-testr>=0.1.0
|
||||
ovs>=2.4.0.dev0 # Apache-2.0
|
||||
paramiko>=1.13.0
|
||||
Parsley
|
||||
passlib
|
||||
Paste
|
||||
PasteDeploy>=1.5.0
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import sys
|
|||
import textwrap
|
||||
|
||||
import fixtures
|
||||
import parsley
|
||||
import pkg_resources
|
||||
import testscenarios
|
||||
import testtools
|
||||
|
|
@ -481,3 +482,213 @@ class TestReqsToContent(testtools.TestCase):
|
|||
''.join(update._REQS_HEADER
|
||||
+ ["foo<=1!python_version=='2.7' # BSD\n"]),
|
||||
reqs)
|
||||
|
||||
|
||||
class TestProjectExtras(testtools.TestCase):
|
||||
|
||||
def test_smoke(self):
|
||||
project = {'setup.cfg': textwrap.dedent(u"""
|
||||
[extras]
|
||||
1 =
|
||||
foo
|
||||
2 =
|
||||
foo # fred
|
||||
bar
|
||||
""")}
|
||||
expected = {
|
||||
'1': '\nfoo',
|
||||
'2': '\nfoo # fred\nbar'
|
||||
}
|
||||
self.assertEqual(expected, update._project_extras(project))
|
||||
|
||||
def test_none(self):
|
||||
project = {'setup.cfg': u"[metadata]\n"}
|
||||
self.assertEqual({}, update._project_extras(project))
|
||||
|
||||
|
||||
class TestExtras(testtools.TestCase):
|
||||
|
||||
def test_none(self):
|
||||
old_content = textwrap.dedent(u"""
|
||||
[metadata]
|
||||
# something something
|
||||
name = fred
|
||||
|
||||
[entry_points]
|
||||
console_scripts =
|
||||
foo = bar:quux
|
||||
""")
|
||||
ini = update.extras_compiled(old_content).ini()
|
||||
self.assertEqual(ini, (old_content, None, ''))
|
||||
|
||||
def test_no_eol(self):
|
||||
old_content = textwrap.dedent(u"""
|
||||
[metadata]
|
||||
# something something
|
||||
name = fred
|
||||
|
||||
[entry_points]
|
||||
console_scripts =
|
||||
foo = bar:quux""")
|
||||
expected1 = textwrap.dedent(u"""
|
||||
[metadata]
|
||||
# something something
|
||||
name = fred
|
||||
|
||||
[entry_points]
|
||||
console_scripts =
|
||||
""")
|
||||
suffix = ' foo = bar:quux'
|
||||
ini = update.extras_compiled(old_content).ini()
|
||||
self.assertEqual(ini, (expected1, None, suffix))
|
||||
|
||||
def test_two_extras_raises(self):
|
||||
old_content = textwrap.dedent(u"""
|
||||
[metadata]
|
||||
# something something
|
||||
name = fred
|
||||
|
||||
[extras]
|
||||
a = b
|
||||
[extras]
|
||||
b = c
|
||||
|
||||
[entry_points]
|
||||
console_scripts =
|
||||
foo = bar:quux
|
||||
""")
|
||||
with testtools.ExpectedException(parsley.ParseError):
|
||||
update.extras_compiled(old_content).ini()
|
||||
|
||||
def test_extras(self):
|
||||
# We get an AST for extras we can use to preserve comments.
|
||||
old_content = textwrap.dedent(u"""
|
||||
[metadata]
|
||||
# something something
|
||||
name = fred
|
||||
|
||||
[extras]
|
||||
# comment1
|
||||
a =
|
||||
b
|
||||
c
|
||||
# comment2
|
||||
# comment3
|
||||
d =
|
||||
e
|
||||
# comment4
|
||||
|
||||
[entry_points]
|
||||
console_scripts =
|
||||
foo = bar:quux
|
||||
""")
|
||||
prefix = textwrap.dedent(u"""
|
||||
[metadata]
|
||||
# something something
|
||||
name = fred
|
||||
|
||||
""")
|
||||
suffix = textwrap.dedent(u"""\
|
||||
[entry_points]
|
||||
console_scripts =
|
||||
foo = bar:quux
|
||||
""")
|
||||
extras = [
|
||||
update.Comment('# comment1\n'),
|
||||
update.Extra('a', '\nb\nc\n'),
|
||||
update.Comment('# comment2\n'),
|
||||
update.Comment('# comment3\n'),
|
||||
update.Extra('d', '\ne\n'),
|
||||
update.Comment('# comment4\n')]
|
||||
ini = update.extras_compiled(old_content).ini()
|
||||
self.assertEqual(ini, (prefix, extras, suffix))
|
||||
|
||||
|
||||
class TestMergeSetupCfg(testtools.TestCase):
|
||||
|
||||
def test_merge_none(self):
|
||||
old_content = textwrap.dedent(u"""
|
||||
[metadata]
|
||||
# something something
|
||||
name = fred
|
||||
|
||||
[entry_points]
|
||||
console_scripts =
|
||||
foo = bar:quux
|
||||
""")
|
||||
merged = update._merge_setup_cfg(old_content, {})
|
||||
self.assertEqual(old_content, merged)
|
||||
|
||||
def test_merge_extras(self):
|
||||
old_content = textwrap.dedent(u"""
|
||||
[metadata]
|
||||
name = fred
|
||||
|
||||
[extras]
|
||||
# Comment
|
||||
a =
|
||||
b
|
||||
# comment
|
||||
c =
|
||||
d
|
||||
|
||||
[entry_points]
|
||||
console_scripts =
|
||||
foo = bar:quux
|
||||
""")
|
||||
blank = update.Requirement('', '', '', '')
|
||||
r1 = update.Requirement('b', '>=1', "python_version=='2.7'", '')
|
||||
r2 = update.Requirement('d', '', '', '# BSD')
|
||||
reqs = {
|
||||
'a': update.Requirements([blank, r1]),
|
||||
'c': update.Requirements([blank, r2])}
|
||||
merged = update._merge_setup_cfg(old_content, reqs)
|
||||
expected = textwrap.dedent(u"""
|
||||
[metadata]
|
||||
name = fred
|
||||
|
||||
[extras]
|
||||
# Comment
|
||||
a =
|
||||
b>=1:python_version=='2.7'
|
||||
# comment
|
||||
c =
|
||||
d # BSD
|
||||
|
||||
[entry_points]
|
||||
console_scripts =
|
||||
foo = bar:quux
|
||||
""")
|
||||
self.assertEqual(expected, merged)
|
||||
|
||||
|
||||
class TestCopyRequires(testtools.TestCase):
|
||||
|
||||
def test_extras_no_change(self):
|
||||
global_content = textwrap.dedent(u"""\
|
||||
foo<2;python_version=='2.7' # BSD
|
||||
foo>1;python_version!='2.7'
|
||||
freddy
|
||||
""")
|
||||
setup_cfg = textwrap.dedent(u"""\
|
||||
[metadata]
|
||||
name = openstack.requirements
|
||||
|
||||
[extras]
|
||||
test =
|
||||
foo<2:python_version=='2.7' # BSD
|
||||
foo>1:python_version!='2.7'
|
||||
opt =
|
||||
freddy
|
||||
""")
|
||||
project = {}
|
||||
project['root'] = '/dev/null'
|
||||
project['requirements'] = {}
|
||||
project['setup.cfg'] = setup_cfg
|
||||
global_reqs = update._parse_reqs(global_content)
|
||||
actions = update._copy_requires(
|
||||
u'', False, False, project, global_reqs, False)
|
||||
self.assertEqual([
|
||||
update.Verbose('Syncing extra [opt]'),
|
||||
update.Verbose('Syncing extra [test]'),
|
||||
update.File('setup.cfg', setup_cfg)], actions)
|
||||
|
|
|
|||
|
|
@ -28,13 +28,16 @@ files will be dropped.
|
|||
|
||||
import collections
|
||||
import errno
|
||||
import io
|
||||
import itertools
|
||||
import optparse
|
||||
import os
|
||||
import os.path
|
||||
import sys
|
||||
|
||||
from parsley import makeGrammar
|
||||
import pkg_resources
|
||||
from six.moves import configparser
|
||||
|
||||
_setup_py_text = """#!/usr/bin/env python
|
||||
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
|
||||
|
|
@ -79,6 +82,25 @@ _REQS_HEADER = [
|
|||
]
|
||||
|
||||
|
||||
Comment = collections.namedtuple('Comment', ['line'])
|
||||
Extra = collections.namedtuple('Extra', ['name', 'content'])
|
||||
|
||||
|
||||
extras_grammar = """
|
||||
ini = (line*:p extras?:e line*:l final:s) -> (''.join(p), e, ''.join(l+[s]))
|
||||
line = ~extras <(~'\\n' anything)* '\\n'>
|
||||
final = <(~'\\n' anything)* >
|
||||
extras = '[' 'e' 'x' 't' 'r' 'a' 's' ']' '\\n'+ body*:b -> b
|
||||
body = comment | extra
|
||||
comment = <'#' (~'\\n' anything)* '\\n'>:c '\\n'* -> comment(c)
|
||||
extra = name:n ' '* '=' line:l cont*:c '\\n'* -> extra(n, ''.join([l] + c))
|
||||
name = <(anything:x ?(x not in '\\n \\t='))+>
|
||||
cont = ' '+ <(~'\\n' anything)* '\\n'>
|
||||
"""
|
||||
extras_compiled = makeGrammar(
|
||||
extras_grammar, {"comment": Comment, "extra": Extra})
|
||||
|
||||
|
||||
# Pure --
|
||||
class Change(object):
|
||||
def __init__(self, name, old, new):
|
||||
|
|
@ -263,17 +285,80 @@ def _copy_requires(
|
|||
non_std_reqs)
|
||||
actions.extend(_actions)
|
||||
actions.append(File(dest_name, _reqs_to_content(reqs)))
|
||||
extras = _project_extras(project)
|
||||
output_extras = {}
|
||||
for extra, content in sorted(extras.items()):
|
||||
dest_name = 'extra-%s' % extra
|
||||
dest_path = "%s[%s]" % (project['root'], extra)
|
||||
dest_sequence = list(_content_to_reqs(content))
|
||||
actions.append(Verbose("Syncing extra [%s]" % extra))
|
||||
_actions, reqs = _sync_requirements_file(
|
||||
global_reqs, dest_sequence, dest_path, softupdate, hacking,
|
||||
non_std_reqs)
|
||||
actions.extend(_actions)
|
||||
output_extras[extra] = reqs
|
||||
dest_path = 'setup.cfg'
|
||||
if suffix:
|
||||
dest_path = "%s.%s" % (dest_path, suffix)
|
||||
actions.append(File(
|
||||
dest_path, _merge_setup_cfg(project['setup.cfg'], output_extras)))
|
||||
return actions
|
||||
|
||||
|
||||
def _reqs_to_content(reqs, marker_sep=';'):
|
||||
lines = list(_REQS_HEADER)
|
||||
def _merge_setup_cfg(old_content, new_extras):
|
||||
# This is ugly. All the existing libraries handle setup.cfg's poorly.
|
||||
prefix, extras, suffix = extras_compiled(old_content).ini()
|
||||
out_extras = []
|
||||
if extras is not None:
|
||||
for extra in extras:
|
||||
if type(extra) is Comment:
|
||||
out_extras.append(extra)
|
||||
elif type(extra) is Extra:
|
||||
if extra.name not in new_extras:
|
||||
out_extras.append(extra)
|
||||
continue
|
||||
e = Extra(
|
||||
extra.name,
|
||||
_reqs_to_content(
|
||||
new_extras[extra.name], ':', ' ', False))
|
||||
out_extras.append(e)
|
||||
else:
|
||||
raise TypeError('unknown type %r' % extra)
|
||||
if out_extras:
|
||||
extras_str = ['[extras]\n']
|
||||
for extra in out_extras:
|
||||
if type(extra) is Comment:
|
||||
extras_str.append(extra.line)
|
||||
else:
|
||||
extras_str.append(extra.name + ' =')
|
||||
extras_str.append(extra.content)
|
||||
if suffix:
|
||||
extras_str.append('\n')
|
||||
extras_str = ''.join(extras_str)
|
||||
else:
|
||||
extras_str = ''
|
||||
return prefix + extras_str + suffix
|
||||
|
||||
|
||||
def _project_extras(project):
|
||||
"""Return a dict of extra-name:content for the extras in setup.cfg."""
|
||||
c = configparser.SafeConfigParser()
|
||||
c.readfp(io.StringIO(project['setup.cfg']))
|
||||
if not c.has_section('extras'):
|
||||
return dict()
|
||||
return dict(c.items('extras'))
|
||||
|
||||
|
||||
def _reqs_to_content(reqs, marker_sep=';', line_prefix='', prefix=True):
|
||||
lines = []
|
||||
if prefix:
|
||||
lines += _REQS_HEADER
|
||||
for req in reqs.reqs:
|
||||
comment_p = ' ' if req.package else ''
|
||||
comment = (comment_p + req.comment if req.comment else '')
|
||||
marker = marker_sep + req.markers if req.markers else ''
|
||||
lines.append(
|
||||
'%s%s%s%s\n' % (req.package, req.specifiers, marker, comment))
|
||||
package = line_prefix + req.package if req.package else ''
|
||||
lines.append('%s%s%s%s\n' % (package, req.specifiers, marker, comment))
|
||||
return u''.join(lines)
|
||||
|
||||
|
||||
|
|
@ -315,7 +400,8 @@ def _safe_read(project, filename, output=None):
|
|||
if output is None:
|
||||
output = project
|
||||
try:
|
||||
with open(project['root'] + '/' + filename, 'rt') as f:
|
||||
path = project['root'] + '/' + filename
|
||||
with io.open(path, 'rt', encoding="utf-8") as f:
|
||||
output[filename] = f.read()
|
||||
except IOError as e:
|
||||
if e.errno != errno.ENOENT:
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
Parsley
|
||||
|
|
@ -73,6 +73,10 @@ mkdir -p $projectdir
|
|||
# Attempt to install all of global requirements
|
||||
install_all_of_gr
|
||||
|
||||
# Install requirements
|
||||
$tmpdir/all_requirements/bin/pip install $REPODIR/requirements
|
||||
UPDATE="$tmpdir/all_requirements/bin/update-requirements"
|
||||
|
||||
for PROJECT in $PROJECTS ; do
|
||||
SHORT_PROJECT=$(basename $PROJECT)
|
||||
if ! grep 'pbr' $REPODIR/$SHORT_PROJECT/setup.py >/dev/null 2>&1
|
||||
|
|
@ -109,7 +113,7 @@ for PROJECT in $PROJECTS ; do
|
|||
|
||||
# set up the project synced with the global requirements
|
||||
sudo chown -R $USER $REPODIR/$SHORT_PROJECT
|
||||
(cd $REPODIR/requirements && python update.py $REPODIR/$SHORT_PROJECT)
|
||||
$UPDATE --source $REPODIR/requirements $REPODIR/$SHORT_PROJECT
|
||||
pushd $REPODIR/$SHORT_PROJECT
|
||||
if ! git diff --exit-code > /dev/null; then
|
||||
git commit -a -m'Update requirements'
|
||||
|
|
|
|||
Loading…
Reference in New Issue