From 4b22b94752e56358a10b7fb772732d9456c8202e Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Wed, 17 Jun 2015 01:30:55 +1200 Subject: [PATCH] Also update extras in setup.cfg. This required some care. There doesn't seem to be a sane Python3 ready comment-preserving ini parser around, so I wrote a minimal-for-our-case one in Parsley. Parsley is already in use in infra in bindep, but I need to add it to global-requirements as this is the first use in a managed project of it. Change-Id: I48de3a2f36e945f75b534f689e3af802bbdc5be9 Depends-On: I7d7e91694c9145fac0ddab8a9de5f789d723c641 Depends-On: I16e967356d5c56f1474ee661b954b3db11a608cb --- global-requirements.txt | 1 + openstack_requirements/tests/test_update.py | 211 ++++++++++++++++++++ openstack_requirements/update.py | 96 ++++++++- requirements.txt | 1 + tools/integration.sh | 6 +- 5 files changed, 309 insertions(+), 6 deletions(-) create mode 100644 requirements.txt diff --git a/global-requirements.txt b/global-requirements.txt index 376b7736ef..d7c7942895 100644 --- a/global-requirements.txt +++ b/global-requirements.txt @@ -92,6 +92,7 @@ os-refresh-config os-testr>=0.1.0 ovs>=2.4.0.dev0 # Apache-2.0 paramiko>=1.13.0 +Parsley passlib Paste PasteDeploy>=1.5.0 diff --git a/openstack_requirements/tests/test_update.py b/openstack_requirements/tests/test_update.py index d4872e5732..195972fcc3 100644 --- a/openstack_requirements/tests/test_update.py +++ b/openstack_requirements/tests/test_update.py @@ -20,6 +20,7 @@ import sys import textwrap import fixtures +import parsley import pkg_resources import testscenarios import testtools @@ -481,3 +482,213 @@ class TestReqsToContent(testtools.TestCase): ''.join(update._REQS_HEADER + ["foo<=1!python_version=='2.7' # BSD\n"]), reqs) + + +class TestProjectExtras(testtools.TestCase): + + def test_smoke(self): + project = {'setup.cfg': textwrap.dedent(u""" +[extras] +1 = + foo +2 = + foo # fred + bar +""")} + expected = { + '1': '\nfoo', + '2': '\nfoo # fred\nbar' + } + self.assertEqual(expected, update._project_extras(project)) + + def test_none(self): + project = {'setup.cfg': u"[metadata]\n"} + self.assertEqual({}, update._project_extras(project)) + + +class TestExtras(testtools.TestCase): + + def test_none(self): + old_content = textwrap.dedent(u""" + [metadata] + # something something + name = fred + + [entry_points] + console_scripts = + foo = bar:quux + """) + ini = update.extras_compiled(old_content).ini() + self.assertEqual(ini, (old_content, None, '')) + + def test_no_eol(self): + old_content = textwrap.dedent(u""" + [metadata] + # something something + name = fred + + [entry_points] + console_scripts = + foo = bar:quux""") + expected1 = textwrap.dedent(u""" + [metadata] + # something something + name = fred + + [entry_points] + console_scripts = + """) + suffix = ' foo = bar:quux' + ini = update.extras_compiled(old_content).ini() + self.assertEqual(ini, (expected1, None, suffix)) + + def test_two_extras_raises(self): + old_content = textwrap.dedent(u""" + [metadata] + # something something + name = fred + + [extras] + a = b + [extras] + b = c + + [entry_points] + console_scripts = + foo = bar:quux + """) + with testtools.ExpectedException(parsley.ParseError): + update.extras_compiled(old_content).ini() + + def test_extras(self): + # We get an AST for extras we can use to preserve comments. + old_content = textwrap.dedent(u""" + [metadata] + # something something + name = fred + + [extras] + # comment1 + a = + b + c + # comment2 + # comment3 + d = + e + # comment4 + + [entry_points] + console_scripts = + foo = bar:quux + """) + prefix = textwrap.dedent(u""" + [metadata] + # something something + name = fred + + """) + suffix = textwrap.dedent(u"""\ + [entry_points] + console_scripts = + foo = bar:quux + """) + extras = [ + update.Comment('# comment1\n'), + update.Extra('a', '\nb\nc\n'), + update.Comment('# comment2\n'), + update.Comment('# comment3\n'), + update.Extra('d', '\ne\n'), + update.Comment('# comment4\n')] + ini = update.extras_compiled(old_content).ini() + self.assertEqual(ini, (prefix, extras, suffix)) + + +class TestMergeSetupCfg(testtools.TestCase): + + def test_merge_none(self): + old_content = textwrap.dedent(u""" + [metadata] + # something something + name = fred + + [entry_points] + console_scripts = + foo = bar:quux + """) + merged = update._merge_setup_cfg(old_content, {}) + self.assertEqual(old_content, merged) + + def test_merge_extras(self): + old_content = textwrap.dedent(u""" + [metadata] + name = fred + + [extras] + # Comment + a = + b + # comment + c = + d + + [entry_points] + console_scripts = + foo = bar:quux + """) + blank = update.Requirement('', '', '', '') + r1 = update.Requirement('b', '>=1', "python_version=='2.7'", '') + r2 = update.Requirement('d', '', '', '# BSD') + reqs = { + 'a': update.Requirements([blank, r1]), + 'c': update.Requirements([blank, r2])} + merged = update._merge_setup_cfg(old_content, reqs) + expected = textwrap.dedent(u""" + [metadata] + name = fred + + [extras] + # Comment + a = + b>=1:python_version=='2.7' + # comment + c = + d # BSD + + [entry_points] + console_scripts = + foo = bar:quux + """) + self.assertEqual(expected, merged) + + +class TestCopyRequires(testtools.TestCase): + + def test_extras_no_change(self): + global_content = textwrap.dedent(u"""\ + foo<2;python_version=='2.7' # BSD + foo>1;python_version!='2.7' + freddy + """) + setup_cfg = textwrap.dedent(u"""\ + [metadata] + name = openstack.requirements + + [extras] + test = + foo<2:python_version=='2.7' # BSD + foo>1:python_version!='2.7' + opt = + freddy + """) + project = {} + project['root'] = '/dev/null' + project['requirements'] = {} + project['setup.cfg'] = setup_cfg + global_reqs = update._parse_reqs(global_content) + actions = update._copy_requires( + u'', False, False, project, global_reqs, False) + self.assertEqual([ + update.Verbose('Syncing extra [opt]'), + update.Verbose('Syncing extra [test]'), + update.File('setup.cfg', setup_cfg)], actions) diff --git a/openstack_requirements/update.py b/openstack_requirements/update.py index af0a4d01f6..e8da15914e 100644 --- a/openstack_requirements/update.py +++ b/openstack_requirements/update.py @@ -28,13 +28,16 @@ files will be dropped. import collections import errno +import io import itertools import optparse import os import os.path import sys +from parsley import makeGrammar import pkg_resources +from six.moves import configparser _setup_py_text = """#!/usr/bin/env python # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. @@ -79,6 +82,25 @@ _REQS_HEADER = [ ] +Comment = collections.namedtuple('Comment', ['line']) +Extra = collections.namedtuple('Extra', ['name', 'content']) + + +extras_grammar = """ +ini = (line*:p extras?:e line*:l final:s) -> (''.join(p), e, ''.join(l+[s])) +line = ~extras <(~'\\n' anything)* '\\n'> +final = <(~'\\n' anything)* > +extras = '[' 'e' 'x' 't' 'r' 'a' 's' ']' '\\n'+ body*:b -> b +body = comment | extra +comment = <'#' (~'\\n' anything)* '\\n'>:c '\\n'* -> comment(c) +extra = name:n ' '* '=' line:l cont*:c '\\n'* -> extra(n, ''.join([l] + c)) +name = <(anything:x ?(x not in '\\n \\t='))+> +cont = ' '+ <(~'\\n' anything)* '\\n'> +""" +extras_compiled = makeGrammar( + extras_grammar, {"comment": Comment, "extra": Extra}) + + # Pure -- class Change(object): def __init__(self, name, old, new): @@ -263,17 +285,80 @@ def _copy_requires( non_std_reqs) actions.extend(_actions) actions.append(File(dest_name, _reqs_to_content(reqs))) + extras = _project_extras(project) + output_extras = {} + for extra, content in sorted(extras.items()): + dest_name = 'extra-%s' % extra + dest_path = "%s[%s]" % (project['root'], extra) + dest_sequence = list(_content_to_reqs(content)) + actions.append(Verbose("Syncing extra [%s]" % extra)) + _actions, reqs = _sync_requirements_file( + global_reqs, dest_sequence, dest_path, softupdate, hacking, + non_std_reqs) + actions.extend(_actions) + output_extras[extra] = reqs + dest_path = 'setup.cfg' + if suffix: + dest_path = "%s.%s" % (dest_path, suffix) + actions.append(File( + dest_path, _merge_setup_cfg(project['setup.cfg'], output_extras))) return actions -def _reqs_to_content(reqs, marker_sep=';'): - lines = list(_REQS_HEADER) +def _merge_setup_cfg(old_content, new_extras): + # This is ugly. All the existing libraries handle setup.cfg's poorly. + prefix, extras, suffix = extras_compiled(old_content).ini() + out_extras = [] + if extras is not None: + for extra in extras: + if type(extra) is Comment: + out_extras.append(extra) + elif type(extra) is Extra: + if extra.name not in new_extras: + out_extras.append(extra) + continue + e = Extra( + extra.name, + _reqs_to_content( + new_extras[extra.name], ':', ' ', False)) + out_extras.append(e) + else: + raise TypeError('unknown type %r' % extra) + if out_extras: + extras_str = ['[extras]\n'] + for extra in out_extras: + if type(extra) is Comment: + extras_str.append(extra.line) + else: + extras_str.append(extra.name + ' =') + extras_str.append(extra.content) + if suffix: + extras_str.append('\n') + extras_str = ''.join(extras_str) + else: + extras_str = '' + return prefix + extras_str + suffix + + +def _project_extras(project): + """Return a dict of extra-name:content for the extras in setup.cfg.""" + c = configparser.SafeConfigParser() + c.readfp(io.StringIO(project['setup.cfg'])) + if not c.has_section('extras'): + return dict() + return dict(c.items('extras')) + + +def _reqs_to_content(reqs, marker_sep=';', line_prefix='', prefix=True): + lines = [] + if prefix: + lines += _REQS_HEADER for req in reqs.reqs: comment_p = ' ' if req.package else '' comment = (comment_p + req.comment if req.comment else '') marker = marker_sep + req.markers if req.markers else '' - lines.append( - '%s%s%s%s\n' % (req.package, req.specifiers, marker, comment)) + package = line_prefix + req.package if req.package else '' + lines.append('%s%s%s%s\n' % (package, req.specifiers, marker, comment)) return u''.join(lines) @@ -315,7 +400,8 @@ def _safe_read(project, filename, output=None): if output is None: output = project try: - with open(project['root'] + '/' + filename, 'rt') as f: + path = project['root'] + '/' + filename + with io.open(path, 'rt', encoding="utf-8") as f: output[filename] = f.read() except IOError as e: if e.errno != errno.ENOENT: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..73dbacf6db --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +Parsley diff --git a/tools/integration.sh b/tools/integration.sh index 2c86490a44..cea44fca29 100644 --- a/tools/integration.sh +++ b/tools/integration.sh @@ -73,6 +73,10 @@ mkdir -p $projectdir # Attempt to install all of global requirements install_all_of_gr +# Install requirements +$tmpdir/all_requirements/bin/pip install $REPODIR/requirements +UPDATE="$tmpdir/all_requirements/bin/update-requirements" + for PROJECT in $PROJECTS ; do SHORT_PROJECT=$(basename $PROJECT) if ! grep 'pbr' $REPODIR/$SHORT_PROJECT/setup.py >/dev/null 2>&1 @@ -109,7 +113,7 @@ for PROJECT in $PROJECTS ; do # set up the project synced with the global requirements sudo chown -R $USER $REPODIR/$SHORT_PROJECT - (cd $REPODIR/requirements && python update.py $REPODIR/$SHORT_PROJECT) + $UPDATE --source $REPODIR/requirements $REPODIR/$SHORT_PROJECT pushd $REPODIR/$SHORT_PROJECT if ! git diff --exit-code > /dev/null; then git commit -a -m'Update requirements'