Also update extras in setup.cfg.

This required some care. There doesn't seem to be a sane Python3 ready
comment-preserving ini parser around, so I wrote a
minimal-for-our-case one in Parsley. Parsley is already in use in
infra in bindep, but I need to add it to global-requirements as this
is the first use in a managed project of it.

Change-Id: I48de3a2f36e945f75b534f689e3af802bbdc5be9
Depends-On: I7d7e91694c9145fac0ddab8a9de5f789d723c641
Depends-On: I16e967356d5c56f1474ee661b954b3db11a608cb

global-requirements.txt

@@ -92,6 +92,7 @@ os-refresh-config
 os-testr>=0.1.0
 ovs>=2.4.0.dev0  # Apache-2.0
 paramiko>=1.13.0
+Parsley
 passlib
 Paste
 PasteDeploy>=1.5.0

openstack_requirements/tests/test_update.py

@@ -20,6 +20,7 @@ import sys
 import textwrap
 
 import fixtures
+import parsley
 import pkg_resources
 import testscenarios
 import testtools
@@ -481,3 +482,213 @@ class TestReqsToContent(testtools.TestCase):
             ''.join(update._REQS_HEADER
                     + ["foo<=1!python_version=='2.7' # BSD\n"]),
             reqs)
+
+
+class TestProjectExtras(testtools.TestCase):
+
+    def test_smoke(self):
+        project = {'setup.cfg': textwrap.dedent(u"""
+[extras]
+1 =
+  foo
+2 =
+  foo # fred
+  bar
+""")}
+        expected = {
+            '1': '\nfoo',
+            '2': '\nfoo # fred\nbar'
+        }
+        self.assertEqual(expected, update._project_extras(project))
+
+    def test_none(self):
+        project = {'setup.cfg': u"[metadata]\n"}
+        self.assertEqual({}, update._project_extras(project))
+
+
+class TestExtras(testtools.TestCase):
+
+    def test_none(self):
+        old_content = textwrap.dedent(u"""
+            [metadata]
+            # something something
+            name = fred
+
+            [entry_points]
+            console_scripts =
+                foo = bar:quux
+            """)
+        ini = update.extras_compiled(old_content).ini()
+        self.assertEqual(ini, (old_content, None, ''))
+
+    def test_no_eol(self):
+        old_content = textwrap.dedent(u"""
+            [metadata]
+            # something something
+            name = fred
+
+            [entry_points]
+            console_scripts =
+                foo = bar:quux""")
+        expected1 = textwrap.dedent(u"""
+            [metadata]
+            # something something
+            name = fred
+
+            [entry_points]
+            console_scripts =
+            """)
+        suffix = '    foo = bar:quux'
+        ini = update.extras_compiled(old_content).ini()
+        self.assertEqual(ini, (expected1, None, suffix))
+
+    def test_two_extras_raises(self):
+        old_content = textwrap.dedent(u"""
+            [metadata]
+            # something something
+            name = fred
+
+            [extras]
+            a = b
+            [extras]
+            b = c
+
+            [entry_points]
+            console_scripts =
+                foo = bar:quux
+            """)
+        with testtools.ExpectedException(parsley.ParseError):
+            update.extras_compiled(old_content).ini()
+
+    def test_extras(self):
+        # We get an AST for extras we can use to preserve comments.
+        old_content = textwrap.dedent(u"""
+            [metadata]
+            # something something
+            name = fred
+
+            [extras]
+            # comment1
+            a =
+             b
+             c
+            # comment2
+            # comment3
+            d =
+             e
+            # comment4
+
+            [entry_points]
+            console_scripts =
+                foo = bar:quux
+            """)
+        prefix = textwrap.dedent(u"""
+            [metadata]
+            # something something
+            name = fred
+
+            """)
+        suffix = textwrap.dedent(u"""\
+            [entry_points]
+            console_scripts =
+                foo = bar:quux
+            """)
+        extras = [
+            update.Comment('# comment1\n'),
+            update.Extra('a', '\nb\nc\n'),
+            update.Comment('# comment2\n'),
+            update.Comment('# comment3\n'),
+            update.Extra('d', '\ne\n'),
+            update.Comment('# comment4\n')]
+        ini = update.extras_compiled(old_content).ini()
+        self.assertEqual(ini, (prefix, extras, suffix))
+
+
+class TestMergeSetupCfg(testtools.TestCase):
+
+    def test_merge_none(self):
+        old_content = textwrap.dedent(u"""
+            [metadata]
+            # something something
+            name = fred
+
+            [entry_points]
+            console_scripts =
+                foo = bar:quux
+            """)
+        merged = update._merge_setup_cfg(old_content, {})
+        self.assertEqual(old_content, merged)
+
+    def test_merge_extras(self):
+        old_content = textwrap.dedent(u"""
+            [metadata]
+            name = fred
+
+            [extras]
+            # Comment
+            a =
+             b
+            # comment
+            c =
+             d
+
+            [entry_points]
+            console_scripts =
+                foo = bar:quux
+            """)
+        blank = update.Requirement('', '', '', '')
+        r1 = update.Requirement('b', '>=1', "python_version=='2.7'", '')
+        r2 = update.Requirement('d', '', '', '# BSD')
+        reqs = {
+            'a': update.Requirements([blank, r1]),
+            'c': update.Requirements([blank, r2])}
+        merged = update._merge_setup_cfg(old_content, reqs)
+        expected = textwrap.dedent(u"""
+            [metadata]
+            name = fred
+
+            [extras]
+            # Comment
+            a =
+              b>=1:python_version=='2.7'
+            # comment
+            c =
+              d # BSD
+
+            [entry_points]
+            console_scripts =
+                foo = bar:quux
+            """)
+        self.assertEqual(expected, merged)
+
+
+class TestCopyRequires(testtools.TestCase):
+
+    def test_extras_no_change(self):
+        global_content = textwrap.dedent(u"""\
+            foo<2;python_version=='2.7' # BSD
+            foo>1;python_version!='2.7'
+            freddy
+            """)
+        setup_cfg = textwrap.dedent(u"""\
+            [metadata]
+            name = openstack.requirements
+
+            [extras]
+            test =
+              foo<2:python_version=='2.7' # BSD
+              foo>1:python_version!='2.7'
+            opt =
+              freddy
+            """)
+        project = {}
+        project['root'] = '/dev/null'
+        project['requirements'] = {}
+        project['setup.cfg'] = setup_cfg
+        global_reqs = update._parse_reqs(global_content)
+        actions = update._copy_requires(
+            u'', False, False, project, global_reqs, False)
+        self.assertEqual([
+            update.Verbose('Syncing extra [opt]'),
+            update.Verbose('Syncing extra [test]'),
+            update.File('setup.cfg', setup_cfg)], actions)

openstack_requirements/update.py

@@ -28,13 +28,16 @@ files will be dropped.
 
 import collections
 import errno
+import io
 import itertools
 import optparse
 import os
 import os.path
 import sys
 
+from parsley import makeGrammar
 import pkg_resources
+from six.moves import configparser
 
 _setup_py_text = """#!/usr/bin/env python
 # Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
@@ -79,6 +82,25 @@ _REQS_HEADER = [
 ]
 
 
+Comment = collections.namedtuple('Comment', ['line'])
+Extra = collections.namedtuple('Extra', ['name', 'content'])
+
+
+extras_grammar = """
+ini = (line*:p extras?:e line*:l final:s) -> (''.join(p), e, ''.join(l+[s]))
+line = ~extras <(~'\\n' anything)* '\\n'>
+final = <(~'\\n' anything)* >
+extras = '[' 'e' 'x' 't' 'r' 'a' 's' ']' '\\n'+ body*:b -> b
+body = comment | extra
+comment = <'#' (~'\\n' anything)* '\\n'>:c '\\n'* -> comment(c)
+extra = name:n ' '* '=' line:l cont*:c '\\n'* -> extra(n, ''.join([l] + c))
+name = <(anything:x ?(x not in '\\n \\t='))+>
+cont = ' '+ <(~'\\n' anything)* '\\n'>
+"""
+extras_compiled = makeGrammar(
+    extras_grammar, {"comment": Comment, "extra": Extra})
+
+
 # Pure --
 class Change(object):
     def __init__(self, name, old, new):
@@ -263,17 +285,80 @@ def _copy_requires(
             non_std_reqs)
         actions.extend(_actions)
         actions.append(File(dest_name, _reqs_to_content(reqs)))
+    extras = _project_extras(project)
+    output_extras = {}
+    for extra, content in sorted(extras.items()):
+        dest_name = 'extra-%s' % extra
+        dest_path = "%s[%s]" % (project['root'], extra)
+        dest_sequence = list(_content_to_reqs(content))
+        actions.append(Verbose("Syncing extra [%s]" % extra))
+        _actions, reqs = _sync_requirements_file(
+            global_reqs, dest_sequence, dest_path, softupdate, hacking,
+            non_std_reqs)
+        actions.extend(_actions)
+        output_extras[extra] = reqs
+    dest_path = 'setup.cfg'
+    if suffix:
+        dest_path = "%s.%s" % (dest_path, suffix)
+    actions.append(File(
+        dest_path, _merge_setup_cfg(project['setup.cfg'], output_extras)))
     return actions
 
 
-def _reqs_to_content(reqs, marker_sep=';'):
+def _merge_setup_cfg(old_content, new_extras):
-    lines = list(_REQS_HEADER)
+    # This is ugly. All the existing libraries handle setup.cfg's poorly.
+    prefix, extras, suffix = extras_compiled(old_content).ini()
+    out_extras = []
+    if extras is not None:
+        for extra in extras:
+            if type(extra) is Comment:
+                out_extras.append(extra)
+            elif type(extra) is Extra:
+                if extra.name not in new_extras:
+                    out_extras.append(extra)
+                    continue
+                e = Extra(
+                    extra.name,
+                    _reqs_to_content(
+                        new_extras[extra.name], ':', '  ', False))
+                out_extras.append(e)
+            else:
+                raise TypeError('unknown type %r' % extra)
+    if out_extras:
+        extras_str = ['[extras]\n']
+        for extra in out_extras:
+            if type(extra) is Comment:
+                extras_str.append(extra.line)
+            else:
+                extras_str.append(extra.name + ' =')
+                extras_str.append(extra.content)
+        if suffix:
+            extras_str.append('\n')
+        extras_str = ''.join(extras_str)
+    else:
+        extras_str = ''
+    return prefix + extras_str + suffix
+
+
+def _project_extras(project):
+    """Return a dict of extra-name:content for the extras in setup.cfg."""
+    c = configparser.SafeConfigParser()
+    c.readfp(io.StringIO(project['setup.cfg']))
+    if not c.has_section('extras'):
+        return dict()
+    return dict(c.items('extras'))
+
+
+def _reqs_to_content(reqs, marker_sep=';', line_prefix='', prefix=True):
+    lines = []
+    if prefix:
+        lines += _REQS_HEADER
     for req in reqs.reqs:
         comment_p = ' ' if req.package else ''
         comment = (comment_p + req.comment if req.comment else '')
         marker = marker_sep + req.markers if req.markers else ''
-        lines.append(
+        package = line_prefix + req.package if req.package else ''
-            '%s%s%s%s\n' % (req.package, req.specifiers, marker, comment))
+        lines.append('%s%s%s%s\n' % (package, req.specifiers, marker, comment))
     return u''.join(lines)
 
 
@@ -315,7 +400,8 @@ def _safe_read(project, filename, output=None):
     if output is None:
         output = project
     try:
-        with open(project['root'] + '/' + filename, 'rt') as f:
+        path = project['root'] + '/' + filename
+        with io.open(path, 'rt', encoding="utf-8") as f:
             output[filename] = f.read()
     except IOError as e:
         if e.errno != errno.ENOENT:

requirements.txt

@@ -0,0 +1 @@
+Parsley

tools/integration.sh

@@ -73,6 +73,10 @@ mkdir -p $projectdir
 # Attempt to install all of global requirements
 install_all_of_gr
 
+# Install requirements
+$tmpdir/all_requirements/bin/pip install $REPODIR/requirements
+UPDATE="$tmpdir/all_requirements/bin/update-requirements"
+
 for PROJECT in $PROJECTS ; do
     SHORT_PROJECT=$(basename $PROJECT)
     if ! grep 'pbr' $REPODIR/$SHORT_PROJECT/setup.py >/dev/null 2>&1
@@ -109,7 +113,7 @@ for PROJECT in $PROJECTS ; do
 
     # set up the project synced with the global requirements
     sudo chown -R $USER $REPODIR/$SHORT_PROJECT
-    (cd $REPODIR/requirements && python update.py $REPODIR/$SHORT_PROJECT)
+    $UPDATE --source $REPODIR/requirements $REPODIR/$SHORT_PROJECT
     pushd $REPODIR/$SHORT_PROJECT
     if ! git diff --exit-code > /dev/null; then
         git commit -a -m'Update requirements'