Fix things up to work with nova.

Add extern/six back - turns out setup_requires isn't transitive.
Make sure .gitignore is right.

.gitignore

@@ -25,3 +25,5 @@ develop-eggs
 .*.swp
 .coverage
 cover
+AUTHORS
+ChangeLog

oslo/packaging/core.py

@@ -13,23 +13,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# Copyright (C) 2013 Association of Universities for Research in Astronomy (AURA)
+# Copyright (C) 2013 Association of Universities for Research in Astronomy
+#                    (AURA)
 
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
-# 
+#
 #     1. Redistributions of source code must retain the above copyright
 #        notice, this list of conditions and the following disclaimer.
-# 
+#
 #     2. Redistributions in binary form must reproduce the above
 #        copyright notice, this list of conditions and the following
 #        disclaimer in the documentation and/or other materials provided
 #        with the distribution.
-# 
+#
 #     3. The name of AURA and its representatives may not be used to
 #        endorse or promote products derived from this software without
 #        specific prior written permission.
-# 
+#
 # THIS SOFTWARE IS PROVIDED BY AURA ``AS IS'' AND ANY EXPRESS OR IMPLIED
 # WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
@@ -48,13 +49,16 @@ import os
 import sys
 import warnings
 
+from distutils import log
+import pkg_resources
 from setuptools.dist import _get_unpatched
-import six
+from .extern import six
 
 from oslo.packaging import util
-
+from oslo.packaging import packaging
 
 _Distribution = _get_unpatched(_Distribution)
+log.set_verbosity(log.INFO)
 
 
 def setup(dist, attr, value):
@@ -72,6 +76,7 @@ def setup(dist, attr, value):
     not work well with distributions that do use a `Distribution` subclass.
     """
 
+    log.info("[oslo.packaging] Processing setup.cfg")
     if not value:
         return
     path = os.path.abspath('setup.cfg')
@@ -94,7 +99,12 @@ def setup(dist, attr, value):
 
         # Handle additional setup processing
         if 'setup_requires' in attrs:
-            attrs = chainload_setups(dist, attrs['setup_requires'], attrs)
+            chainload_setups(dist, attrs['setup_requires'])
+
+        for ep in pkg_resources.iter_entry_points(
+                'oslo.packaging.attr_filters'):
+            filter_method = ep.load()
+            attrs = filter_method(attrs)
 
         # Skips 'options' and 'licence' support which are rarely used; may add
         # back in later if demanded
@@ -117,10 +127,8 @@ def setup(dist, attr, value):
         # Some people apparently take "version number" too literally :)
         dist.metadata.version = str(dist.metadata.version)
 
-    dist.command_options = util.DefaultGetDict(lambda: util.IgnoreDict(ignore))
 
-
-def chainload_setups(dist, requires_list, attrs):
+def chainload_setups(dist, requires_list):
     try:
         import pip.command.install
     except ImportError:
@@ -132,13 +140,6 @@ def chainload_setups(dist, requires_list, attrs):
         cmd.ensure_finalized()
         cmd.easy_install("req")
         import pip.command.install
-    import pkg_resources
 
     pip_install = pip.command.install.InstallCommand()
     pip_install.run({}, requires_list)
-
-    for ep in pkg_resources.iter_entry_points('oslo.packaging.attr_filters'):
-        filter_method = ep.load()
-        attrs = filter_method(attrs)
-
-    return attrs

oslo/packaging/extern/six.py

@@ -0,0 +1,386 @@
+# Copyright (c) 2010-2011 Benjamin Peterson
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Utilities for writing code that runs on Python 2 and 3"""
+
+import operator
+import sys
+import types
+
+__author__ = "Benjamin Peterson <benjamin@python.org>"
+__version__ = "1.2.0"
+
+
+# True if we are running on Python 3.
+PY3 = sys.version_info[0] == 3
+
+if PY3:
+    string_types = str,
+    integer_types = int,
+    class_types = type,
+    text_type = str
+    binary_type = bytes
+
+    MAXSIZE = sys.maxsize
+else:
+    string_types = basestring,
+    integer_types = (int, long)
+    class_types = (type, types.ClassType)
+    text_type = unicode
+    binary_type = str
+
+    if sys.platform == "java":
+        # Jython always uses 32 bits.
+        MAXSIZE = int((1 << 31) - 1)
+    else:
+        # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
+        class X(object):
+            def __len__(self):
+                return 1 << 31
+        try:
+            len(X())
+        except OverflowError:
+            # 32-bit
+            MAXSIZE = int((1 << 31) - 1)
+        else:
+            # 64-bit
+            MAXSIZE = int((1 << 63) - 1)
+            del X
+
+
+def _add_doc(func, doc):
+    """Add documentation to a function."""
+    func.__doc__ = doc
+
+
+def _import_module(name):
+    """Import module, returning the module after the last dot."""
+    __import__(name)
+    return sys.modules[name]
+
+
+class _LazyDescr(object):
+
+    def __init__(self, name):
+        self.name = name
+
+    def __get__(self, obj, tp):
+        result = self._resolve()
+        setattr(obj, self.name, result)
+        # This is a bit ugly, but it avoids running this again.
+        delattr(tp, self.name)
+        return result
+
+
+class MovedModule(_LazyDescr):
+
+    def __init__(self, name, old, new=None):
+        super(MovedModule, self).__init__(name)
+        if PY3:
+            if new is None:
+                new = name
+            self.mod = new
+        else:
+            self.mod = old
+
+    def _resolve(self):
+        return _import_module(self.mod)
+
+
+class MovedAttribute(_LazyDescr):
+
+    def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
+        super(MovedAttribute, self).__init__(name)
+        if PY3:
+            if new_mod is None:
+                new_mod = name
+            self.mod = new_mod
+            if new_attr is None:
+                if old_attr is None:
+                    new_attr = name
+                else:
+                    new_attr = old_attr
+            self.attr = new_attr
+        else:
+            self.mod = old_mod
+            if old_attr is None:
+                old_attr = name
+            self.attr = old_attr
+
+    def _resolve(self):
+        module = _import_module(self.mod)
+        return getattr(module, self.attr)
+
+
+
+class _MovedItems(types.ModuleType):
+    """Lazy loading of moved objects"""
+
+
+_moved_attributes = [
+    MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
+    MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
+    MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
+    MovedAttribute("map", "itertools", "builtins", "imap", "map"),
+    MovedAttribute("reload_module", "__builtin__", "imp", "reload"),
+    MovedAttribute("reduce", "__builtin__", "functools"),
+    MovedAttribute("StringIO", "StringIO", "io"),
+    MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
+    MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
+
+    MovedModule("builtins", "__builtin__"),
+    MovedModule("configparser", "ConfigParser"),
+    MovedModule("copyreg", "copy_reg"),
+    MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
+    MovedModule("http_cookies", "Cookie", "http.cookies"),
+    MovedModule("html_entities", "htmlentitydefs", "html.entities"),
+    MovedModule("html_parser", "HTMLParser", "html.parser"),
+    MovedModule("http_client", "httplib", "http.client"),
+    MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
+    MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
+    MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
+    MovedModule("cPickle", "cPickle", "pickle"),
+    MovedModule("queue", "Queue"),
+    MovedModule("reprlib", "repr"),
+    MovedModule("socketserver", "SocketServer"),
+    MovedModule("tkinter", "Tkinter"),
+    MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
+    MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
+    MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
+    MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
+    MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
+    MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
+    MovedModule("tkinter_colorchooser", "tkColorChooser",
+                "tkinter.colorchooser"),
+    MovedModule("tkinter_commondialog", "tkCommonDialog",
+                "tkinter.commondialog"),
+    MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_font", "tkFont", "tkinter.font"),
+    MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
+    MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
+                "tkinter.simpledialog"),
+    MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
+    MovedModule("winreg", "_winreg"),
+]
+for attr in _moved_attributes:
+    setattr(_MovedItems, attr.name, attr)
+del attr
+
+moves = sys.modules["six.moves"] = _MovedItems("moves")
+
+
+def add_move(move):
+    """Add an item to six.moves."""
+    setattr(_MovedItems, move.name, move)
+
+
+def remove_move(name):
+    """Remove item from six.moves."""
+    try:
+        delattr(_MovedItems, name)
+    except AttributeError:
+        try:
+            del moves.__dict__[name]
+        except KeyError:
+            raise AttributeError("no such move, %r" % (name,))
+
+
+if PY3:
+    _meth_func = "__func__"
+    _meth_self = "__self__"
+
+    _func_code = "__code__"
+    _func_defaults = "__defaults__"
+
+    _iterkeys = "keys"
+    _itervalues = "values"
+    _iteritems = "items"
+else:
+    _meth_func = "im_func"
+    _meth_self = "im_self"
+
+    _func_code = "func_code"
+    _func_defaults = "func_defaults"
+
+    _iterkeys = "iterkeys"
+    _itervalues = "itervalues"
+    _iteritems = "iteritems"
+
+
+try:
+    advance_iterator = next
+except NameError:
+    def advance_iterator(it):
+        return it.next()
+next = advance_iterator
+
+
+if PY3:
+    def get_unbound_function(unbound):
+        return unbound
+
+    Iterator = object
+
+    def callable(obj):
+        return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
+else:
+    def get_unbound_function(unbound):
+        return unbound.im_func
+
+    class Iterator(object):
+
+        def next(self):
+            return type(self).__next__(self)
+
+    callable = callable
+_add_doc(get_unbound_function,
+         """Get the function out of a possibly unbound function""")
+
+
+get_method_function = operator.attrgetter(_meth_func)
+get_method_self = operator.attrgetter(_meth_self)
+get_function_code = operator.attrgetter(_func_code)
+get_function_defaults = operator.attrgetter(_func_defaults)
+
+
+def iterkeys(d):
+    """Return an iterator over the keys of a dictionary."""
+    return iter(getattr(d, _iterkeys)())
+
+def itervalues(d):
+    """Return an iterator over the values of a dictionary."""
+    return iter(getattr(d, _itervalues)())
+
+def iteritems(d):
+    """Return an iterator over the (key, value) pairs of a dictionary."""
+    return iter(getattr(d, _iteritems)())
+
+
+if PY3:
+    def b(s):
+        return s.encode("latin-1")
+    def u(s):
+        return s
+    if sys.version_info[1] <= 1:
+        def int2byte(i):
+            return bytes((i,))
+    else:
+        # This is about 2x faster than the implementation above on 3.2+
+        int2byte = operator.methodcaller("to_bytes", 1, "big")
+    import io
+    StringIO = io.StringIO
+    BytesIO = io.BytesIO
+else:
+    def b(s):
+        return s
+    def u(s):
+        return unicode(s, "unicode_escape")
+    int2byte = chr
+    import StringIO
+    StringIO = BytesIO = StringIO.StringIO
+_add_doc(b, """Byte literal""")
+_add_doc(u, """Text literal""")
+
+
+if PY3:
+    import builtins
+    exec_ = getattr(builtins, "exec")
+
+
+    def reraise(tp, value, tb=None):
+        if value.__traceback__ is not tb:
+            raise value.with_traceback(tb)
+        raise value
+
+
+    print_ = getattr(builtins, "print")
+    del builtins
+
+else:
+    def exec_(code, globs=None, locs=None):
+        """Execute code in a namespace."""
+        if globs is None:
+            frame = sys._getframe(1)
+            globs = frame.f_globals
+            if locs is None:
+                locs = frame.f_locals
+            del frame
+        elif locs is None:
+            locs = globs
+        exec("""exec code in globs, locs""")
+
+
+    exec_("""def reraise(tp, value, tb=None):
+    raise tp, value, tb
+""")
+
+
+    def print_(*args, **kwargs):
+        """The new-style print function."""
+        fp = kwargs.pop("file", sys.stdout)
+        if fp is None:
+            return
+        def write(data):
+            if not isinstance(data, basestring):
+                data = str(data)
+            fp.write(data)
+        want_unicode = False
+        sep = kwargs.pop("sep", None)
+        if sep is not None:
+            if isinstance(sep, unicode):
+                want_unicode = True
+            elif not isinstance(sep, str):
+                raise TypeError("sep must be None or a string")
+        end = kwargs.pop("end", None)
+        if end is not None:
+            if isinstance(end, unicode):
+                want_unicode = True
+            elif not isinstance(end, str):
+                raise TypeError("end must be None or a string")
+        if kwargs:
+            raise TypeError("invalid keyword arguments to print()")
+        if not want_unicode:
+            for arg in args:
+                if isinstance(arg, unicode):
+                    want_unicode = True
+                    break
+        if want_unicode:
+            newline = unicode("\n")
+            space = unicode(" ")
+        else:
+            newline = "\n"
+            space = " "
+        if sep is None:
+            sep = space
+        if end is None:
+            end = newline
+        for i, arg in enumerate(args):
+            if i:
+                write(sep)
+            write(arg)
+        write(end)
+
+_add_doc(reraise, """Reraise an exception.""")
+
+
+def with_metaclass(meta, base=object):
+    """Create a base class with a metaclass."""
+    return meta("NewBase", (base,), {})

oslo/packaging/packaging.py

@@ -26,8 +26,12 @@ import re
 import subprocess
 import sys
 
+from distutils import log
+import setuptools
 from setuptools.command import sdist
 
+log.set_verbosity(log.INFO)
+
 
 def parse_mailmap(mailmap='.mailmap'):
     mapping = {}
@@ -145,11 +149,12 @@ def _get_git_directory():
 
 def write_git_changelog():
     """Write a changelog based on the git changelog."""
+    log.info('[oslo.packaging] Writing ChangeLog')
     new_changelog = 'ChangeLog'
     git_dir = _get_git_directory()
     if not os.getenv('SKIP_WRITE_GIT_CHANGELOG'):
         if git_dir:
-            git_log_cmd = 'git --git-dir=%s log --stat' % git_dir
+            git_log_cmd = 'git --git-dir=%s log' % git_dir
             changelog = _run_shell_command(git_log_cmd)
             mailmap = _parse_git_mailmap(git_dir)
             with open(new_changelog, "w") as changelog_file:
@@ -160,6 +165,7 @@ def write_git_changelog():
 
 def generate_authors():
     """Create AUTHORS file using git commits."""
+    log.info('[oslo.packaging] Generating AUTHORS')
     jenkins_email = 'jenkins@review.(openstack|stackforge).org'
     old_authors = 'AUTHORS.in'
     new_authors = 'AUTHORS'
@@ -224,7 +230,8 @@ def get_cmdclass():
             builders = ['html', 'man']
 
             def generate_autoindex(self):
-                print "**Autodocumenting from %s" % os.path.abspath(os.curdir)
+                log.info("[oslo.packaging] Autodocumenting from %s"
+                         % os.path.abspath(os.curdir))
                 modules = {}
                 option_dict = self.distribution.get_option_dict('build_sphinx')
                 source_dir = os.path.join(option_dict['source_dir'][1], 'api')
@@ -249,7 +256,8 @@ def get_cmdclass():
                         values = dict(module=module, heading=heading,
                                       underline=underline)
 
-                        print "Generating %s" % output_filename
+                        log.info("[oslo.packaging] Generating %s"
+                                 % output_filename)
                         with open(output_filename, 'w') as output_file:
                             output_file.write(_rst_template % values)
                         autoindex.write("   %s.rst\n" % module)
@@ -359,6 +367,17 @@ def get_version(package_name, pre_version=None):
                     " tarball, or access to an upstream git repository.")
 
 
+def smart_find_packages(package_list):
+    """Run find_packages the way we intend."""
+    packages = []
+    for pkg in package_list:
+        pkg_path = pkg.replace('.', os.path.sep)
+        packages.append(pkg)
+        packages.extend(['%s.%s' % (pkg, f)
+                         for f in setuptools.find_packages(pkg_path)])
+    return list(set(packages))
+
+
 def attr_filter(attrs):
     """Filter attrs parsed from a setup.cfg to inject our defaults."""
     attrs['version'] = get_version(attrs['name'], attrs.get('version', None))
@@ -366,4 +385,5 @@ def attr_filter(attrs):
     attrs['install_requires'] = parse_requirements()
     attrs['dependency_links'] = parse_dependency_links()
     attrs['include_package_data'] = True
+    attrs['packages'] = smart_find_packages(attrs['packages'])
     return attrs

oslo/packaging/util.py

@@ -79,9 +79,9 @@ from setuptools.dist import Distribution
 from setuptools.extension import Extension
 
 try:
-    import configparser.RawConfigParser as RawConfigParser
+    from  ConfigParser import RawConfigParser
 except ImportError:
-    import ConfigParser.RawConfigParser as RawConfigParser
+    from configparser import RawConfigParser
 
 from oslo.packaging import packaging
 
@@ -274,9 +274,9 @@ def cfg_to_args(path='setup.cfg'):
     return kwargs
 
 
-def filtered_args(attr):
+def filtered_args(path='setup.cfg'):
     """Process and pass on the attrs dict."""
-    return packaging.attr_filter(attr)
+    return packaging.attr_filter(cfg_to_args(path))
 
 
 def setup_cfg_to_setup_kwargs(config):

setup.cfg

@@ -1,5 +1,6 @@
 [metadata]
 name = oslo.packaging
+version = 0.1
 author = OpenStack
 author-email = openstack-dev@lists.openstack.org
 summary = OpenStack's setup automation in a reusable form
@@ -19,7 +20,6 @@ classifier =
 [files]
 packages =
     oslo
-    oslo.packaging
 namespace_packages =
     oslo
 

setup.py

@@ -14,10 +14,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from setuptools import setup
+import setuptools
 
 # See setup.cfg for the project metadata.
-from oslo.setup import util import filtered_args
+from oslo.packaging import util
 
 
-setup(**filtered_args())
+# Use our internals directly, so that we don't chicken-and-egg needing to
+# install an entry point before using ourself.
+setuptools.setup(**util.filtered_args())