openstack
/
pbr
Code Issues Proposed changes

Teach pbr to read extras and env markers 

This adds support for reading extras from setup.cfg.

It also adds support for handling environment markers, both in the
extras section and in install_requires and in requirements.txt.

Change-Id: I6fd8276012e65f82934df9c374613b1ce6856b5a
This commit is contained in:
James Polley 2015-05-11 15:19:37 +10:00
parent 44ee5f0d0c
commit 2b29c4fc2b
5 changed files with 235 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
doc/source/index.rst
View File

@ -123,6 +123,45 @@ version number used to install the package):
Only the first file found is used to install the list of packages it
contains.
Extra requirements
------------------
Groups of optional dependencies (`"extra" requirements
<https://www.python.org/dev/peps/pep-0426/#extras-optional-dependencies>`_)
can be described in your setup.cfg, rather than needing to be added to
setup.py. An example (which also demonstrates the use of environment
markers) is shown below.
Environment markers
-------------------
Environment markers are `conditional dependencies
<https://www.python.org/dev/peps/pep-0426/#environment-markers>`_
which can be added to the requirements (or to a group of extra
requirements) automatically, depending on the environment the
installer is running in. They can be added to requirements in the
requirements file, or to extras definied in setup.cfg - but the format
is slightly different for each.
For ``requirements.txt``::
argparse; python=='2.6'
will result in the package depending on ``argparse`` only if it's being
installed into python2.6
For extras specifed in setup.cfg, add an ``extras`` section. For
instance, to create two groups of extra requirements with additional
constraints on the environment, you can use::
[extras]
security =
aleph
bet :python_environment=='3.2'
gimel :python_environment=='2.7'
testing =
quux :python_environment=='2.7'
long_description
----------------

56
pbr/tests/test_packaging.py
View File

@ -40,10 +40,14 @@
import os
import re
import sys
import tempfile
import textwrap
import fixtures
import mock
import pkg_resources
import six
import testscenarios
from testtools import matchers
@ -417,5 +421,57 @@ class TestVersions(base.BaseTestCase):
self.assertEqual('1.3.0.0a1', version)
class TestRequirementParsing(base.BaseTestCase):
def test_requirement_parsing(self):
tempdir = self.useFixture(fixtures.TempDir()).path
requirements = os.path.join(tempdir, 'requirements.txt')
with open(requirements, 'wt') as f:
f.write(textwrap.dedent(six.u("""\
bar
quux<1.0; python_version=='2.6'
""")))
setup_cfg = os.path.join(tempdir, 'setup.cfg')
with open(setup_cfg, 'wt') as f:
f.write(textwrap.dedent(six.u("""\
[metadata]
name = test_reqparse
[extras]
test =
foo
baz>3.2 :python_version=='2.7'
""")))
# pkg_resources.split_sections uses None as the title of an
# anonymous section instead of the empty string. Weird.
expected_requirements = {
None: ['bar'],
":python_version=='2.6'": ['quux<1.0'],
"test:python_version=='2.7'": ['baz>3.2'],
"test": ['foo']
}
setup_py = os.path.join(tempdir, 'setup.py')
with open(setup_py, 'wt') as f:
f.write(textwrap.dedent(six.u("""\
#!/usr/bin/env python
import setuptools
setuptools.setup(
setup_requires=['pbr'],
pbr=True,
)
""")))
self._run_cmd(sys.executable, (setup_py, 'egg_info'),
allow_fail=False, cwd=tempdir)
egg_info = os.path.join(tempdir, 'test_reqparse.egg-info')
requires_txt = os.path.join(egg_info, 'requires.txt')
with open(requires_txt, 'rt') as requires:
generated_requirements = dict(
pkg_resources.split_sections(requires))
self.assertEqual(expected_requirements, generated_requirements)
def load_tests(loader, in_tests, pattern):
return testscenarios.load_tests_apply_scenarios(loader, in_tests, pattern)

80
pbr/tests/test_util.py Normal file
View File

@ -0,0 +1,80 @@
# Copyright (c) 2015 Hewlett-Packard Development Company, L.P. (HP)
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import io
import textwrap
import six
from six.moves import configparser
import testscenarios
from pbr.tests import base
from pbr import util
class TestExtrasRequireParsingScenarios(base.BaseTestCase):
scenarios = [
('simple_extras', {
'config_text': """
[extras]
first =
foo
bar==1.0
second =
baz>=3.2
foo
""",
'expected_extra_requires': {'first': ['foo', 'bar==1.0'],
'second': ['baz>=3.2', 'foo']}
}),
('with_markers', {
'config_text': """
[extras]
test =
foo:python_version=='2.6'
bar
baz<1.6 :python_version=='2.6'
""",
'expected_extra_requires': {
"test:python_version=='2.6'": ['foo', 'baz<1.6'],
"test": ['bar']}}),
('no_extras', {
'config_text': """
[metadata]
long_description = foo
""",
'expected_extra_requires':
{}
})]
def config_from_ini(self, ini):
config = {}
parser = configparser.SafeConfigParser()
ini = textwrap.dedent(six.u(ini))
parser.readfp(io.StringIO(ini))
for section in parser.sections():
config[section] = dict(parser.items(section))
return config
def test_extras_parsing(self):
config = self.config_from_ini(self.config_text)
kwargs = util.setup_cfg_to_setup_kwargs(config)
self.assertEqual(self.expected_extra_requires,
kwargs['extras_require'])
def load_tests(loader, in_tests, pattern):
return testscenarios.load_tests_apply_scenarios(loader, in_tests, pattern)

59
pbr/util.py
View File

@ -280,6 +280,10 @@ def setup_cfg_to_setup_kwargs(config):
kwargs = {}
# Temporarily holds install_reqires and extra_requires while we
# parse env_markers.
all_requirements = {}
for arg in D1_D2_SETUP_ARGS:
if len(D1_D2_SETUP_ARGS[arg]) == 2:
# The distutils field name is different than distutils2's.
@ -326,6 +330,17 @@ def setup_cfg_to_setup_kwargs(config):
# setuptools
in_cfg_value = [_VERSION_SPEC_RE.sub(r'\1\2', pred)
for pred in in_cfg_value]
if arg == 'install_requires':
# Split install_requires into package,env_marker tuples
# These will be re-assembled later
install_requires = []
requirement_pattern = '(?P<package>[^;]*);?(?P<env_marker>.*)$'
for requirement in in_cfg_value:
m = re.match(requirement_pattern, requirement)
requirement_package = m.group('package').strip()
env_marker = m.group('env_marker').strip()
install_requires.append((requirement_package,env_marker))
all_requirements[''] = install_requires
elif arg == 'package_dir':
in_cfg_value = {'': in_cfg_value}
elif arg in ('package_data', 'data_files'):
@ -367,6 +382,50 @@ def setup_cfg_to_setup_kwargs(config):
kwargs[arg] = in_cfg_value
# Transform requirements with embedded environment markers to
# setuptools' supported marker-per-requirement format.
#
# install_requires are treated as a special case of extras, before
# being put back in the expected place
#
# fred =
# foo:marker
# bar
# -> {'fred': ['bar'], 'fred:marker':['foo']}
if 'extras' in config:
requirement_pattern = '(?P<package>[^:]*):?(?P<env_marker>.*)$'
extras = config['extras']
for extra in extras:
extra_requirements = []
requirements = split_multiline(extras[extra])
for requirement in requirements:
m = re.match(requirement_pattern, requirement)
extras_value = m.group('package').strip()
env_marker = m.group('env_marker')
extra_requirements.append((extras_value,env_marker))
all_requirements[extra] = extra_requirements
# Transform the full list of requirements into:
# - install_requires, for those that have no extra and no
# env_marker
# - named extras, for those with an extra name (which may include
# an env_marker)
# - and as a special case, install_requires with an env_marker are
# treated as named extras where the name is the empty string
extras_require = {}
for req_group in all_requirements:
for requirement, env_marker in all_requirements[req_group]:
if env_marker:
extras_key = '%s:%s' % (req_group, env_marker)
else:
extras_key = req_group
extras_require.setdefault(extras_key, []).append(requirement)
kwargs['install_requires'] = extras_require.pop('', [])
kwargs['extras_require'] = extras_require
return kwargs

1
test-requirements.txt
View File

@ -5,6 +5,7 @@ hacking>=0.9.2,<0.10
mock>=1.0
python-subunit>=0.0.18
sphinx>=1.1.2,<1.2
six>=1.9.0
testrepository>=0.0.18
testresources>=0.2.4
testscenarios>=0.4