Add support for virtualenv 20.x

virtualenv is undergoing a rewrite and has changed how the programmatic API works [1]. Switch to the "new way". While we're here, we also need to get Python 2.7 tests passing again. That requires dropping support for upper-constraints and using our own, limited local constraints based on supported Python versions. We also need to migrate integration tests since those run with Python 3 now. Update the scenarios for pip/setuptools in integration testing similarly. Finally clean up the installation of all devstack repos as requirements are managed different now. Instead of worrying about syncing them we use constraints. [1] https://github.com/pypa/virtualenv/issues/1585#issuecomment-585228492 Change-Id: I493e88985d2c4d09612fea4d20d8ffa20043a0cb Signed-off-by: Stephen Finucane <sfinucan@redhat.com> Depends-On: https://review.opendev.org/739014
2020-04-14 14:46:21 +01:00 · 2020-04-14 14:46:21 +01:00 · 73c11c6267
parent 9257b68435
commit 73c11c6267
7 changed files with 78 additions and 53 deletions
--- a/lower-constraints.txt
+++ b/lower-constraints.txt
@ -22,7 +22,7 @@ pytz==2013.6
 PyYAML==3.12
 reno==2.5.0
 requests==2.14.2
-six==1.10.0
+six==1.12.0
 snowballstemmer==1.2.1
 Sphinx==1.6.5
 sphinxcontrib-apidoc==0.2.0
@ -34,4 +34,4 @@ testscenarios==0.4
 testtools==2.2.0
 traceback2==1.4.0
 unittest2==1.1.0
-virtualenv==14.0.6
+virtualenv==20.0.3
--- a/pbr/tests/test_core.py
+++ b/pbr/tests/test_core.py
@ -40,6 +40,7 @@

 import glob
 import os
+import sys
 import tarfile

 import fixtures
@ -113,6 +114,12 @@ class TestCore(base.BaseTestCase):
    def test_console_script_develop(self):
        """Test that we develop a non-pkg-resources console script."""

+        if sys.version_info < (3, 0):
+            self.skipTest(
+                'Fails with recent virtualenv due to '
+                'https://github.com/pypa/virtualenv/issues/1638'
+            )
+
        if os.name == 'nt':
            self.skipTest('Windows support is passthrough')

--- a/pbr/tests/test_integration.py
+++ b/pbr/tests/test_integration.py
@ -11,7 +11,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
 import os.path
+import pkg_resources
 import shlex
 import sys

@ -77,19 +82,35 @@ class TestIntegration(base.BaseTestCase):
        # We don't break these into separate tests because we'd need separate
        # source dirs to isolate from side effects of running pip, and the
        # overheads of setup would start to beat the benefits of parallelism.
-        self.useFixture(base.CapturedSubprocess(
-            'sync-req',
-            ['python', 'update.py', os.path.join(REPODIR, self.short_name)],
-            cwd=os.path.join(REPODIR, 'requirements')))
-        self.useFixture(base.CapturedSubprocess(
-            'commit-requirements',
-            'git diff --quiet || git commit -amrequirements',
-            cwd=os.path.join(REPODIR, self.short_name), shell=True))
-        path = os.path.join(
-            self.useFixture(fixtures.TempDir()).path, 'project')
-        self.useFixture(base.CapturedSubprocess(
-            'clone',
-            ['git', 'clone', os.path.join(REPODIR, self.short_name), path]))
+        path = os.path.join(REPODIR, self.short_name)
+        setup_cfg = os.path.join(path, 'setup.cfg')
+        project_name = pkg_resources.safe_name(self.short_name).lower()
+        # These projects should all have setup.cfg files but we'll be careful
+        if os.path.exists(setup_cfg):
+            config = configparser.ConfigParser()
+            config.read(setup_cfg)
+            if config.has_section('metadata'):
+                raw_name = config.get('metadata', 'name',
+                                      fallback='notapackagename')
+                # Technically we should really only need to use the raw
+                # name because all our projects should be good and use
+                # normalized names but they don't...
+                project_name = pkg_resources.safe_name(raw_name).lower()
+        constraints = os.path.join(REPODIR, 'requirements',
+                                   'upper-constraints.txt')
+        tmp_constraints = os.path.join(
+            self.useFixture(fixtures.TempDir()).path,
+            'upper-constraints.txt')
+        # We need to filter out the package we are installing to avoid
+        # conflicts with the constraints.
+        with open(constraints, 'r') as src:
+            with open(tmp_constraints, 'w') as dest:
+                for line in src:
+                    constraint = line.split('===')[0]
+                    if project_name != constraint:
+                        dest.write(line)
+        pip_cmd = PIP_CMD + ['-c', tmp_constraints]
+
        venv = self.useFixture(
            test_packaging.Venv('sdist',
                                modules=['pip', 'wheel', PBRVERSION],
@ -105,7 +126,7 @@ class TestIntegration(base.BaseTestCase):
        filename = os.path.join(
            path, 'dist', os.listdir(os.path.join(path, 'dist'))[0])
        self.useFixture(base.CapturedSubprocess(
-            'tarball', [python] + PIP_CMD + [filename]))
+            'tarball', [python] + pip_cmd + [filename]))
        venv = self.useFixture(
            test_packaging.Venv('install-git',
                                modules=['pip', 'wheel', PBRVERSION],
@ -113,7 +134,7 @@ class TestIntegration(base.BaseTestCase):
        root = venv.path
        python = venv.python
        self.useFixture(base.CapturedSubprocess(
-            'install-git', [python] + PIP_CMD + ['git+file://' + path]))
+            'install-git', [python] + pip_cmd + ['git+file://' + path]))
        if self.short_name == 'nova':
            found = False
            for _, _, filenames in os.walk(root):
@ -127,7 +148,7 @@ class TestIntegration(base.BaseTestCase):
        root = venv.path
        python = venv.python
        self.useFixture(base.CapturedSubprocess(
-            'install-e', [python] + PIP_CMD + ['-e', path]))
+            'install-e', [python] + pip_cmd + ['-e', path]))


 class TestInstallWithoutPbr(base.BaseTestCase):
@ -188,12 +209,16 @@ class TestInstallWithoutPbr(base.BaseTestCase):
 class TestMarkersPip(base.BaseTestCase):

    scenarios = [
-        ('pip-1.5', {'modules': ['pip>=1.5,<1.6']}),
-        ('pip-6.0', {'modules': ['pip>=6.0,<6.1']}),
        ('pip-latest', {'modules': ['pip']}),
-        ('setuptools-EL7', {'modules': ['pip==1.4.1', 'setuptools==0.9.8']}),
-        ('setuptools-Trusty', {'modules': ['pip==1.5', 'setuptools==2.2']}),
-        ('setuptools-minimum', {'modules': ['pip==1.5', 'setuptools==0.7.2']}),
+        ('setuptools-Bionic', {
+            'modules': ['pip==9.0.1', 'setuptools==39.0.1']}),
+        ('setuptools-Stretch', {
+            'modules': ['pip==9.0.1', 'setuptools==33.1.1']}),
+        ('setuptools-EL8', {'modules': ['pip==9.0.3', 'setuptools==39.2.0']}),
+        ('setuptools-Buster', {
+            'modules': ['pip==18.1', 'setuptools==40.8.0']}),
+        ('setuptools-Focal', {
+            'modules': ['pip==20.0.2', 'setuptools==45.2.0']}),
    ]

    @testtools.skipUnless(
@ -240,25 +265,17 @@ class TestLTSSupport(base.BaseTestCase):
    # These versions come from the versions installed from the 'virtualenv'
    # command from the 'python-virtualenv' package.
    scenarios = [
-        ('EL7', {'modules': ['pip==1.4.1', 'setuptools==0.9.8'],
-                 'py3support': True}),  # And EPEL6
-        ('Trusty', {'modules': ['pip==1.5', 'setuptools==2.2'],
-                    'py3support': True}),
-        ('Jessie', {'modules': ['pip==1.5.6', 'setuptools==5.5.1'],
-                    'py3support': True}),
-        # Wheezy has pip1.1, which cannot be called with '-m pip'
-        # So we'll use a different version of pip here.
-        ('WheezyPrecise', {'modules': ['pip==1.4.1', 'setuptools==0.6c11'],
-                           'py3support': False})
+        ('Bionic', {'modules': ['pip==9.0.1', 'setuptools==39.0.1']}),
+        ('Stretch', {'modules': ['pip==9.0.1', 'setuptools==33.1.1']}),
+        ('EL8', {'modules': ['pip==9.0.3', 'setuptools==39.2.0']}),
+        ('Buster', {'modules': ['pip==18.1', 'setuptools==40.8.0']}),
+        ('Focal', {'modules': ['pip==20.0.2', 'setuptools==45.2.0']}),
    ]

    @testtools.skipUnless(
        os.environ.get('PBR_INTEGRATION', None) == '1',
        'integration tests not enabled')
    def test_lts_venv_default_versions(self):
-        if (sys.version_info[0] == 3 and not self.py3support):
-            self.skipTest('This combination will not install with py3, '
-                          'skipping test')
        venv = self.useFixture(
            test_packaging.Venv('setuptools', modules=self.modules))
        bin_python = venv.python
--- a/pbr/tests/test_packaging.py
+++ b/pbr/tests/test_packaging.py
@ -183,7 +183,8 @@ class Venv(fixtures.Fixture):

    def _setUp(self):
        path = self.useFixture(fixtures.TempDir()).path
-        virtualenv.create_environment(path, clear=True)
+        virtualenv.cli_run([path])
+
        python = os.path.join(path, 'bin', 'python')
        command = [python] + self.pip_cmd + ['-U']
        if self.modules and len(self.modules) > 0:
--- a/test-requirements.txt
+++ b/test-requirements.txt
@ -6,12 +6,13 @@ wheel>=0.32.0 # MIT
 fixtures>=3.0.0 # Apache-2.0/BSD
 hacking>=1.1.0,<1.2.0 # Apache-2.0
 mock>=2.0.0 # BSD
-six>=1.10.0 # MIT
-stestr>=2.1.0 # Apache-2.0
+six>=1.12.0 # MIT
+stestr>=2.1.0,<3.0;python_version=='2.7' # Apache-2.0
+stestr>=2.1.0;python_version>='3.0' # Apache-2.0
 testresources>=2.0.0 # Apache-2.0/BSD
 testscenarios>=0.4 # Apache-2.0/BSD
 testtools>=2.2.0 # MIT
-virtualenv>=14.0.6 # MIT
+virtualenv>=20.0.3 # MIT
 coverage!=4.4,>=4.0 # Apache-2.0

 # optionally exposed by distutils commands
--- a/tools/integration.sh
+++ b/tools/integration.sh
@ -97,10 +97,6 @@ name = test_project
 [entry_points]
 console_scripts =
    test_cmd = test_project:main
-
-[global]
-setup-hooks =
-    pbr.hooks.setup_hook
 EOF

 cat <<EOF > setup.py
@ -115,18 +111,19 @@ from socket import error as SocketError
 try:
    setuptools.setup(
        setup_requires=['pbr'],
-        pbr=True)
+        pbr=True,
+    )
 except (SocketError, Timeout):
    setuptools.setup(
        setup_requires=['pbr'],
-        pbr=True)
-
+        pbr=True,
+    )
 EOF

 mkdir test_project
 cat <<EOF > test_project/__init__.py
 def main():
-    print "Test cmd"
+    print("Test cmd")
 EOF

 epvenv=$eptest/venv
@ -165,6 +162,8 @@ export REPODIR
 export WHEELHOUSE
 export OS_TEST_TIMEOUT=600
 cd $REPODIR/pbr
-tox -epy27 --notest
-.tox/py27/bin/python -m pip install ${REPODIR}/requirements
-tox -epy27 -- test_integration
+mkvenv .venv
+source .venv/bin/activate
+pip install -r test-requirements.txt
+pip install ${REPODIR}/requirements
+stestr run --suppress-attachments test_integration
--- a/tox.ini
+++ b/tox.ini
@ -11,8 +11,9 @@ setenv =
  OS_STDOUT_CAPTURE={env:OS_STDOUT_CAPTURE:1}
  OS_STDERR_CAPTURE={env:OS_STDERR_CAPTURE:1}
  OS_TEST_TIMEOUT={env:OS_TEST_TIMEOUT:60}
+# NOTE(stephenfin): pbr intentionally does not use constraints since we support
+# a broader range of Python versions than OpenStack as a whole
 deps =
-  -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
  -r{toxinidir}/test-requirements.txt
 commands = stestr run --suppress-attachments {posargs}

@ -22,7 +23,6 @@ commands = flake8 {posargs}
 [testenv:docs]
 whitelist_externals = rm
 deps =
-  -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
  -r{toxinidir}/doc/requirements.txt
 commands =
  rm -rf doc/build doc/source/reference/api