From 4469a813cb68d99b81440dd33f5f7e9a1f892c27 Mon Sep 17 00:00:00 2001
From: Chris Dent <cdent@anticdent.org>
Date: Wed, 31 Oct 2018 15:44:16 +0000
Subject: [PATCH] Correct lower-constraints.txt and the related tox job

While exploring removing unused packages from lower-constraints.txt it
became clear that the lower-constraints job was not working as expected:
Because our tox config has usedevelop=True 'setup.py develop' is called
to install the placement package after the install command is called.
This means that the lower-constraints are clobbered.

I had mistakenly assumed that turning off 'usedevelop', which causes
'setup.py install' would not make any difference, because it usually
installs dependencies too. It turns out however, that when using pbr
and within a git working dir, it does not. That took some time to
figure out. Oh well.

This change makes it so that we create the tox environment using
usedevelop=False and with our own install_command, to avoid upper
constraints conflicting with lower constraints.

This flagged up a few changes, the main one being that we did not have
a new enough version of keystonemiddleware in order to require use of
www_authenticate_uri. requirements.txt is updated for this as well.

And PasteDeploy needed to be updated to work with Python 3's notion
of namespace packages.

psycopg2 need a newer version to work with Postgresql 10.

oslotest needs to be raised to 3.4.0 because the tests in
cmd.test_manage use features to control what is capture by the Output
fixture from oslotest. Note that the lower-constraints job found
this problem and also demonstrates why we must run the lower-constraints
job without upper-constraints being involved. upper-constraints will
"win" and we don't want that. The point of the job is find packages
where lower-constraints are wrong, so it must "win".

The end result here is a lower-constraints.txt file that starts from
the lower-constraints.txt defined by nova, and then is adapted to
update the versions of packages that were not up to date, remove
those packages which are no longer present, and add some that are
now required.

Change-Id: Id66a28f7ace6fc2adf0e1201d9de5f901234d870
---
 lower-constraints.txt | 78 ++++---------------------------------------
 requirements.txt      |  2 +-
 test-requirements.txt |  2 +-
 tox.ini               | 10 ++++++
 4 files changed, 18 insertions(+), 74 deletions(-)

diff --git a/lower-constraints.txt b/lower-constraints.txt
index 0dbb17383..7d5a8d7aa 100644
--- a/lower-constraints.txt
+++ b/lower-constraints.txt
@@ -1,31 +1,15 @@
 alembic==0.9.8
 amqp==2.2.2
 appdirs==1.4.3
-asn1crypto==0.24.0
 attrs==17.4.0
-automaton==1.14.0
 Babel==2.3.4
-enum34==1.0.4
 bandit==1.1.0
-bcrypt==3.1.4
 cachetools==2.0.1
-castellan==0.16.0
-certifi==2018.1.18
-cffi==1.11.5
-chardet==3.0.4
-cliff==2.11.0
-cmd2==0.8.1
 colorama==0.3.9
 contextlib2==0.5.5
 coverage==4.0
-cryptography==2.1
-cursive==0.2.1
-ddt==1.0.1
 debtcollector==1.19.0
 decorator==3.4.0
-deprecation==2.0
-dogpile.cache==0.6.5
-enum-compat==0.0.2
 eventlet==0.18.2
 extras==1.0.0
 fasteners==0.14.1
@@ -33,26 +17,20 @@ fixtures==3.0.0
 flake8==2.5.5
 future==0.16.0
 futurist==1.6.0
-futures==3.0.0
 gabbi==1.35.0
 gitdb2==2.0.3
 GitPython==2.1.8
 greenlet==0.4.10
 hacking==0.12.0
-idna==2.6
 iso8601==0.1.11
 Jinja2==2.10
-jmespath==0.9.3
-jsonpatch==1.21
-jsonpath-rw-ext==1.1.3
 jsonpath-rw==1.4.0
-jsonpointer==2.0
+jsonpath-rw-ext==1.1.3
 jsonschema==2.6.0
 keystoneauth1==3.9.0
-keystonemiddleware==4.17.0
+keystonemiddleware==4.18.0
 kombu==4.1.0
 linecache2==1.0.0
-lxml==3.4.1
 Mako==1.0.7
 MarkupSafe==1.0
 mccabe==0.2.1
@@ -60,22 +38,12 @@ microversion-parse==0.2.1
 mock==2.0.0
 monotonic==1.4
 mox3==0.20.0
-msgpack==0.5.6
-munch==2.2.0
+msgpack-python==0.5.6
 netaddr==0.7.18
 netifaces==0.10.4
-networkx==1.11
-numpy==1.14.2
-openstacksdk==0.12.0
-os-brick==2.5.0
 os-client-config==1.29.0
 os-service-types==1.2.0
 os-traits==0.4.0
-os-vif==1.7.0
-os-win==3.0.0
-os-xenapi==0.3.3
-osc-lib==1.10.0
-oslo.cache==1.26.0
 oslo.concurrency==3.26.0
 oslo.config==6.1.0
 oslo.context==2.19.2
@@ -85,65 +53,39 @@ oslo.log==3.36.0
 oslo.messaging==6.3.0
 oslo.middleware==3.31.0
 oslo.policy==1.35.0
-oslo.privsep==1.23.0
-oslo.reports==1.18.0
-oslo.rootwrap==5.8.0
 oslo.serialization==2.18.0
 oslo.service==1.24.0
 oslo.upgradecheck==0.1.1
 oslo.utils==3.37.0
 oslo.versionedobjects==1.31.2
-oslo.vmware==2.17.0
-oslotest==3.2.0
-osprofiler==1.4.0
-packaging==17.1
-paramiko==2.0.0
+oslotest==3.4.0
 Paste==2.0.2
-PasteDeploy==1.5.0
+PasteDeploy==1.5.2
 pbr==2.0.0
 pep8==1.5.7
-pika-pool==0.1.3
-pika==0.10.0
 pluggy==0.6.0
 ply==3.11
 prettytable==0.7.1
-psutil==3.2.2
-psycopg2==2.6.2
+psycopg2==2.7
 py==1.5.2
-pyasn1-modules==0.2.1
-pyasn1==0.4.2
 pycadf==2.7.0
-pycparser==2.18
 pyflakes==0.8.1
 pyinotify==0.9.6
 PyMySQL==0.7.6
-PyNaCl==1.2.1
-pyOpenSSL==17.5.0
 pyparsing==2.2.0
-pyperclip==1.6.0
-pypowervm==1.1.15
-pyroute2==0.4.21
 pytest==3.4.2
-python-barbicanclient==4.5.2
-python-cinderclient==3.3.0
 python-dateutil==2.5.3
 python-editor==1.0.3
-python-glanceclient==2.8.0
-python-ironicclient==2.3.0
 python-keystoneclient==3.15.0
 python-mimeparse==1.6.0
-python-neutronclient==6.7.0
 python-subunit==1.2.0
 pytz==2018.3
 PyYAML==3.12
 repoze.lru==0.7
-requests-mock==1.2.0
 requests==2.14.2
 requestsexceptions==1.4.0
-retrying==1.3.3
 rfc3986==0.3.1
 Routes==2.3.1
-simplejson==3.13.2
 six==1.10.0
 smmap2==2.0.3
 SQLAlchemy==1.0.10
@@ -151,24 +93,16 @@ sqlparse==0.2.4
 statsd==3.2.2
 stestr==1.0.0
 stevedore==1.20.0
-setuptools==21.0.0
-suds-jurko==0.6
-taskflow==2.16.0
 Tempita==0.5.2
 tenacity==4.9.0
 testrepository==0.0.20
 testresources==2.0.0
 testscenarios==0.4
 testtools==2.2.0
-tooz==1.58.0
 traceback2==1.4.0
 unittest2==1.1.0
 urllib3==1.22
 vine==1.1.4
-voluptuous==0.11.1
-warlock==1.3.0
 WebOb==1.8.2
-websockify==0.8.0
 wrapt==1.10.11
 wsgi-intercept==1.7.0
-zVMCloudConnector==1.1.1
diff --git a/requirements.txt b/requirements.txt
index b46d39918..e006da98e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,7 @@
 
 pbr!=2.1.0,>=2.0.0 # Apache-2.0
 SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT
-keystonemiddleware>=4.17.0 # Apache-2.0
+keystonemiddleware>=4.18.0 # Apache-2.0
 Routes>=2.3.1 # MIT
 WebOb>=1.8.2 # MIT
 jsonschema<3.0.0,>=2.6.0 # MIT
diff --git a/test-requirements.txt b/test-requirements.txt
index 10fe251b7..1b4d1249b 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -8,7 +8,7 @@ fixtures>=3.0.0 # Apache-2.0/BSD
 mock>=2.0.0 # BSD
 psycopg2>=2.6.2 # LGPL/ZPL
 PyMySQL>=0.7.6 # MIT License
-oslotest>=3.2.0 # Apache-2.0
+oslotest>=3.4.0 # Apache-2.0
 stestr>=1.0.0 # Apache-2.0
 testtools>=2.2.0 # MIT
 bandit>=1.1.0 # Apache-2.0
diff --git a/tox.ini b/tox.ini
index 4b7409a91..9108c529e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -194,7 +194,17 @@ commands =
 
 [testenv:lower-constraints]
 basepython = python3
+# When using pbr and in a git repo, 'setup.py install' does not install
+# packages. 'setup.py develop', used when usedevelop is True, does.
+usedevelop = False
+# Use our own install_command to turn off upper constraints, which conflicts
+# with lower constraints.
+install_command = pip install {opts} {packages}
 deps =
   -c{toxinidir}/lower-constraints.txt
   -r{toxinidir}/test-requirements.txt
   -r{toxinidir}/requirements.txt
+# Test with both functional and unit tests.
+commands =
+  {[testenv]commands}
+  stestr --test-path=./placement/tests run {posargs}