From e2bed1b72fcac3b689951372e4ac33784736c9aa Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Wed, 28 Jul 2021 11:19:57 +1000 Subject: [PATCH] Revert "Workaround for new pip 20.3 behavior" This reverts commit 7a3a7ce876a37376fe0dca7278e41a4f46867daa and bcd0acf6c0b5d6501e91133c3a937b3fc40f7122 and part of f1ed7c77c50ac28cb58c9f7ed885c6a3e0a75403 which all cap our pip installs. Given the pip ecosystem can often incorporate major changes, tracking upstream at least generally gives us one problem at a time to solve rather than trying to handle version jumps when LTS distros update. The new dependency resolver included some changes that disallow setting URL's like "file:///path/to/project#egg=project" in constraints. Apparently the fact it used to work was an accident of the requires/constraints mechanism; it does make some sense as the URL doesn't really have a version-number that the resolver can put in an ordering graph. The _setup_package_with_constraints_edit function comment highlights what this is trying to do # Updates the constraints from REQUIREMENTS_DIR to reflect the # future installed state of this package. This ensures when we # install this package we get the from source version. In other words; if constraints has "foo==1.2.3" and Zuul has checked out "foo" for testing, we have to make sure pip doesn't choose version 1.2.3 from pypi. It seems like removing the entry from upper-requirements.txt is the important part; adding the URL path to the on-disk version was just something that seemed to work at the time, but isn't really necessary. We will install the package in question which will be the latest version (from Zuul checkout) and without the package in upper-requirements.txt nothing will try and downgrade it. Therefore the solution proposed here is to remove the adding of the URL parts. This allows us to uncap pip and restore testing with the new dependency resolver. Closes-Bug: #1906322 Change-Id: Ib9ba52147199a9d6d0293182d5db50c4a567d677 (cherry picked from commit 6b9a5646225a766f6240e2a1a93a92b82e088aa0) --- inc/python | 7 ++++--- lib/tempest | 3 --- tools/cap-pip.txt | 1 - tools/fixup_stuff.sh | 14 ++++++++++++++ tools/install_pip.sh | 9 +-------- 5 files changed, 19 insertions(+), 15 deletions(-) delete mode 100644 tools/cap-pip.txt diff --git a/inc/python b/inc/python index 8941fd038d..9382d352dc 100644 --- a/inc/python +++ b/inc/python @@ -378,12 +378,13 @@ function _setup_package_with_constraints_edit { project_dir=$(cd $project_dir && pwd) if [ -n "$REQUIREMENTS_DIR" ]; then - # Constrain this package to this project directory from here on out. + # Remove this package from constraints before we install it. + # That way, later installs won't "downgrade" the install from + # source we are about to do. local name name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg) $REQUIREMENTS_DIR/.venv/bin/edit-constraints \ - $REQUIREMENTS_DIR/upper-constraints.txt -- $name \ - "$flags file://$project_dir#egg=$name" + $REQUIREMENTS_DIR/upper-constraints.txt -- $name fi setup_package $bindep $project_dir "$flags" $extras diff --git a/lib/tempest b/lib/tempest index fb55931803..e62dce2c9d 100644 --- a/lib/tempest +++ b/lib/tempest @@ -750,9 +750,6 @@ function install_tempest { set_tempest_venv_constraints $tmp_u_c_m tox -r --notest -efull - # TODO: remove the trailing pip constraint when a proper fix - # arrives for bug https://bugs.launchpad.net/devstack/+bug/1906322 - $TEMPEST_DIR/.tox/tempest/bin/pip install -U -r $RC_DIR/tools/cap-pip.txt # NOTE(mtreinish) Respect constraints in the tempest full venv, things that # are using a tox job other than full will not be respecting constraints but # running pip install -U on tempest requirements diff --git a/tools/cap-pip.txt b/tools/cap-pip.txt deleted file mode 100644 index 8ee551b261..0000000000 --- a/tools/cap-pip.txt +++ /dev/null @@ -1 +0,0 @@ -pip<20.3 diff --git a/tools/fixup_stuff.sh b/tools/fixup_stuff.sh index 25f726892f..c910a70c5b 100755 --- a/tools/fixup_stuff.sh +++ b/tools/fixup_stuff.sh @@ -192,6 +192,20 @@ function fixup_ovn_centos { yum_install centos-release-openstack-victoria } +function fixup_ubuntu { + if ! is_ubuntu; then + return + fi + + # Since pip10, pip will refuse to uninstall files from packages + # that were created with distutils (rather than more modern + # setuptools). This is because it technically doesn't have a + # manifest of what to remove. However, in most cases, simply + # overwriting works. So this hacks around those packages that + # have been dragged in by some other system dependency + sudo rm -rf /usr/lib/python3/dist-packages/PyYAML-*.egg-info +} + function fixup_all { fixup_keystone fixup_ubuntu diff --git a/tools/install_pip.sh b/tools/install_pip.sh index c7bb2c6f55..735de112f7 100755 --- a/tools/install_pip.sh +++ b/tools/install_pip.sh @@ -100,9 +100,7 @@ function install_get_pip { die $LINENO "Download of get-pip.py failed" touch $_local_pip.downloaded fi - # TODO: remove the trailing pip constraint when a proper fix - # arrives for bug https://bugs.launchpad.net/devstack/+bug/1906322 - sudo -H -E python${PYTHON3_VERSION} $_local_pip -c $TOOLS_DIR/cap-pip.txt + sudo -H -E python${PYTHON3_VERSION} $_local_pip } @@ -139,11 +137,6 @@ if [[ -n $PYPI_ALTERNATIVE_URL ]]; then configure_pypi_alternative_url fi -# Just use system pkgs on Focal -if [[ "$DISTRO" == focal ]]; then - exit 0 -fi - # Eradicate any and all system packages # Python in fedora/suse depends on the python-pip package so removing it