Adjust the way the venv is built

It seems a newer package upstream that neutron depends on, depends on a
newer version of argparse to exist in the venv. Because of the way we
were creating the venv before, sometimes packages would not get
installed in the venv because they existed on the host. This was causing
problems for the pip requirements list build process. This patchset
relieves the issues found and hopes to prevent this type of breakage in
the future.

Change-Id: Ie52b8b8707ed5f9306adc78c7b50b1f28d634823
This commit is contained in:
Sam Yaple 2017-10-14 16:01:41 -04:00
parent e3d41e874b
commit 1d7787eff6
5 changed files with 37 additions and 31 deletions

11
scripts/fetch_wheels.sh Executable file
View File

@ -0,0 +1,11 @@
#!/bin/bash
set -ex
$(dirname $0)/fetch_wheels.py
mkdir -p /tmp/wheels/
# NOTE(SamYaple): We exclude all files starting with '.' as these can be
# control files for AUFS which have special meaning on AUFS backed file
# stores.
tar xf /tmp/wheels.tar.gz --exclude='.*' -C /tmp/wheels/

View File

@ -14,7 +14,7 @@ case ${distro} in
ca-certificates \
netbase \
python \
python-pip \
virtualenv \
lsb-release \
sudo
;;
@ -22,7 +22,8 @@ case ${distro} in
yum upgrade -y
yum install -y --setopt=skip_missing_names_on_install=False \
git \
python-pip \
python \
python-virtualenv \
redhat-lsb-core \
sudo
;;
@ -37,6 +38,7 @@ if [[ "${PROJECT}" == 'requirements' ]]; then
exit 0
fi
$(dirname $0)/fetch_wheels.sh
$(dirname $0)/setup_pip.sh
$(dirname $0)/clone_project.sh
$(dirname $0)/pip_install.sh \
@ -45,7 +47,8 @@ $(dirname $0)/pip_install.sh \
pymysql \
python-memcached \
uwsgi \
bindep
bindep \
${PIP_PACKAGES}
PACKAGES=($(bindep -f /opt/loci/bindep.txt -b ${PROJECT} ${PROFILES} || :))
@ -60,7 +63,6 @@ case ${distro} in
if [[ ! -z ${PACKAGES} ]]; then
apt-get install -y --no-install-recommends ${PACKAGES[@]}
fi
pip uninstall -y virtualenv
apt-get purge -y --auto-remove \
git \
python-pip
@ -70,7 +72,6 @@ case ${distro} in
if [[ ! -z ${PACKAGES} ]]; then
yum -y --setopt=skip_missing_names_on_install=False install ${PACKAGES[@]}
fi
pip uninstall -y virtualenv
yum -y autoremove \
git \
python-pip
@ -82,5 +83,11 @@ case ${distro} in
;;
esac
# NOTE(SamYaple): Removing this file allows python to use libraries outside of
# the virtualenv if they do not exist inside the venv. This is a requirement
# for using python-rbd which is not pip installable and is only available in
# packaged form.
rm /var/lib/openstack/lib/python*/no-global-site-packages.txt
rm -rf /tmp/* /root/.cache
find /usr/ /var/ -type f -name "*.pyc" -delete

View File

@ -4,12 +4,4 @@ set -ex
packages=$@
$(dirname $0)/fetch_wheels.py
mkdir -p /tmp/wheels/
# NOTE(SamYaple): We exclude all files starting with '.' as these can be
# control files for AUFS which have special meaning on AUFS backed file
# stores.
tar xf /tmp/wheels.tar.gz --exclude='.*' -C /tmp/wheels/
pip install --no-cache-dir --no-index --no-compile --find-links /tmp/wheels/ ${packages} ${PIP_PACKAGES}
pip install --no-cache-dir --no-index --no-compile --upgrade --find-links /tmp/wheels/ ${packages}

View File

@ -56,8 +56,7 @@ case ${distro} in
libz-dev \
pkg-config \
python-dev \
python-pip \
python-virtualenv
python-pip
;;
centos)
yum upgrade -y
@ -87,7 +86,6 @@ case ${distro} in
python \
python-devel \
python-pip \
python-virtualenv \
libgcrypt \
nss-util \
systemd-devel
@ -100,7 +98,6 @@ esac
$(dirname $0)/setup_pip.sh
$(dirname $0)/clone_project.sh
mv /tmp/requirements/{global-requirements.txt,upper-constraints.txt} /
# NOTE(SamYaple): Build all deps in parallel. This is safe because we are
@ -110,7 +107,7 @@ split -l1 /upper-constraints.txt
ls -1 | xargs -n1 -P20 -t pip wheel --no-deps --wheel-dir / -c /upper-constraints.txt -r
popd
# NOTE(SamYaple): Handle packages not in global-requirements
additional_packages=(bindep==2.5.0 uwsgi)
additional_packages=(argparse bindep==2.5.0 pip setuptools uwsgi wheel virtualenv)
echo "${additional_packages[@]}" | xargs -n1 -P20 pip wheel --wheel-dir / -c /upper-constraints.txt
# NOTE(SamYaple): We want to purge all files that are not wheels or txt to

View File

@ -2,15 +2,14 @@
set -ex
pip install -U virtualenv
# NOTE(SamYaple): --system-site-packages flag allows python to use libraries
# outside of the virtualenv if they do not exist inside the venv. This is a
# requirement for using python-rbd which is not pip installable and is only
# available in packaged form.
# --no-pip --no-setuptools --no-wheel is declared because it was breaking pypi
# mirrors until setuptools is setup properly
virtualenv --no-pip --no-setuptools --no-wheel --system-site-packages /var/lib/openstack/
source /var/lib/openstack/bin/activate
pip install -U pip
pip install -U setuptools wheel
# NOTE(SamYaple): This little dance allows us to install the latest pip and
# setuptools without get_pip.py or the python-pip package (which is in epel on
# centos)
if (( $(virtualenv --version | cut -d. -f1) >= 14 )); then
SETUPTOOLS="--no-setuptools"
fi
virtualenv --extra-search-dir=/tmp/wheels ${SETUPTOOLS} /tmp/venv
source /tmp/venv/bin/activate
pip install --upgrade virtualenv
hash -r
virtualenv --extra-search-dir=/tmp/wheels /var/lib/openstack