Enforce upper-constraints when building ramdisks

Currently, building ramdisks installs ironic-python-agent without any
upper-constraints. This causes the package to be installed with newer,
untested dependencies.

This commits introduces a tool to generate a local upper-constraints
file based on predefined strategies (below). Additionally, the fallback
to the openstack/requirements uses the URL defined in tox.ini instead of
redefining it. This prevents having to keep track of two separate
variables when releasing.

upper-constraints lookup strategies (in order):

  * UPPER_CONSTRAINTS_FILE points to a local file
  * UPPER_CONSTRAINTS_FILE points to a URL
  * /opt/stack/new/requirements/upper-constraints.txt
  * upper-constraints.txt from openstack/requirements git repository

Partial-bug: #1616554
Change-Id: Ib5c0c57cafdb6ffd7456e61f3b1bb5fa57520e5a
This commit is contained in:
Mathieu Mitchell 2016-08-22 15:43:39 -04:00
parent 007654a9b6
commit a0ca6ce157
7 changed files with 108 additions and 9 deletions

3
.gitignore vendored
View File

@ -24,3 +24,6 @@ doc/build
# release notes build # release notes build
releasenotes/build releasenotes/build
# upper-constraints handling for image builds
upper-constraints.txt

View File

@ -39,11 +39,10 @@ RUN apt-mark manual python-minimal
# Install requirements separately, because pip understands a git+https url # Install requirements separately, because pip understands a git+https url
# while setuptools doesn't # while setuptools doesn't
RUN proxy.sh pip install --upgrade pip RUN proxy.sh pip install --upgrade pip
# TODO(jroll) use upper-constraints here RUN proxy.sh pip install -c /tmp/ironic-python-agent/upper-constraints.txt --no-cache-dir -r /tmp/ironic-python-agent/requirements.txt
RUN proxy.sh pip install --no-cache-dir -r /tmp/ironic-python-agent/requirements.txt
# This will succeed because all the dependencies were installed previously # This will succeed because all the dependencies were installed previously
RUN proxy.sh pip install --no-cache-dir /tmp/ironic-python-agent RUN proxy.sh pip install -c /tmp/ironic-python-agent/upper-constraints.txt --no-cache-dir /tmp/ironic-python-agent
# Remove no longer needed packages # Remove no longer needed packages
# NOTE(jroll) leave git to avoid strange apt issues in downstream Dockerfiles # NOTE(jroll) leave git to avoid strange apt issues in downstream Dockerfiles

View File

@ -0,0 +1,9 @@
#!/bin/bash
# NOTE(mmitchell): This extracts the URL defined as the default value for
# UPPER_CONSTRAINTS_FILE in tox.ini. This is used by image
# builders to avoid duplicating the default value in multiple
# scripts. This is specially done to leverage the release
# tools that automatically update the tox.ini when projects
# are released.
sed -n 's/^.*{env:UPPER_CONSTRAINTS_FILE\:\([^}]*\)}.*$/\1/p' $1 | head -n1

View File

@ -0,0 +1,79 @@
#!/bin/bash -eu
SCRIPT_NAME=$(basename $0)
COMMON_ROOT=$(dirname $0)
DESTINATION="$1"
TOX_INI_UPPER_CONSTRAINT_URL="$(${COMMON_ROOT}/extract_upper_constraints_from_tox_ini.sh ${COMMON_ROOT}/../../tox.ini)"
copy() {
local src=$1
local destination=$2
if test -z "${src}"; then
return 1
fi
if test -e "${src}"; then
log "File '${src}' exists. Using as upper-constraints."
cp "${src}" "${destination}"
else
log "File '${src}' not found. Skipping local file strategy."
return 1
fi
return 0
}
download() {
local url=$1
local destination=$2
if test -z "${url}"; then
return 1
else
log "Downloading from '${url}'"
curl ${url} -o "${destination}"
fi
return 0
}
log() {
echo "${SCRIPT_NAME}: ${@}"
}
fail() {
log ${@}
exit 1
}
upper_constraints_is_not_null() {
test "${UPPER_CONSTRAINTS_FILE:-""}" != ""
}
copy_uc() {
copy "${UPPER_CONSTRAINTS_FILE:-""}" "${DESTINATION}"
}
download_uc() {
download "${UPPER_CONSTRAINTS_FILE:-""}" "${DESTINATION}"
}
copy_new_requirements_uc() {
copy "/opt/stack/new/requirements/upper-constraints.txt" "${DESTINATION}"
}
download_from_tox_ini_url() {
log "tox.ini indicates '${TOX_INI_UPPER_CONSTRAINT_URL}' as fallback."
download "${TOX_INI_UPPER_CONSTRAINT_URL}" "${DESTINATION}"
}
log "Generating local constraints file..."
if upper_constraints_is_not_null; then
log "UPPER_CONSTRAINTS_FILE is defined as '${UPPER_CONSTRAINTS_FILE:-""}'"
copy_uc || download_uc || fail "Failed to copy or download file indicated in UPPER_CONSTRAINTS_FILE."
else
log "UPPER_CONSTRAINTS_FILE is not defined. Using fallback strategies."
copy_new_requirements_uc || \
download_from_tox_ini_url || fail "Failed to download upper-constraints.txt from '${TOX_INI_UPPER_CONSTRAINT_URL}'."
fi

View File

@ -7,6 +7,7 @@
set -e set -e
OUTPUT_FILE="oem/container.tar.gz" OUTPUT_FILE="oem/container.tar.gz"
IPA_ROOT=$(readlink -f $(dirname $0)/../../)
# If there's already a container.tar.gz, don't overwrite it -- instead, bail # If there's already a container.tar.gz, don't overwrite it -- instead, bail
if [[ -e "${OUTPUT_FILE}" ]]; then if [[ -e "${OUTPUT_FILE}" ]]; then
@ -15,7 +16,10 @@ if [[ -e "${OUTPUT_FILE}" ]]; then
fi fi
# Build the docker image # Build the docker image
cd ../../ # Everything from ${IPA_ROOT} will be available under /tmp/ironic-python-agent in Docker
cd ${IPA_ROOT}
imagebuild/common/generate_upper_constraints.sh ${IPA_ROOT}/upper-constraints.txt
# TODO(jlvilla): Once Docker 1.9 is widely deployed, switch to using the 'ARG' # TODO(jlvilla): Once Docker 1.9 is widely deployed, switch to using the 'ARG'
# command which was added in Docker 1.9. Currently Ubuntu 14.04 uses Docker # command which was added in Docker 1.9. Currently Ubuntu 14.04 uses Docker

View File

@ -65,6 +65,9 @@ cd ../..
rm -rf *.egg-info rm -rf *.egg-info
python setup.py sdist --dist-dir "$BUILDDIR/tmp/localpip" --quiet python setup.py sdist --dist-dir "$BUILDDIR/tmp/localpip" --quiet
cp requirements.txt $BUILDDIR/tmp/ipa-requirements.txt cp requirements.txt $BUILDDIR/tmp/ipa-requirements.txt
imagebuild/common/generate_upper_constraints.sh upper-constraints.txt
cp upper-constraints.txt $BUILDDIR/tmp/upper-constraints.txt
cd $WORKDIR cd $WORKDIR
sudo cp /etc/resolv.conf $BUILDDIR/etc/resolv.conf sudo cp /etc/resolv.conf $BUILDDIR/etc/resolv.conf
@ -88,10 +91,10 @@ done < $WORKDIR/build_files/buildreqs.lst
# Build python wheels # Build python wheels
$CHROOT_CMD python /tmp/get-pip.py $CHROOT_CMD python /tmp/get-pip.py
$CHROOT_CMD pip install pbr $CHROOT_CMD pip install pbr
$CHROOT_CMD pip wheel --wheel-dir /tmp/wheels setuptools $CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels setuptools
$CHROOT_CMD pip wheel --wheel-dir /tmp/wheels pip $CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels pip
$CHROOT_CMD pip wheel --wheel-dir /tmp/wheels -r /tmp/ipa-requirements.txt $CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels -r /tmp/ipa-requirements.txt
$CHROOT_CMD pip wheel --no-index --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip --find-links=/tmp/wheels ironic-python-agent $CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --no-index --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip --find-links=/tmp/wheels ironic-python-agent
# Build tgt # Build tgt
rm -rf $WORKDIR/build_files/tgt.tcz rm -rf $WORKDIR/build_files/tgt.tcz

View File

@ -76,7 +76,9 @@ $CHROOT_CMD depmod -a `$WORKDIR/build_files/fakeuname -r`
# If flag is set install the python now # If flag is set install the python now
if $BUILD_AND_INSTALL_TINYIPA ; then if $BUILD_AND_INSTALL_TINYIPA ; then
$CHROOT_CMD python /tmp/get-pip.py --no-wheel --no-index --find-links=file:///tmp/wheelhouse ironic_python_agent cp -a $BUILDDIR/tmp/upper-constraints.txt $FINALDIR/tmp/upper-constraints.txt
$CHROOT_CMD python /tmp/get-pip.py -c /tmp/upper-constraints.txt --no-wheel --no-index --find-links=file:///tmp/wheelhouse ironic_python_agent
rm -rf $FINALDIR/tmp/upper-constraints.txt
rm -rf $FINALDIR/tmp/wheelhouse rm -rf $FINALDIR/tmp/wheelhouse
rm -rf $FINALDIR/tmp/get-pip.py rm -rf $FINALDIR/tmp/get-pip.py
fi fi