Improve ansible host & venv bootstrap

The current ansible bootstrap process tried to
pin the versions of pip, setuptools and wheel
on the host and also uses inherited python
packages from the host. This causes problems
when the host has a version of setuptools which
is cannot be changed (perhaps due to some bug)
or when the host has otherwise undesirable
python packages.

The ansible bootstrap process only needs to
be concerned with whether pip is installed
and understands how to use constraints.

From there we can bootstrap the venv using
get-pip and completely avoid package conflicts
with the host. Once Ansible is bootstrapped,
the pip_install role will ensure that pip,
setuptools and wheel are correctly bootstrapped
and pinned across all hosts.

The pip_install role now uses constraints for
the initial get-pip [1], so we can also remove
the list of pip pins from group_vars.

We remove pip, setuptools and wheel from the
requirements.txt to allow the packages on the
host to be installed in the bootstrap without
forcing a change of pip/setuptools/wheel at the
same time (which causes failures in some
circumstances).

[1] https://review.openstack.org/483905

Change-Id: Ida84fb6bb726e1332f0e29ade51b67a5721f0785
This commit is contained in:
Jesse Pretorius 2017-07-18 17:50:08 +01:00 committed by Jesse Pretorius (odyssey4me)
parent 44e0714fae
commit 5f7000e185
7 changed files with 45 additions and 62 deletions

View File

@ -15,13 +15,6 @@
pip_install_package_state: "{{ package_state }}"
# These are pinned to ensure exactly the same behaviour forever!
# These pins are updated through the sources-branch-updater script
pip_packages:
- pip==9.0.1
- setuptools==36.2.0
- wheel==0.29.0
pip_links:
- { name: "openstack_release", link: "{{ repo_release_path }}/" }
pip_lock_to_internal_repo: "{{ (pip_links | length) >= 1 }}"

View File

@ -1,9 +1,6 @@
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
pip>=7.1.0 # MIT
setuptools!=24.0.0,!=34.0.0,!=34.0.1,!=34.0.2,!=34.0.3,!=34.1.0,!=34.1.1,!=34.2.0,!=34.3.0,!=34.3.1,!=34.3.2,>=16.0 # PSF/ZPL
wheel # MIT
pyasn1!=0.2.3 # BSD
pyOpenSSL>=0.14 # Apache-2.0
requests>=2.14.2 # Apache-2.0

View File

@ -33,10 +33,12 @@ export ANSIBLE_ROLE_FETCH_MODE=${ANSIBLE_ROLE_FETCH_MODE:-git-clone}
# This script should be executed from the root directory of the cloned repo
cd "$(dirname "${0}")/.."
## Functions -----------------------------------------------------------------
info_block "Checking for required libraries." 2> /dev/null ||
source scripts/scripts-library.sh
## Main ----------------------------------------------------------------------
info_block "Bootstrapping System with Ansible"
@ -73,9 +75,6 @@ case ${DISTRO_ID} in
;;
esac
# Install pip
get_pip
# Ensure we use the HTTPS/HTTP proxy with pip if it is specified
PIP_OPTS=""
if [ -n "$HTTPS_PROXY" ]; then
@ -94,7 +93,17 @@ UPPER_CONSTRAINTS_PROTO=$([ "$PYTHON_VERSION" == $(echo -e "$PYTHON_VERSION\n2.7
# Set the location of the constraints to use for all pip installations
export UPPER_CONSTRAINTS_FILE=${UPPER_CONSTRAINTS_FILE:-"$UPPER_CONSTRAINTS_PROTO://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?id=$(awk '/requirements_git_install_branch:/ {print $2}' playbooks/defaults/repo_packages/openstack_services.yml)"}
# Make sure that host requirements are installed
# Install pip on the host if it is not already installed,
# but also make sure that it is at least version 9.x or above.
PIP_VERSION=$(pip --version 2>/dev/null | awk '{print $2}' | cut -d. -f1)
if [[ "${PIP_VERSION}" -lt "9" ]]; then
get_pip ${PYTHON_EXEC_PATH}
# Ensure that our shell knows about the new pip
hash -r pip
fi
# Install the requirements for the various python scripts
# on to the host, including virtualenv.
pip install ${PIP_OPTS} \
--requirement requirements.txt \
--constraint ${UPPER_CONSTRAINTS_FILE} \
@ -112,24 +121,24 @@ if [ -f "/opt/ansible-runtime/bin/python" ]; then
fi
virtualenv --python=${PYTHON_EXEC_PATH} \
--clear \
--no-pip --no-setuptools --no-wheel \
/opt/ansible-runtime
# Install pip, setuptools and wheel into the venv
get_pip /opt/ansible-runtime/bin/python
# The vars used to prepare the Ansible runtime venv
PIP_OPTS+=" --upgrade"
PIP_COMMAND="/opt/ansible-runtime/bin/pip"
PIP_OPTS+=" --constraint global-requirement-pins.txt"
PIP_OPTS+=" --constraint ${UPPER_CONSTRAINTS_FILE}"
# When upgrading there will already be a pip.conf file locking pip down to the
# repo server, in such cases it may be necessary to use --isolated because the
# repo server does not meet the specified requirements.
# Ensure we are running the required versions of pip, wheel and setuptools
${PIP_COMMAND} install ${PIP_OPTS} ${PIP_INSTALL_OPTIONS} || ${PIP_COMMAND} install ${PIP_OPTS} --isolated ${PIP_INSTALL_OPTIONS}
# Set the constraints now that we know we're using the right version of pip
PIP_OPTS+=" --constraint global-requirement-pins.txt --constraint ${UPPER_CONSTRAINTS_FILE}"
# Install the required packages for ansible
$PIP_COMMAND install $PIP_OPTS -r requirements.txt ${ANSIBLE_PACKAGE} || $PIP_COMMAND install --isolated $PIP_OPTS -r requirements.txt ${ANSIBLE_PACKAGE}
# Install ansible and the other required packages
${PIP_COMMAND} install ${PIP_OPTS} -r requirements.txt ${ANSIBLE_PACKAGE} \
|| ${PIP_COMMAND} install --isolated ${PIP_OPTS} -r requirements.txt ${ANSIBLE_PACKAGE}
# Ensure that Ansible binaries run from the venv
pushd /opt/ansible-runtime/bin

View File

@ -20,7 +20,6 @@ LINE='----------------------------------------------------------------------'
MAX_RETRIES=${MAX_RETRIES:-5}
ANSIBLE_PARAMETERS=${ANSIBLE_PARAMETERS:-""}
STARTTIME="${STARTTIME:-$(date +%s)}"
PIP_INSTALL_OPTIONS=${PIP_INSTALL_OPTIONS:-'pip==9.0.1 setuptools==36.2.0 wheel==0.29.0 '}
COMMAND_LOGS=${COMMAND_LOGS:-"/openstack/log/ansible_cmd_logs"}
# The default SSHD configuration has MaxSessions = 10. If a deployer changes
@ -196,40 +195,29 @@ function get_instance_info {
function get_pip {
# check if pip is already installed
if [ "$(which pip)" ]; then
# The python executable to use when executing get-pip is passed
# as a parameter to this function.
GETPIP_PYTHON_EXEC_PATH="${1}"
# make sure that the right pip base packages are installed
# If this fails retry with --isolated to bypass the repo server because the repo server will not have
# been updated at this point to include any newer pip packages.
pip install --upgrade ${PIP_INSTALL_OPTIONS} || pip install --upgrade --isolated ${PIP_INSTALL_OPTIONS}
# Ensure that our shell knows about the new pip
hash -r pip
# when pip is not installed, install it
# Download the get-pip script using the primary or secondary URL
GETPIP_CMD="curl --silent --show-error --retry 5"
GETPIP_FILE="/opt/get-pip.py"
# If GET_PIP_URL is set, then just use it
if [ -n "${GET_PIP_URL:-}" ]; then
${GETPIP_CMD} ${GET_PIP_URL} > ${GETPIP_FILE}
else
# Download the get-pip script using the primary or secondary URL
GETPIP_CMD="curl --silent --show-error --retry 5"
GETPIP_FILE="/opt/get-pip.py"
# If GET_PIP_URL is set, then just use it
if [ -n "${GET_PIP_URL:-}" ]; then
${CURL_CMD} ${GET_PIP_URL} > ${GETPIP_FILE}
else
# Otherwise, try the two standard URL's
${GETPIP_CMD} https://bootstrap.pypa.io/get-pip.py > ${GETPIP_FILE}\
|| ${GETPIP_CMD} https://raw.githubusercontent.com/pypa/get-pip/master/get-pip.py > ${GETPIP_FILE}
fi
if head -n 1 /opt/get-pip.py | grep python; then
python /opt/get-pip.py ${PIP_INSTALL_OPTIONS}
return
fi
echo "A suitable download location for get-pip.py could not be found."
exit_fail
# Otherwise, try the two standard URL's
${GETPIP_CMD} https://bootstrap.pypa.io/get-pip.py > ${GETPIP_FILE}\
|| ${GETPIP_CMD} https://raw.githubusercontent.com/pypa/get-pip/master/get-pip.py > ${GETPIP_FILE}
fi
${GETPIP_PYTHON_EXEC_PATH} ${GETPIP_FILE} \
pip setuptools wheel \
--constraint global-requirement-pins.txt \
|| ${GETPIP_PYTHON_EXEC_PATH} ${GETPIP_FILE} \
pip setuptools wheel \
--constraint global-requirement-pins.txt \
--isolated
}
function get_bowling_ball_tests {

View File

@ -188,19 +188,16 @@ sync_roles_and_packages() {
}
#
# Updates pip options using PIP_CURRENT_OPTIONS env variable
# Updates global requirement pins for pip, setuptools and wheel
#
update_pip_options() {
# Update the PIP_INSTALL_OPTIONS with the current versions of pip, wheel and setuptools
PIP_CURRENT_OPTIONS=$(./scripts/get-pypi-pkg-version.py -p pip setuptools wheel -l horizontal)
sed -i.bak "s|^PIP_INSTALL_OPTIONS=.*|PIP_INSTALL_OPTIONS=\$\{PIP_INSTALL_OPTIONS:-'${PIP_CURRENT_OPTIONS}'\}|" scripts/scripts-library.sh
for pin in ${PIP_CURRENT_OPTIONS}; do
sed -i.bak "s|^$(echo ${pin} | cut -f1 -d=).*|${pin}|" global-requirement-pins.txt
sed -i.bak "s|^ - $(echo ${pin} | cut -f1 -d=).*| - ${pin}|" group_vars/all/pip.yml
done
echo "Updated pip install options/pins"
echo "Updated global requirement pins"
}
#

View File

@ -22,6 +22,7 @@
- role: "pip_install"
- role: "bootstrap-host"
vars:
ansible_python_interpreter: "/usr/bin/python"
openstack_confd_entries: "{{ confd_overrides[bootstrap_host_scenario] }}"
bootstrap_host_scenario: "{{ lookup('env','SCENARIO') | default('aio', true) }}"
confd_overrides:

View File

@ -23,9 +23,7 @@ packages_install:
- linux-image-extra-{{ ansible_kernel }}
- lvm2
- python2.7
- python-crypto
- python-dev
- python-yaml
- sshpass
- tmux
- vim