Improve ansible host & venv bootstrap
The current ansible bootstrap process tried to pin the versions of pip, setuptools and wheel on the host and also uses inherited python packages from the host. This causes problems when the host has a version of setuptools which is cannot be changed (perhaps due to some bug) or when the host has otherwise undesirable python packages. The ansible bootstrap process only needs to be concerned with whether pip is installed and understands how to use constraints. From there we can bootstrap the venv using get-pip and completely avoid package conflicts with the host. Once Ansible is bootstrapped, the pip_install role will ensure that pip, setuptools and wheel are correctly bootstrapped and pinned across all hosts. The pip_install role now uses constraints for the initial get-pip [1], so we can also remove the list of pip pins from group_vars. We remove pip, setuptools and wheel from the requirements.txt to allow the packages on the host to be installed in the bootstrap without forcing a change of pip/setuptools/wheel at the same time (which causes failures in some circumstances). [1] https://review.openstack.org/483905 Change-Id: Ida84fb6bb726e1332f0e29ade51b67a5721f0785
This commit is contained in:
parent
44e0714fae
commit
5f7000e185
|
@ -15,13 +15,6 @@
|
|||
|
||||
pip_install_package_state: "{{ package_state }}"
|
||||
|
||||
# These are pinned to ensure exactly the same behaviour forever!
|
||||
# These pins are updated through the sources-branch-updater script
|
||||
pip_packages:
|
||||
- pip==9.0.1
|
||||
- setuptools==36.2.0
|
||||
- wheel==0.29.0
|
||||
|
||||
pip_links:
|
||||
- { name: "openstack_release", link: "{{ repo_release_path }}/" }
|
||||
pip_lock_to_internal_repo: "{{ (pip_links | length) >= 1 }}"
|
||||
|
|
|
@ -1,9 +1,6 @@
|
|||
# The order of packages is significant, because pip processes them in the order
|
||||
# of appearance. Changing the order has an impact on the overall integration
|
||||
# process, which may cause wedges in the gate later.
|
||||
pip>=7.1.0 # MIT
|
||||
setuptools!=24.0.0,!=34.0.0,!=34.0.1,!=34.0.2,!=34.0.3,!=34.1.0,!=34.1.1,!=34.2.0,!=34.3.0,!=34.3.1,!=34.3.2,>=16.0 # PSF/ZPL
|
||||
wheel # MIT
|
||||
pyasn1!=0.2.3 # BSD
|
||||
pyOpenSSL>=0.14 # Apache-2.0
|
||||
requests>=2.14.2 # Apache-2.0
|
||||
|
|
|
@ -33,10 +33,12 @@ export ANSIBLE_ROLE_FETCH_MODE=${ANSIBLE_ROLE_FETCH_MODE:-git-clone}
|
|||
# This script should be executed from the root directory of the cloned repo
|
||||
cd "$(dirname "${0}")/.."
|
||||
|
||||
|
||||
## Functions -----------------------------------------------------------------
|
||||
info_block "Checking for required libraries." 2> /dev/null ||
|
||||
source scripts/scripts-library.sh
|
||||
|
||||
|
||||
## Main ----------------------------------------------------------------------
|
||||
info_block "Bootstrapping System with Ansible"
|
||||
|
||||
|
@ -73,9 +75,6 @@ case ${DISTRO_ID} in
|
|||
;;
|
||||
esac
|
||||
|
||||
# Install pip
|
||||
get_pip
|
||||
|
||||
# Ensure we use the HTTPS/HTTP proxy with pip if it is specified
|
||||
PIP_OPTS=""
|
||||
if [ -n "$HTTPS_PROXY" ]; then
|
||||
|
@ -94,7 +93,17 @@ UPPER_CONSTRAINTS_PROTO=$([ "$PYTHON_VERSION" == $(echo -e "$PYTHON_VERSION\n2.7
|
|||
# Set the location of the constraints to use for all pip installations
|
||||
export UPPER_CONSTRAINTS_FILE=${UPPER_CONSTRAINTS_FILE:-"$UPPER_CONSTRAINTS_PROTO://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?id=$(awk '/requirements_git_install_branch:/ {print $2}' playbooks/defaults/repo_packages/openstack_services.yml)"}
|
||||
|
||||
# Make sure that host requirements are installed
|
||||
# Install pip on the host if it is not already installed,
|
||||
# but also make sure that it is at least version 9.x or above.
|
||||
PIP_VERSION=$(pip --version 2>/dev/null | awk '{print $2}' | cut -d. -f1)
|
||||
if [[ "${PIP_VERSION}" -lt "9" ]]; then
|
||||
get_pip ${PYTHON_EXEC_PATH}
|
||||
# Ensure that our shell knows about the new pip
|
||||
hash -r pip
|
||||
fi
|
||||
|
||||
# Install the requirements for the various python scripts
|
||||
# on to the host, including virtualenv.
|
||||
pip install ${PIP_OPTS} \
|
||||
--requirement requirements.txt \
|
||||
--constraint ${UPPER_CONSTRAINTS_FILE} \
|
||||
|
@ -112,24 +121,24 @@ if [ -f "/opt/ansible-runtime/bin/python" ]; then
|
|||
fi
|
||||
virtualenv --python=${PYTHON_EXEC_PATH} \
|
||||
--clear \
|
||||
--no-pip --no-setuptools --no-wheel \
|
||||
/opt/ansible-runtime
|
||||
|
||||
# Install pip, setuptools and wheel into the venv
|
||||
get_pip /opt/ansible-runtime/bin/python
|
||||
|
||||
# The vars used to prepare the Ansible runtime venv
|
||||
PIP_OPTS+=" --upgrade"
|
||||
PIP_COMMAND="/opt/ansible-runtime/bin/pip"
|
||||
PIP_OPTS+=" --constraint global-requirement-pins.txt"
|
||||
PIP_OPTS+=" --constraint ${UPPER_CONSTRAINTS_FILE}"
|
||||
|
||||
# When upgrading there will already be a pip.conf file locking pip down to the
|
||||
# repo server, in such cases it may be necessary to use --isolated because the
|
||||
# repo server does not meet the specified requirements.
|
||||
|
||||
# Ensure we are running the required versions of pip, wheel and setuptools
|
||||
${PIP_COMMAND} install ${PIP_OPTS} ${PIP_INSTALL_OPTIONS} || ${PIP_COMMAND} install ${PIP_OPTS} --isolated ${PIP_INSTALL_OPTIONS}
|
||||
|
||||
# Set the constraints now that we know we're using the right version of pip
|
||||
PIP_OPTS+=" --constraint global-requirement-pins.txt --constraint ${UPPER_CONSTRAINTS_FILE}"
|
||||
|
||||
# Install the required packages for ansible
|
||||
$PIP_COMMAND install $PIP_OPTS -r requirements.txt ${ANSIBLE_PACKAGE} || $PIP_COMMAND install --isolated $PIP_OPTS -r requirements.txt ${ANSIBLE_PACKAGE}
|
||||
# Install ansible and the other required packages
|
||||
${PIP_COMMAND} install ${PIP_OPTS} -r requirements.txt ${ANSIBLE_PACKAGE} \
|
||||
|| ${PIP_COMMAND} install --isolated ${PIP_OPTS} -r requirements.txt ${ANSIBLE_PACKAGE}
|
||||
|
||||
# Ensure that Ansible binaries run from the venv
|
||||
pushd /opt/ansible-runtime/bin
|
||||
|
|
|
@ -20,7 +20,6 @@ LINE='----------------------------------------------------------------------'
|
|||
MAX_RETRIES=${MAX_RETRIES:-5}
|
||||
ANSIBLE_PARAMETERS=${ANSIBLE_PARAMETERS:-""}
|
||||
STARTTIME="${STARTTIME:-$(date +%s)}"
|
||||
PIP_INSTALL_OPTIONS=${PIP_INSTALL_OPTIONS:-'pip==9.0.1 setuptools==36.2.0 wheel==0.29.0 '}
|
||||
COMMAND_LOGS=${COMMAND_LOGS:-"/openstack/log/ansible_cmd_logs"}
|
||||
|
||||
# The default SSHD configuration has MaxSessions = 10. If a deployer changes
|
||||
|
@ -196,40 +195,29 @@ function get_instance_info {
|
|||
|
||||
function get_pip {
|
||||
|
||||
# check if pip is already installed
|
||||
if [ "$(which pip)" ]; then
|
||||
# The python executable to use when executing get-pip is passed
|
||||
# as a parameter to this function.
|
||||
GETPIP_PYTHON_EXEC_PATH="${1}"
|
||||
|
||||
# make sure that the right pip base packages are installed
|
||||
# If this fails retry with --isolated to bypass the repo server because the repo server will not have
|
||||
# been updated at this point to include any newer pip packages.
|
||||
pip install --upgrade ${PIP_INSTALL_OPTIONS} || pip install --upgrade --isolated ${PIP_INSTALL_OPTIONS}
|
||||
|
||||
# Ensure that our shell knows about the new pip
|
||||
hash -r pip
|
||||
|
||||
# when pip is not installed, install it
|
||||
# Download the get-pip script using the primary or secondary URL
|
||||
GETPIP_CMD="curl --silent --show-error --retry 5"
|
||||
GETPIP_FILE="/opt/get-pip.py"
|
||||
# If GET_PIP_URL is set, then just use it
|
||||
if [ -n "${GET_PIP_URL:-}" ]; then
|
||||
${GETPIP_CMD} ${GET_PIP_URL} > ${GETPIP_FILE}
|
||||
else
|
||||
|
||||
# Download the get-pip script using the primary or secondary URL
|
||||
GETPIP_CMD="curl --silent --show-error --retry 5"
|
||||
GETPIP_FILE="/opt/get-pip.py"
|
||||
# If GET_PIP_URL is set, then just use it
|
||||
if [ -n "${GET_PIP_URL:-}" ]; then
|
||||
${CURL_CMD} ${GET_PIP_URL} > ${GETPIP_FILE}
|
||||
else
|
||||
# Otherwise, try the two standard URL's
|
||||
${GETPIP_CMD} https://bootstrap.pypa.io/get-pip.py > ${GETPIP_FILE}\
|
||||
|| ${GETPIP_CMD} https://raw.githubusercontent.com/pypa/get-pip/master/get-pip.py > ${GETPIP_FILE}
|
||||
fi
|
||||
|
||||
if head -n 1 /opt/get-pip.py | grep python; then
|
||||
python /opt/get-pip.py ${PIP_INSTALL_OPTIONS}
|
||||
return
|
||||
fi
|
||||
|
||||
echo "A suitable download location for get-pip.py could not be found."
|
||||
exit_fail
|
||||
# Otherwise, try the two standard URL's
|
||||
${GETPIP_CMD} https://bootstrap.pypa.io/get-pip.py > ${GETPIP_FILE}\
|
||||
|| ${GETPIP_CMD} https://raw.githubusercontent.com/pypa/get-pip/master/get-pip.py > ${GETPIP_FILE}
|
||||
fi
|
||||
|
||||
${GETPIP_PYTHON_EXEC_PATH} ${GETPIP_FILE} \
|
||||
pip setuptools wheel \
|
||||
--constraint global-requirement-pins.txt \
|
||||
|| ${GETPIP_PYTHON_EXEC_PATH} ${GETPIP_FILE} \
|
||||
pip setuptools wheel \
|
||||
--constraint global-requirement-pins.txt \
|
||||
--isolated
|
||||
}
|
||||
|
||||
function get_bowling_ball_tests {
|
||||
|
|
|
@ -188,19 +188,16 @@ sync_roles_and_packages() {
|
|||
}
|
||||
|
||||
#
|
||||
# Updates pip options using PIP_CURRENT_OPTIONS env variable
|
||||
# Updates global requirement pins for pip, setuptools and wheel
|
||||
#
|
||||
update_pip_options() {
|
||||
# Update the PIP_INSTALL_OPTIONS with the current versions of pip, wheel and setuptools
|
||||
PIP_CURRENT_OPTIONS=$(./scripts/get-pypi-pkg-version.py -p pip setuptools wheel -l horizontal)
|
||||
sed -i.bak "s|^PIP_INSTALL_OPTIONS=.*|PIP_INSTALL_OPTIONS=\$\{PIP_INSTALL_OPTIONS:-'${PIP_CURRENT_OPTIONS}'\}|" scripts/scripts-library.sh
|
||||
|
||||
for pin in ${PIP_CURRENT_OPTIONS}; do
|
||||
sed -i.bak "s|^$(echo ${pin} | cut -f1 -d=).*|${pin}|" global-requirement-pins.txt
|
||||
sed -i.bak "s|^ - $(echo ${pin} | cut -f1 -d=).*| - ${pin}|" group_vars/all/pip.yml
|
||||
done
|
||||
|
||||
echo "Updated pip install options/pins"
|
||||
echo "Updated global requirement pins"
|
||||
}
|
||||
|
||||
#
|
||||
|
|
|
@ -22,6 +22,7 @@
|
|||
- role: "pip_install"
|
||||
- role: "bootstrap-host"
|
||||
vars:
|
||||
ansible_python_interpreter: "/usr/bin/python"
|
||||
openstack_confd_entries: "{{ confd_overrides[bootstrap_host_scenario] }}"
|
||||
bootstrap_host_scenario: "{{ lookup('env','SCENARIO') | default('aio', true) }}"
|
||||
confd_overrides:
|
||||
|
|
|
@ -23,9 +23,7 @@ packages_install:
|
|||
- linux-image-extra-{{ ansible_kernel }}
|
||||
- lvm2
|
||||
- python2.7
|
||||
- python-crypto
|
||||
- python-dev
|
||||
- python-yaml
|
||||
- sshpass
|
||||
- tmux
|
||||
- vim
|
||||
|
|
Loading…
Reference in New Issue