Create pip manifest files of installed packages

Create the pip-manifest element to encapsulate the saving and use
of pip manifest files.

Ensure that it installs prior to any elements that should be able
to use it.

Write pip manifest files for installs in:
    * os-svc-install
    * os-apply-config
    * os-collect-config
    * os-refresh-config
    * openstack-clients

Enable using a manifest file to reinstall exact versions of
packages in a subsequent run through the DIB_PIP_MANIFEST_*
environment variables.

Change-Id: I4d4ab5921c534727b48cb9969ec8ecfd2c26e6ed
This commit is contained in:
Jon-Paul Sullivan 2014-03-12 19:47:41 +00:00
parent d6d087e3a1
commit 1c998eb93f
19 changed files with 320 additions and 58 deletions

View File

@ -3,35 +3,45 @@
set -eux
client=$1
repo=python-${client}client
client_manifest=$(get-pip-manifest ${repo})
venv_base=/opt/stack/venvs
# We would like to use --system-site-packages here but if requirements.txt
# contains libraries that are installed globally with versions that don't
# satisfy our requirements.txt, we end up using the incorrect global library.
# Because the global site-packages appears first in sys.path
# TODO : Add this back in when we are using virtualenv >= 1.11
virtualenv /opt/stack/venvs/$repo
virtualenv $venv_base/$repo
set +u
source /opt/stack/venvs/$repo/bin/activate
source $venv_base/$repo/bin/activate
set -u
# Need setuptools>=1.0 to manage connections when
# downloading from pypi using http_proxy and https_proxy
pip install -U 'setuptools>=1.0'
# bug #1293812 : Avoid easy_install triggering on pbr.
pip install -U 'pbr>=0.5.21,<1.0'
pushd /opt/stack/$repo
if [ -e requirements.txt ]; then
pip install -r requirements.txt
elif [ -e tools/pip-requires ]; then
pip install -r tools/pip-requires
if [ -n "$client_manifest" ]; then
use-pip-manifest $client_manifest
else
# Need setuptools>=1.0 to manage connections when
# downloading from pypi using http_proxy and https_proxy
pip install -U 'setuptools>=1.0'
# bug #1293812 : Avoid easy_install triggering on pbr.
pip install -U 'pbr>=0.5.21,<1.0'
if [ -e requirements.txt ]; then
pip install -r requirements.txt
elif [ -e tools/pip-requires ]; then
pip install -r tools/pip-requires
fi
fi
# Always replay this, as we cannot use the entry this would generate in the manifest
python setup.py develop
ln -s /opt/stack/venvs/$repo/bin/$client /usr/local/bin/$client
# Write the manifest of what was installed
write-pip-manifest $repo
ln -s $venv_base/$repo/bin/$client /usr/local/bin/$client
popd
set +u

View File

@ -1,2 +1,3 @@
source-repositories
pip-and-virtualenv
pip-manifest

View File

@ -1 +1,2 @@
os-refresh-config
pip-manifest

View File

@ -1,16 +1,25 @@
#!/bin/bash
set -eux
manifest=$(get-pip-manifest os-apply-config)
virtualenv --setuptools /opt/stack/venvs/os-apply-config
# bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which
# doesn't manage correctly HTTPS sockets when downloading pbr from
# https://pypi.python.org/simple/ if using http_proxy and https_proxy
# envvars
/opt/stack/venvs/os-apply-config/bin/pip install -U 'setuptools>=1.0'
# bug #1293812 : Avoid easy_install triggering on pbr.
/opt/stack/venvs/os-apply-config/bin/pip install -U 'pbr>=0.6,<1.0'
/opt/stack/venvs/os-apply-config/bin/pip install -U os-apply-config
if [ -n "$manifest" ]; then
use-pip-manifest $manifest
else
# bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which
# doesn't manage correctly HTTPS sockets when downloading pbr from
# https://pypi.python.org/simple/ if using http_proxy and https_proxy
# envvars
/opt/stack/venvs/os-apply-config/bin/pip install -U 'setuptools>=1.0'
# bug #1293812 : Avoid easy_install triggering on pbr.
/opt/stack/venvs/os-apply-config/bin/pip install -U 'pbr>=0.6,<1.0'
/opt/stack/venvs/os-apply-config/bin/pip install -U os-apply-config
fi
# Write the manifest of what was installed
write-pip-manifest os-apply-config
ln -s /opt/stack/venvs/os-apply-config/bin/os-apply-config /usr/local/bin/os-apply-config
ln -s /opt/stack/venvs/os-apply-config/bin/os-config-applier /usr/local/bin/os-config-applier

View File

@ -1,3 +1,4 @@
source-repositories
os-refresh-config
os-apply-config
pip-manifest

View File

@ -1,16 +1,25 @@
#!/bin/bash
set -eux
manifest=$(get-pip-manifest os-collect-config)
install-packages build-essential libz-dev libxslt-dev libxml2-dev python-dev
virtualenv --setuptools /opt/stack/venvs/os-collect-config
# Need setuptools>=1.0 to manage connections when
# downloading from pypi using http_proxy and https_proxy
/opt/stack/venvs/os-collect-config/bin/pip install -U 'setuptools>=1.0'
# bug #1293812 : Avoid easy_install triggering on pbr.
/opt/stack/venvs/os-collect-config/bin/pip install -U 'pbr>=0.6,<1.0'
/opt/stack/venvs/os-collect-config/bin/pip install -U os-collect-config
if [ -n "$manifest" ]; then
use-pip-manifest $manifest
else
# Need setuptools>=1.0 to manage connections when
# downloading from pypi using http_proxy and https_proxy
/opt/stack/venvs/os-collect-config/bin/pip install -U 'setuptools>=1.0'
# bug #1293812 : Avoid easy_install triggering on pbr.
/opt/stack/venvs/os-collect-config/bin/pip install -U 'pbr>=0.6,<1.0'
/opt/stack/venvs/os-collect-config/bin/pip install -U os-collect-config
fi
# Write the manifest of what was installed
write-pip-manifest os-collect-config
ln -s /opt/stack/venvs/os-collect-config/bin/os-collect-config /usr/local/bin/os-collect-config

View File

@ -2,3 +2,4 @@ pip-and-virtualenv
source-repositories
os-collect-config
os-apply-config
pip-manifest

View File

@ -4,17 +4,26 @@
set -eux
manifest=$(get-pip-manifest os-refresh-config)
# pip and virtualenv is installed by the pip-and-virtualenv element
virtualenv --setuptools /opt/stack/venvs/os-refresh-config
# Need setuptools>=1.0 to manage connections when
# downloading from pypi using http_proxy and https_proxy
/opt/stack/venvs/os-refresh-config/bin/pip install -U pip
/opt/stack/venvs/os-refresh-config/bin/pip install -U 'setuptools>=1.0'
# bug #1293812 : Avoid easy_install triggering on pbr.
/opt/stack/venvs/os-refresh-config/bin/pip install -U 'pbr>=0.5.21,<1.0'
/opt/stack/venvs/os-refresh-config/bin/pip install -U os-refresh-config
if [ -n "$manifest" ]; then
use-pip-manifest $manifest
else
# Need setuptools>=1.0 to manage connections when
# downloading from pypi using http_proxy and https_proxy
/opt/stack/venvs/os-refresh-config/bin/pip install -U pip
/opt/stack/venvs/os-refresh-config/bin/pip install -U 'setuptools>=1.0'
# bug #1293812 : Avoid easy_install triggering on pbr.
/opt/stack/venvs/os-refresh-config/bin/pip install -U 'pbr>=0.5.21,<1.0'
/opt/stack/venvs/os-refresh-config/bin/pip install -U os-refresh-config
fi
# Write the manifest of what was installed
write-pip-manifest os-refresh-config
ln -s /opt/stack/venvs/os-refresh-config/bin/os-refresh-config /usr/local/bin/os-refresh-config

View File

@ -5,6 +5,8 @@ function python_install() {
local svc_root=$1
local install_dir=$2
local system_site_packages=${3:-"False"}
local name=$(basename $install_dir)
local svc_manifest=$(get-pip-manifest $name)
SITE_PCKGS="--no-site-packages"
if [ $system_site_packages == "True" ]; then
@ -17,31 +19,39 @@ function python_install() {
source $install_dir/bin/activate
set -u
if [ -e $svc_root/requirements.txt ]; then
reqs=$svc_root/requirements.txt
elif [ -e $svc_root/tools/pip-requires ]; then
reqs=$svc_root/tools/pip-requires
# If given an exact deps list, use it, and upgrade to the local git service
if [ -n "$svc_manifest" ]; then
use-pip-manifest $svc_manifest
else
reqs=""
if [ -e $svc_root/requirements.txt ]; then
reqs=$svc_root/requirements.txt
elif [ -e $svc_root/tools/pip-requires ]; then
reqs=$svc_root/tools/pip-requires
else
reqs=""
fi
# bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which
# doesn't manage correctly HTTPS sockets when downloading pbr from
# https://pypi.python.org/simple/ if using http_proxy and
# https_proxy envvars
pip install -U 'setuptools>=1.0'
# bug #1293812 : Avoid easy_install triggering on pbr.
pip install -U 'pbr>=0.5.21,<1.0'
if [ -n "$reqs" ] ; then
pip install -r $reqs
# FIXME: pip requires doesn't include MySQL-python
pip install MySQL-python
fi
fi
# bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which
# doesn't manage correctly HTTPS sockets when downloading pbr from
# https://pypi.python.org/simple/ if using http_proxy and
# https_proxy envvars
pip install -U 'setuptools>=1.0'
# bug #1293812 : Avoid easy_install triggering on pbr.
pip install -U 'pbr>=0.5.21,<1.0'
if [ -n "$reqs" ] ; then
pip install -r $reqs
# FIXME: pip requires doesn't include MySQL-python
pip install MySQL-python
fi
# Always replay this, as we cannot use the entry this would generate in the manifest
pip install $svc_root
# Write the manifest of what was installed
write-pip-manifest $name
set +u
deactivate
set -u

View File

@ -1 +1,2 @@
pip-and-virtualenv
pip-manifest

View File

@ -0,0 +1,63 @@
Utility element to enable repeatable pip installs
=================================================
Set this element as a dependency to make the utility scripts available,
and then use them as appropriate.
## Usage
This element makes a number of scripts available for use in image building, and
performs actions to copy specified manifests into the image for use in building.
It also copies manifests generated during the build back to the build environment
during cleanup.
## bin
Utility scripts for use in other elements to create and reuse pip manifests,
as detailed in the usage section.
### get-pip-manifest
Echoes the name of the pip manifest file if one has been copied in, or just
returns. The caller passes the name associated with their element, which
should be descriptive of the element, to get the correct value.
For example, the nova element calls `get-pip-manifest nova`.
The name of the element is transformed to conform with bash variable naming
rules, so any charaters that are not [A-Za-z0-9] are replaced with '\_'.
### use-pip-manifest
Uses the given manifest to perform the pip installs necessary.
Note that any development versions listed in the manifest are not reinstalled.
The reason for this is that development versions are expected to have been
installed from a source other than pypi or the mirror in use, and so development
versions will not be reinstallable without extra information.
The exact details with respect to this are for the relevant consuming element to
determine.
### write-pip-manifest
Calls pip freeze and writes the versions of all of the packages currently
installed to a manifest file.
The format of the manifest is the standard python requirements format that is
generated by the "pip freeze" command.
## extra-data.d
### 75-inject-pip-manifests
Copies any pip manifest specified in DIB\_PIP\_MANIFEST\_\* environment variables
into the image chroot environment.
## install.d
### 01-pip-manifest
Installs the scripts in this element into the image for later use by other elements.
## cleanup.d
### 01-copy-pip-manifests
Runs outside of the chroot and copies created manifests back to the
build environment for re-use in repeating the image build.

View File

@ -0,0 +1,25 @@
#!/bin/bash
#
# Copyright 2014 Hewlett-Packard Development Company, L.P.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
set -eux
name=${1:?"Usage: ${0} <name>"}
build_manifest=/etc/dib-pip-manifests/dib-pip-build-manifest-${name//[^A-Za-z0-9]/_}
if [[ -f "${build_manifest}" ]]; then
echo "${build_manifest}"
fi

View File

@ -0,0 +1,25 @@
#!/bin/bash
#
# Copyright 2014 Hewlett-Packard Development Company, L.P.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
set -eux
manifest=${1:?"Usage: ${0} <manifest>"}
# Comment out the dev versions listed, as they may not be reinstallable
sed -i "s/^\(.*[.]dev.*\)$/# \1/g" $manifest
pip install --no-deps -r $manifest

View File

@ -0,0 +1,22 @@
#!/bin/bash
#
# Copyright 2014 Hewlett-Packard Development Company, L.P.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
set -eux
name=${1:?"Usage: ${0} <name>"}
pip freeze | tee /etc/dib-pip-manifests/dib-pip-manifest-${name//[^A-Za-z0-9]/_}

View File

@ -0,0 +1,24 @@
#!/bin/bash
#
# Copyright 2014 Hewlett-Packard Development Company, L.P.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
set -eux
if [ -d $TMP_MOUNT_PATH/etc/dib-pip-manifests ]; then
mkdir -p ${IMAGE_NAME}-manifests
cp --no-preserve=ownership -rv $TMP_MOUNT_PATH/etc/dib-pip-manifests \
${IMAGE_NAME}-manifests/
fi

View File

@ -0,0 +1,27 @@
#!/bin/bash
#
# Copyright 2014 Hewlett-Packard Development Company, L.P.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
set -eux
dib_pip_manifest_dir=$TMP_MOUNT_PATH/etc/dib-pip-manifests
sudo mkdir $dib_pip_manifest_dir
# Find all of the pip manifests and copy them into the image for use in install.d
for manifest in ${!DIB_PIP_MANIFEST_*}
do
sudo cp ${!manifest} $dib_pip_manifest_dir/dib-pip-build-manifest-${manifest##DIB_PIP_MANIFEST_}
done

View File

@ -0,0 +1,22 @@
#!/bin/bash
#
# Copyright 2014 Hewlett-Packard Development Company, L.P.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
set -eux
install -m 0755 -o root -g root $(dirname $0)/../bin/get-pip-manifest /usr/local/bin/get-pip-manifest
install -m 0755 -o root -g root $(dirname $0)/../bin/write-pip-manifest /usr/local/bin/write-pip-manifest
install -m 0755 -o root -g root $(dirname $0)/../bin/use-pip-manifest /usr/local/bin/use-pip-manifest

View File

@ -0,0 +1 @@
os-svc-install

View File

@ -0,0 +1 @@
os-svc-install