Create pip manifest files of installed packages
Create the pip-manifest element to encapsulate the saving and use of pip manifest files. Ensure that it installs prior to any elements that should be able to use it. Write pip manifest files for installs in: * os-svc-install * os-apply-config * os-collect-config * os-refresh-config * openstack-clients Enable using a manifest file to reinstall exact versions of packages in a subsequent run through the DIB_PIP_MANIFEST_* environment variables. Change-Id: I4d4ab5921c534727b48cb9969ec8ecfd2c26e6ed
This commit is contained in:
parent
d6d087e3a1
commit
1c998eb93f
|
@ -3,35 +3,45 @@
|
|||
set -eux
|
||||
|
||||
client=$1
|
||||
|
||||
repo=python-${client}client
|
||||
client_manifest=$(get-pip-manifest ${repo})
|
||||
venv_base=/opt/stack/venvs
|
||||
|
||||
# We would like to use --system-site-packages here but if requirements.txt
|
||||
# contains libraries that are installed globally with versions that don't
|
||||
# satisfy our requirements.txt, we end up using the incorrect global library.
|
||||
# Because the global site-packages appears first in sys.path
|
||||
# TODO : Add this back in when we are using virtualenv >= 1.11
|
||||
virtualenv /opt/stack/venvs/$repo
|
||||
virtualenv $venv_base/$repo
|
||||
set +u
|
||||
source /opt/stack/venvs/$repo/bin/activate
|
||||
source $venv_base/$repo/bin/activate
|
||||
set -u
|
||||
|
||||
# Need setuptools>=1.0 to manage connections when
|
||||
# downloading from pypi using http_proxy and https_proxy
|
||||
pip install -U 'setuptools>=1.0'
|
||||
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
pip install -U 'pbr>=0.5.21,<1.0'
|
||||
|
||||
|
||||
pushd /opt/stack/$repo
|
||||
if [ -e requirements.txt ]; then
|
||||
pip install -r requirements.txt
|
||||
elif [ -e tools/pip-requires ]; then
|
||||
pip install -r tools/pip-requires
|
||||
if [ -n "$client_manifest" ]; then
|
||||
use-pip-manifest $client_manifest
|
||||
else
|
||||
# Need setuptools>=1.0 to manage connections when
|
||||
# downloading from pypi using http_proxy and https_proxy
|
||||
pip install -U 'setuptools>=1.0'
|
||||
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
pip install -U 'pbr>=0.5.21,<1.0'
|
||||
|
||||
if [ -e requirements.txt ]; then
|
||||
pip install -r requirements.txt
|
||||
elif [ -e tools/pip-requires ]; then
|
||||
pip install -r tools/pip-requires
|
||||
fi
|
||||
fi
|
||||
|
||||
# Always replay this, as we cannot use the entry this would generate in the manifest
|
||||
python setup.py develop
|
||||
ln -s /opt/stack/venvs/$repo/bin/$client /usr/local/bin/$client
|
||||
|
||||
# Write the manifest of what was installed
|
||||
write-pip-manifest $repo
|
||||
|
||||
ln -s $venv_base/$repo/bin/$client /usr/local/bin/$client
|
||||
popd
|
||||
|
||||
set +u
|
||||
|
|
|
@ -1,2 +1,3 @@
|
|||
source-repositories
|
||||
pip-and-virtualenv
|
||||
pip-manifest
|
||||
|
|
|
@ -1 +1,2 @@
|
|||
os-refresh-config
|
||||
pip-manifest
|
||||
|
|
|
@ -1,16 +1,25 @@
|
|||
#!/bin/bash
|
||||
set -eux
|
||||
|
||||
manifest=$(get-pip-manifest os-apply-config)
|
||||
|
||||
virtualenv --setuptools /opt/stack/venvs/os-apply-config
|
||||
|
||||
# bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which
|
||||
# doesn't manage correctly HTTPS sockets when downloading pbr from
|
||||
# https://pypi.python.org/simple/ if using http_proxy and https_proxy
|
||||
# envvars
|
||||
/opt/stack/venvs/os-apply-config/bin/pip install -U 'setuptools>=1.0'
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
/opt/stack/venvs/os-apply-config/bin/pip install -U 'pbr>=0.6,<1.0'
|
||||
/opt/stack/venvs/os-apply-config/bin/pip install -U os-apply-config
|
||||
if [ -n "$manifest" ]; then
|
||||
use-pip-manifest $manifest
|
||||
else
|
||||
# bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which
|
||||
# doesn't manage correctly HTTPS sockets when downloading pbr from
|
||||
# https://pypi.python.org/simple/ if using http_proxy and https_proxy
|
||||
# envvars
|
||||
/opt/stack/venvs/os-apply-config/bin/pip install -U 'setuptools>=1.0'
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
/opt/stack/venvs/os-apply-config/bin/pip install -U 'pbr>=0.6,<1.0'
|
||||
/opt/stack/venvs/os-apply-config/bin/pip install -U os-apply-config
|
||||
fi
|
||||
|
||||
# Write the manifest of what was installed
|
||||
write-pip-manifest os-apply-config
|
||||
|
||||
ln -s /opt/stack/venvs/os-apply-config/bin/os-apply-config /usr/local/bin/os-apply-config
|
||||
ln -s /opt/stack/venvs/os-apply-config/bin/os-config-applier /usr/local/bin/os-config-applier
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
source-repositories
|
||||
os-refresh-config
|
||||
os-apply-config
|
||||
pip-manifest
|
||||
|
|
|
@ -1,16 +1,25 @@
|
|||
#!/bin/bash
|
||||
set -eux
|
||||
|
||||
manifest=$(get-pip-manifest os-collect-config)
|
||||
|
||||
install-packages build-essential libz-dev libxslt-dev libxml2-dev python-dev
|
||||
|
||||
virtualenv --setuptools /opt/stack/venvs/os-collect-config
|
||||
|
||||
# Need setuptools>=1.0 to manage connections when
|
||||
# downloading from pypi using http_proxy and https_proxy
|
||||
/opt/stack/venvs/os-collect-config/bin/pip install -U 'setuptools>=1.0'
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
/opt/stack/venvs/os-collect-config/bin/pip install -U 'pbr>=0.6,<1.0'
|
||||
/opt/stack/venvs/os-collect-config/bin/pip install -U os-collect-config
|
||||
if [ -n "$manifest" ]; then
|
||||
use-pip-manifest $manifest
|
||||
else
|
||||
# Need setuptools>=1.0 to manage connections when
|
||||
# downloading from pypi using http_proxy and https_proxy
|
||||
/opt/stack/venvs/os-collect-config/bin/pip install -U 'setuptools>=1.0'
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
/opt/stack/venvs/os-collect-config/bin/pip install -U 'pbr>=0.6,<1.0'
|
||||
/opt/stack/venvs/os-collect-config/bin/pip install -U os-collect-config
|
||||
fi
|
||||
|
||||
# Write the manifest of what was installed
|
||||
write-pip-manifest os-collect-config
|
||||
|
||||
ln -s /opt/stack/venvs/os-collect-config/bin/os-collect-config /usr/local/bin/os-collect-config
|
||||
|
||||
|
|
|
@ -2,3 +2,4 @@ pip-and-virtualenv
|
|||
source-repositories
|
||||
os-collect-config
|
||||
os-apply-config
|
||||
pip-manifest
|
||||
|
|
|
@ -4,17 +4,26 @@
|
|||
|
||||
set -eux
|
||||
|
||||
manifest=$(get-pip-manifest os-refresh-config)
|
||||
|
||||
# pip and virtualenv is installed by the pip-and-virtualenv element
|
||||
|
||||
virtualenv --setuptools /opt/stack/venvs/os-refresh-config
|
||||
|
||||
# Need setuptools>=1.0 to manage connections when
|
||||
# downloading from pypi using http_proxy and https_proxy
|
||||
/opt/stack/venvs/os-refresh-config/bin/pip install -U pip
|
||||
/opt/stack/venvs/os-refresh-config/bin/pip install -U 'setuptools>=1.0'
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
/opt/stack/venvs/os-refresh-config/bin/pip install -U 'pbr>=0.5.21,<1.0'
|
||||
/opt/stack/venvs/os-refresh-config/bin/pip install -U os-refresh-config
|
||||
if [ -n "$manifest" ]; then
|
||||
use-pip-manifest $manifest
|
||||
else
|
||||
# Need setuptools>=1.0 to manage connections when
|
||||
# downloading from pypi using http_proxy and https_proxy
|
||||
/opt/stack/venvs/os-refresh-config/bin/pip install -U pip
|
||||
/opt/stack/venvs/os-refresh-config/bin/pip install -U 'setuptools>=1.0'
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
/opt/stack/venvs/os-refresh-config/bin/pip install -U 'pbr>=0.5.21,<1.0'
|
||||
/opt/stack/venvs/os-refresh-config/bin/pip install -U os-refresh-config
|
||||
fi
|
||||
|
||||
# Write the manifest of what was installed
|
||||
write-pip-manifest os-refresh-config
|
||||
|
||||
ln -s /opt/stack/venvs/os-refresh-config/bin/os-refresh-config /usr/local/bin/os-refresh-config
|
||||
|
||||
|
|
|
@ -5,6 +5,8 @@ function python_install() {
|
|||
local svc_root=$1
|
||||
local install_dir=$2
|
||||
local system_site_packages=${3:-"False"}
|
||||
local name=$(basename $install_dir)
|
||||
local svc_manifest=$(get-pip-manifest $name)
|
||||
|
||||
SITE_PCKGS="--no-site-packages"
|
||||
if [ $system_site_packages == "True" ]; then
|
||||
|
@ -17,31 +19,39 @@ function python_install() {
|
|||
source $install_dir/bin/activate
|
||||
set -u
|
||||
|
||||
if [ -e $svc_root/requirements.txt ]; then
|
||||
reqs=$svc_root/requirements.txt
|
||||
elif [ -e $svc_root/tools/pip-requires ]; then
|
||||
reqs=$svc_root/tools/pip-requires
|
||||
# If given an exact deps list, use it, and upgrade to the local git service
|
||||
if [ -n "$svc_manifest" ]; then
|
||||
use-pip-manifest $svc_manifest
|
||||
else
|
||||
reqs=""
|
||||
if [ -e $svc_root/requirements.txt ]; then
|
||||
reqs=$svc_root/requirements.txt
|
||||
elif [ -e $svc_root/tools/pip-requires ]; then
|
||||
reqs=$svc_root/tools/pip-requires
|
||||
else
|
||||
reqs=""
|
||||
fi
|
||||
|
||||
# bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which
|
||||
# doesn't manage correctly HTTPS sockets when downloading pbr from
|
||||
# https://pypi.python.org/simple/ if using http_proxy and
|
||||
# https_proxy envvars
|
||||
pip install -U 'setuptools>=1.0'
|
||||
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
pip install -U 'pbr>=0.5.21,<1.0'
|
||||
|
||||
if [ -n "$reqs" ] ; then
|
||||
pip install -r $reqs
|
||||
# FIXME: pip requires doesn't include MySQL-python
|
||||
pip install MySQL-python
|
||||
fi
|
||||
fi
|
||||
|
||||
# bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which
|
||||
# doesn't manage correctly HTTPS sockets when downloading pbr from
|
||||
# https://pypi.python.org/simple/ if using http_proxy and
|
||||
# https_proxy envvars
|
||||
pip install -U 'setuptools>=1.0'
|
||||
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
pip install -U 'pbr>=0.5.21,<1.0'
|
||||
|
||||
if [ -n "$reqs" ] ; then
|
||||
pip install -r $reqs
|
||||
# FIXME: pip requires doesn't include MySQL-python
|
||||
pip install MySQL-python
|
||||
fi
|
||||
|
||||
# Always replay this, as we cannot use the entry this would generate in the manifest
|
||||
pip install $svc_root
|
||||
|
||||
# Write the manifest of what was installed
|
||||
write-pip-manifest $name
|
||||
|
||||
set +u
|
||||
deactivate
|
||||
set -u
|
||||
|
|
|
@ -1 +1,2 @@
|
|||
pip-and-virtualenv
|
||||
pip-manifest
|
||||
|
|
|
@ -0,0 +1,63 @@
|
|||
Utility element to enable repeatable pip installs
|
||||
=================================================
|
||||
|
||||
Set this element as a dependency to make the utility scripts available,
|
||||
and then use them as appropriate.
|
||||
|
||||
## Usage
|
||||
This element makes a number of scripts available for use in image building, and
|
||||
performs actions to copy specified manifests into the image for use in building.
|
||||
|
||||
It also copies manifests generated during the build back to the build environment
|
||||
during cleanup.
|
||||
|
||||
## bin
|
||||
Utility scripts for use in other elements to create and reuse pip manifests,
|
||||
as detailed in the usage section.
|
||||
|
||||
### get-pip-manifest
|
||||
Echoes the name of the pip manifest file if one has been copied in, or just
|
||||
returns. The caller passes the name associated with their element, which
|
||||
should be descriptive of the element, to get the correct value.
|
||||
|
||||
For example, the nova element calls `get-pip-manifest nova`.
|
||||
|
||||
The name of the element is transformed to conform with bash variable naming
|
||||
rules, so any charaters that are not [A-Za-z0-9] are replaced with '\_'.
|
||||
|
||||
### use-pip-manifest
|
||||
Uses the given manifest to perform the pip installs necessary.
|
||||
|
||||
Note that any development versions listed in the manifest are not reinstalled.
|
||||
|
||||
The reason for this is that development versions are expected to have been
|
||||
installed from a source other than pypi or the mirror in use, and so development
|
||||
versions will not be reinstallable without extra information.
|
||||
|
||||
The exact details with respect to this are for the relevant consuming element to
|
||||
determine.
|
||||
|
||||
### write-pip-manifest
|
||||
Calls pip freeze and writes the versions of all of the packages currently
|
||||
installed to a manifest file.
|
||||
|
||||
The format of the manifest is the standard python requirements format that is
|
||||
generated by the "pip freeze" command.
|
||||
|
||||
## extra-data.d
|
||||
|
||||
### 75-inject-pip-manifests
|
||||
Copies any pip manifest specified in DIB\_PIP\_MANIFEST\_\* environment variables
|
||||
into the image chroot environment.
|
||||
|
||||
## install.d
|
||||
|
||||
### 01-pip-manifest
|
||||
Installs the scripts in this element into the image for later use by other elements.
|
||||
|
||||
## cleanup.d
|
||||
|
||||
### 01-copy-pip-manifests
|
||||
Runs outside of the chroot and copies created manifests back to the
|
||||
build environment for re-use in repeating the image build.
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
set -eux
|
||||
|
||||
name=${1:?"Usage: ${0} <name>"}
|
||||
build_manifest=/etc/dib-pip-manifests/dib-pip-build-manifest-${name//[^A-Za-z0-9]/_}
|
||||
|
||||
if [[ -f "${build_manifest}" ]]; then
|
||||
echo "${build_manifest}"
|
||||
fi
|
|
@ -0,0 +1,25 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
set -eux
|
||||
|
||||
manifest=${1:?"Usage: ${0} <manifest>"}
|
||||
|
||||
# Comment out the dev versions listed, as they may not be reinstallable
|
||||
sed -i "s/^\(.*[.]dev.*\)$/# \1/g" $manifest
|
||||
|
||||
pip install --no-deps -r $manifest
|
|
@ -0,0 +1,22 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
set -eux
|
||||
|
||||
name=${1:?"Usage: ${0} <name>"}
|
||||
|
||||
pip freeze | tee /etc/dib-pip-manifests/dib-pip-manifest-${name//[^A-Za-z0-9]/_}
|
|
@ -0,0 +1,24 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
set -eux
|
||||
|
||||
if [ -d $TMP_MOUNT_PATH/etc/dib-pip-manifests ]; then
|
||||
mkdir -p ${IMAGE_NAME}-manifests
|
||||
cp --no-preserve=ownership -rv $TMP_MOUNT_PATH/etc/dib-pip-manifests \
|
||||
${IMAGE_NAME}-manifests/
|
||||
fi
|
|
@ -0,0 +1,27 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
set -eux
|
||||
|
||||
dib_pip_manifest_dir=$TMP_MOUNT_PATH/etc/dib-pip-manifests
|
||||
sudo mkdir $dib_pip_manifest_dir
|
||||
|
||||
# Find all of the pip manifests and copy them into the image for use in install.d
|
||||
for manifest in ${!DIB_PIP_MANIFEST_*}
|
||||
do
|
||||
sudo cp ${!manifest} $dib_pip_manifest_dir/dib-pip-build-manifest-${manifest##DIB_PIP_MANIFEST_}
|
||||
done
|
|
@ -0,0 +1,22 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
set -eux
|
||||
|
||||
install -m 0755 -o root -g root $(dirname $0)/../bin/get-pip-manifest /usr/local/bin/get-pip-manifest
|
||||
install -m 0755 -o root -g root $(dirname $0)/../bin/write-pip-manifest /usr/local/bin/write-pip-manifest
|
||||
install -m 0755 -o root -g root $(dirname $0)/../bin/use-pip-manifest /usr/local/bin/use-pip-manifest
|
|
@ -0,0 +1 @@
|
|||
os-svc-install
|
|
@ -0,0 +1 @@
|
|||
os-svc-install
|
Loading…
Reference in New Issue