Merge "Create pip manifest files of installed packages"
This commit is contained in:
commit
78517e5a09
|
@ -3,35 +3,45 @@
|
|||
set -eux
|
||||
|
||||
client=$1
|
||||
|
||||
repo=python-${client}client
|
||||
client_manifest=$(get-pip-manifest ${repo})
|
||||
venv_base=/opt/stack/venvs
|
||||
|
||||
# We would like to use --system-site-packages here but if requirements.txt
|
||||
# contains libraries that are installed globally with versions that don't
|
||||
# satisfy our requirements.txt, we end up using the incorrect global library.
|
||||
# Because the global site-packages appears first in sys.path
|
||||
# TODO : Add this back in when we are using virtualenv >= 1.11
|
||||
virtualenv /opt/stack/venvs/$repo
|
||||
virtualenv $venv_base/$repo
|
||||
set +u
|
||||
source /opt/stack/venvs/$repo/bin/activate
|
||||
source $venv_base/$repo/bin/activate
|
||||
set -u
|
||||
|
||||
# Need setuptools>=1.0 to manage connections when
|
||||
# downloading from pypi using http_proxy and https_proxy
|
||||
pip install -U 'setuptools>=1.0'
|
||||
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
pip install -U 'pbr>=0.5.21,<1.0'
|
||||
|
||||
|
||||
pushd /opt/stack/$repo
|
||||
if [ -e requirements.txt ]; then
|
||||
pip install -r requirements.txt
|
||||
elif [ -e tools/pip-requires ]; then
|
||||
pip install -r tools/pip-requires
|
||||
if [ -n "$client_manifest" ]; then
|
||||
use-pip-manifest $client_manifest
|
||||
else
|
||||
# Need setuptools>=1.0 to manage connections when
|
||||
# downloading from pypi using http_proxy and https_proxy
|
||||
pip install -U 'setuptools>=1.0'
|
||||
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
pip install -U 'pbr>=0.5.21,<1.0'
|
||||
|
||||
if [ -e requirements.txt ]; then
|
||||
pip install -r requirements.txt
|
||||
elif [ -e tools/pip-requires ]; then
|
||||
pip install -r tools/pip-requires
|
||||
fi
|
||||
fi
|
||||
|
||||
# Always replay this, as we cannot use the entry this would generate in the manifest
|
||||
python setup.py develop
|
||||
ln -s /opt/stack/venvs/$repo/bin/$client /usr/local/bin/$client
|
||||
|
||||
# Write the manifest of what was installed
|
||||
write-pip-manifest $repo
|
||||
|
||||
ln -s $venv_base/$repo/bin/$client /usr/local/bin/$client
|
||||
popd
|
||||
|
||||
set +u
|
||||
|
|
|
@ -1,2 +1,3 @@
|
|||
source-repositories
|
||||
pip-and-virtualenv
|
||||
pip-manifest
|
||||
|
|
|
@ -1 +1,2 @@
|
|||
os-refresh-config
|
||||
pip-manifest
|
||||
|
|
|
@ -1,16 +1,25 @@
|
|||
#!/bin/bash
|
||||
set -eux
|
||||
|
||||
manifest=$(get-pip-manifest os-apply-config)
|
||||
|
||||
virtualenv --setuptools /opt/stack/venvs/os-apply-config
|
||||
|
||||
# bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which
|
||||
# doesn't manage correctly HTTPS sockets when downloading pbr from
|
||||
# https://pypi.python.org/simple/ if using http_proxy and https_proxy
|
||||
# envvars
|
||||
/opt/stack/venvs/os-apply-config/bin/pip install -U 'setuptools>=1.0'
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
/opt/stack/venvs/os-apply-config/bin/pip install -U 'pbr>=0.6,<1.0'
|
||||
/opt/stack/venvs/os-apply-config/bin/pip install -U os-apply-config
|
||||
if [ -n "$manifest" ]; then
|
||||
use-pip-manifest $manifest
|
||||
else
|
||||
# bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which
|
||||
# doesn't manage correctly HTTPS sockets when downloading pbr from
|
||||
# https://pypi.python.org/simple/ if using http_proxy and https_proxy
|
||||
# envvars
|
||||
/opt/stack/venvs/os-apply-config/bin/pip install -U 'setuptools>=1.0'
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
/opt/stack/venvs/os-apply-config/bin/pip install -U 'pbr>=0.6,<1.0'
|
||||
/opt/stack/venvs/os-apply-config/bin/pip install -U os-apply-config
|
||||
fi
|
||||
|
||||
# Write the manifest of what was installed
|
||||
write-pip-manifest os-apply-config
|
||||
|
||||
ln -s /opt/stack/venvs/os-apply-config/bin/os-apply-config /usr/local/bin/os-apply-config
|
||||
ln -s /opt/stack/venvs/os-apply-config/bin/os-config-applier /usr/local/bin/os-config-applier
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
source-repositories
|
||||
os-refresh-config
|
||||
os-apply-config
|
||||
pip-manifest
|
||||
|
|
|
@ -1,16 +1,25 @@
|
|||
#!/bin/bash
|
||||
set -eux
|
||||
|
||||
manifest=$(get-pip-manifest os-collect-config)
|
||||
|
||||
install-packages build-essential libz-dev libxslt-dev libxml2-dev python-dev
|
||||
|
||||
virtualenv --setuptools /opt/stack/venvs/os-collect-config
|
||||
|
||||
# Need setuptools>=1.0 to manage connections when
|
||||
# downloading from pypi using http_proxy and https_proxy
|
||||
/opt/stack/venvs/os-collect-config/bin/pip install -U 'setuptools>=1.0'
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
/opt/stack/venvs/os-collect-config/bin/pip install -U 'pbr>=0.6,<1.0'
|
||||
/opt/stack/venvs/os-collect-config/bin/pip install -U os-collect-config
|
||||
if [ -n "$manifest" ]; then
|
||||
use-pip-manifest $manifest
|
||||
else
|
||||
# Need setuptools>=1.0 to manage connections when
|
||||
# downloading from pypi using http_proxy and https_proxy
|
||||
/opt/stack/venvs/os-collect-config/bin/pip install -U 'setuptools>=1.0'
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
/opt/stack/venvs/os-collect-config/bin/pip install -U 'pbr>=0.6,<1.0'
|
||||
/opt/stack/venvs/os-collect-config/bin/pip install -U os-collect-config
|
||||
fi
|
||||
|
||||
# Write the manifest of what was installed
|
||||
write-pip-manifest os-collect-config
|
||||
|
||||
ln -s /opt/stack/venvs/os-collect-config/bin/os-collect-config /usr/local/bin/os-collect-config
|
||||
|
||||
|
|
|
@ -2,3 +2,4 @@ pip-and-virtualenv
|
|||
source-repositories
|
||||
os-collect-config
|
||||
os-apply-config
|
||||
pip-manifest
|
||||
|
|
|
@ -4,17 +4,26 @@
|
|||
|
||||
set -eux
|
||||
|
||||
manifest=$(get-pip-manifest os-refresh-config)
|
||||
|
||||
# pip and virtualenv is installed by the pip-and-virtualenv element
|
||||
|
||||
virtualenv --setuptools /opt/stack/venvs/os-refresh-config
|
||||
|
||||
# Need setuptools>=1.0 to manage connections when
|
||||
# downloading from pypi using http_proxy and https_proxy
|
||||
/opt/stack/venvs/os-refresh-config/bin/pip install -U pip
|
||||
/opt/stack/venvs/os-refresh-config/bin/pip install -U 'setuptools>=1.0'
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
/opt/stack/venvs/os-refresh-config/bin/pip install -U 'pbr>=0.5.21,<1.0'
|
||||
/opt/stack/venvs/os-refresh-config/bin/pip install -U os-refresh-config
|
||||
if [ -n "$manifest" ]; then
|
||||
use-pip-manifest $manifest
|
||||
else
|
||||
# Need setuptools>=1.0 to manage connections when
|
||||
# downloading from pypi using http_proxy and https_proxy
|
||||
/opt/stack/venvs/os-refresh-config/bin/pip install -U pip
|
||||
/opt/stack/venvs/os-refresh-config/bin/pip install -U 'setuptools>=1.0'
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
/opt/stack/venvs/os-refresh-config/bin/pip install -U 'pbr>=0.5.21,<1.0'
|
||||
/opt/stack/venvs/os-refresh-config/bin/pip install -U os-refresh-config
|
||||
fi
|
||||
|
||||
# Write the manifest of what was installed
|
||||
write-pip-manifest os-refresh-config
|
||||
|
||||
ln -s /opt/stack/venvs/os-refresh-config/bin/os-refresh-config /usr/local/bin/os-refresh-config
|
||||
|
||||
|
|
|
@ -5,6 +5,8 @@ function python_install() {
|
|||
local svc_root=$1
|
||||
local install_dir=$2
|
||||
local system_site_packages=${3:-"False"}
|
||||
local name=$(basename $install_dir)
|
||||
local svc_manifest=$(get-pip-manifest $name)
|
||||
|
||||
SITE_PCKGS="--no-site-packages"
|
||||
if [ $system_site_packages == "True" ]; then
|
||||
|
@ -17,31 +19,39 @@ function python_install() {
|
|||
source $install_dir/bin/activate
|
||||
set -u
|
||||
|
||||
if [ -e $svc_root/requirements.txt ]; then
|
||||
reqs=$svc_root/requirements.txt
|
||||
elif [ -e $svc_root/tools/pip-requires ]; then
|
||||
reqs=$svc_root/tools/pip-requires
|
||||
# If given an exact deps list, use it, and upgrade to the local git service
|
||||
if [ -n "$svc_manifest" ]; then
|
||||
use-pip-manifest $svc_manifest
|
||||
else
|
||||
reqs=""
|
||||
if [ -e $svc_root/requirements.txt ]; then
|
||||
reqs=$svc_root/requirements.txt
|
||||
elif [ -e $svc_root/tools/pip-requires ]; then
|
||||
reqs=$svc_root/tools/pip-requires
|
||||
else
|
||||
reqs=""
|
||||
fi
|
||||
|
||||
# bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which
|
||||
# doesn't manage correctly HTTPS sockets when downloading pbr from
|
||||
# https://pypi.python.org/simple/ if using http_proxy and
|
||||
# https_proxy envvars
|
||||
pip install -U 'setuptools>=1.0'
|
||||
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
pip install -U 'pbr>=0.5.21,<1.0'
|
||||
|
||||
if [ -n "$reqs" ] ; then
|
||||
pip install -r $reqs
|
||||
# FIXME: pip requires doesn't include MySQL-python
|
||||
pip install MySQL-python
|
||||
fi
|
||||
fi
|
||||
|
||||
# bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which
|
||||
# doesn't manage correctly HTTPS sockets when downloading pbr from
|
||||
# https://pypi.python.org/simple/ if using http_proxy and
|
||||
# https_proxy envvars
|
||||
pip install -U 'setuptools>=1.0'
|
||||
|
||||
# bug #1293812 : Avoid easy_install triggering on pbr.
|
||||
pip install -U 'pbr>=0.5.21,<1.0'
|
||||
|
||||
if [ -n "$reqs" ] ; then
|
||||
pip install -r $reqs
|
||||
# FIXME: pip requires doesn't include MySQL-python
|
||||
pip install MySQL-python
|
||||
fi
|
||||
|
||||
# Always replay this, as we cannot use the entry this would generate in the manifest
|
||||
pip install $svc_root
|
||||
|
||||
# Write the manifest of what was installed
|
||||
write-pip-manifest $name
|
||||
|
||||
set +u
|
||||
deactivate
|
||||
set -u
|
||||
|
|
|
@ -1 +1,2 @@
|
|||
pip-and-virtualenv
|
||||
pip-manifest
|
||||
|
|
|
@ -0,0 +1,63 @@
|
|||
Utility element to enable repeatable pip installs
|
||||
=================================================
|
||||
|
||||
Set this element as a dependency to make the utility scripts available,
|
||||
and then use them as appropriate.
|
||||
|
||||
## Usage
|
||||
This element makes a number of scripts available for use in image building, and
|
||||
performs actions to copy specified manifests into the image for use in building.
|
||||
|
||||
It also copies manifests generated during the build back to the build environment
|
||||
during cleanup.
|
||||
|
||||
## bin
|
||||
Utility scripts for use in other elements to create and reuse pip manifests,
|
||||
as detailed in the usage section.
|
||||
|
||||
### get-pip-manifest
|
||||
Echoes the name of the pip manifest file if one has been copied in, or just
|
||||
returns. The caller passes the name associated with their element, which
|
||||
should be descriptive of the element, to get the correct value.
|
||||
|
||||
For example, the nova element calls `get-pip-manifest nova`.
|
||||
|
||||
The name of the element is transformed to conform with bash variable naming
|
||||
rules, so any charaters that are not [A-Za-z0-9] are replaced with '\_'.
|
||||
|
||||
### use-pip-manifest
|
||||
Uses the given manifest to perform the pip installs necessary.
|
||||
|
||||
Note that any development versions listed in the manifest are not reinstalled.
|
||||
|
||||
The reason for this is that development versions are expected to have been
|
||||
installed from a source other than pypi or the mirror in use, and so development
|
||||
versions will not be reinstallable without extra information.
|
||||
|
||||
The exact details with respect to this are for the relevant consuming element to
|
||||
determine.
|
||||
|
||||
### write-pip-manifest
|
||||
Calls pip freeze and writes the versions of all of the packages currently
|
||||
installed to a manifest file.
|
||||
|
||||
The format of the manifest is the standard python requirements format that is
|
||||
generated by the "pip freeze" command.
|
||||
|
||||
## extra-data.d
|
||||
|
||||
### 75-inject-pip-manifests
|
||||
Copies any pip manifest specified in DIB\_PIP\_MANIFEST\_\* environment variables
|
||||
into the image chroot environment.
|
||||
|
||||
## install.d
|
||||
|
||||
### 01-pip-manifest
|
||||
Installs the scripts in this element into the image for later use by other elements.
|
||||
|
||||
## cleanup.d
|
||||
|
||||
### 01-copy-pip-manifests
|
||||
Runs outside of the chroot and copies created manifests back to the
|
||||
build environment for re-use in repeating the image build.
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
set -eux
|
||||
|
||||
name=${1:?"Usage: ${0} <name>"}
|
||||
build_manifest=/etc/dib-pip-manifests/dib-pip-build-manifest-${name//[^A-Za-z0-9]/_}
|
||||
|
||||
if [[ -f "${build_manifest}" ]]; then
|
||||
echo "${build_manifest}"
|
||||
fi
|
|
@ -0,0 +1,25 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
set -eux
|
||||
|
||||
manifest=${1:?"Usage: ${0} <manifest>"}
|
||||
|
||||
# Comment out the dev versions listed, as they may not be reinstallable
|
||||
sed -i "s/^\(.*[.]dev.*\)$/# \1/g" $manifest
|
||||
|
||||
pip install --no-deps -r $manifest
|
|
@ -0,0 +1,22 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
set -eux
|
||||
|
||||
name=${1:?"Usage: ${0} <name>"}
|
||||
|
||||
pip freeze | tee /etc/dib-pip-manifests/dib-pip-manifest-${name//[^A-Za-z0-9]/_}
|
|
@ -0,0 +1,24 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
set -eux
|
||||
|
||||
if [ -d $TMP_MOUNT_PATH/etc/dib-pip-manifests ]; then
|
||||
mkdir -p ${IMAGE_NAME}-manifests
|
||||
cp --no-preserve=ownership -rv $TMP_MOUNT_PATH/etc/dib-pip-manifests \
|
||||
${IMAGE_NAME}-manifests/
|
||||
fi
|
|
@ -0,0 +1,27 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
set -eux
|
||||
|
||||
dib_pip_manifest_dir=$TMP_MOUNT_PATH/etc/dib-pip-manifests
|
||||
sudo mkdir $dib_pip_manifest_dir
|
||||
|
||||
# Find all of the pip manifests and copy them into the image for use in install.d
|
||||
for manifest in ${!DIB_PIP_MANIFEST_*}
|
||||
do
|
||||
sudo cp ${!manifest} $dib_pip_manifest_dir/dib-pip-build-manifest-${manifest##DIB_PIP_MANIFEST_}
|
||||
done
|
|
@ -0,0 +1,22 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
set -eux
|
||||
|
||||
install -m 0755 -o root -g root $(dirname $0)/../bin/get-pip-manifest /usr/local/bin/get-pip-manifest
|
||||
install -m 0755 -o root -g root $(dirname $0)/../bin/write-pip-manifest /usr/local/bin/write-pip-manifest
|
||||
install -m 0755 -o root -g root $(dirname $0)/../bin/use-pip-manifest /usr/local/bin/use-pip-manifest
|
|
@ -0,0 +1 @@
|
|||
os-svc-install
|
|
@ -0,0 +1 @@
|
|||
os-svc-install
|
Loading…
Reference in New Issue