From 1c998eb93f6938c10a546c9daf15b5bd109e9198 Mon Sep 17 00:00:00 2001 From: Jon-Paul Sullivan Date: Wed, 12 Mar 2014 19:47:41 +0000 Subject: [PATCH] Create pip manifest files of installed packages Create the pip-manifest element to encapsulate the saving and use of pip manifest files. Ensure that it installs prior to any elements that should be able to use it. Write pip manifest files for installs in: * os-svc-install * os-apply-config * os-collect-config * os-refresh-config * openstack-clients Enable using a manifest file to reinstall exact versions of packages in a subsequent run through the DIB_PIP_MANIFEST_* environment variables. Change-Id: I4d4ab5921c534727b48cb9969ec8ecfd2c26e6ed --- .../bin/install-openstack-client | 42 ++++++++----- elements/openstack-clients/element-deps | 1 + elements/os-apply-config/element-deps | 1 + .../10-os-config-applier | 25 +++++--- elements/os-collect-config/element-deps | 1 + .../10-os-collect-config | 21 +++++-- elements/os-refresh-config/element-deps | 1 + .../10-os-refresh-config | 23 ++++--- elements/os-svc-install/bin/os-svc-install | 52 ++++++++------- elements/os-svc-install/element-deps | 1 + elements/pip-manifest/README.md | 63 +++++++++++++++++++ elements/pip-manifest/bin/get-pip-manifest | 25 ++++++++ elements/pip-manifest/bin/use-pip-manifest | 25 ++++++++ elements/pip-manifest/bin/write-pip-manifest | 22 +++++++ .../cleanup.d/01-copy-pip-manifests | 24 +++++++ .../extra-data.d/75-inject-pip-manifests | 27 ++++++++ .../pip-manifest/install.d/01-pip-manifest | 22 +++++++ elements/swift/element-deps | 1 + elements/tempest/element-deps | 1 + 19 files changed, 320 insertions(+), 58 deletions(-) create mode 100644 elements/pip-manifest/README.md create mode 100755 elements/pip-manifest/bin/get-pip-manifest create mode 100755 elements/pip-manifest/bin/use-pip-manifest create mode 100755 elements/pip-manifest/bin/write-pip-manifest create mode 100755 elements/pip-manifest/cleanup.d/01-copy-pip-manifests create mode 100755 elements/pip-manifest/extra-data.d/75-inject-pip-manifests create mode 100755 elements/pip-manifest/install.d/01-pip-manifest create mode 100644 elements/swift/element-deps create mode 100644 elements/tempest/element-deps diff --git a/elements/openstack-clients/bin/install-openstack-client b/elements/openstack-clients/bin/install-openstack-client index e0fcd3759..bb6f64658 100755 --- a/elements/openstack-clients/bin/install-openstack-client +++ b/elements/openstack-clients/bin/install-openstack-client @@ -3,35 +3,45 @@ set -eux client=$1 - repo=python-${client}client +client_manifest=$(get-pip-manifest ${repo}) +venv_base=/opt/stack/venvs # We would like to use --system-site-packages here but if requirements.txt # contains libraries that are installed globally with versions that don't # satisfy our requirements.txt, we end up using the incorrect global library. # Because the global site-packages appears first in sys.path # TODO : Add this back in when we are using virtualenv >= 1.11 -virtualenv /opt/stack/venvs/$repo +virtualenv $venv_base/$repo set +u -source /opt/stack/venvs/$repo/bin/activate +source $venv_base/$repo/bin/activate set -u -# Need setuptools>=1.0 to manage connections when -# downloading from pypi using http_proxy and https_proxy -pip install -U 'setuptools>=1.0' - -# bug #1293812 : Avoid easy_install triggering on pbr. -pip install -U 'pbr>=0.5.21,<1.0' - - pushd /opt/stack/$repo -if [ -e requirements.txt ]; then - pip install -r requirements.txt -elif [ -e tools/pip-requires ]; then - pip install -r tools/pip-requires +if [ -n "$client_manifest" ]; then + use-pip-manifest $client_manifest +else + # Need setuptools>=1.0 to manage connections when + # downloading from pypi using http_proxy and https_proxy + pip install -U 'setuptools>=1.0' + + # bug #1293812 : Avoid easy_install triggering on pbr. + pip install -U 'pbr>=0.5.21,<1.0' + + if [ -e requirements.txt ]; then + pip install -r requirements.txt + elif [ -e tools/pip-requires ]; then + pip install -r tools/pip-requires + fi fi + +# Always replay this, as we cannot use the entry this would generate in the manifest python setup.py develop -ln -s /opt/stack/venvs/$repo/bin/$client /usr/local/bin/$client + +# Write the manifest of what was installed +write-pip-manifest $repo + +ln -s $venv_base/$repo/bin/$client /usr/local/bin/$client popd set +u diff --git a/elements/openstack-clients/element-deps b/elements/openstack-clients/element-deps index 887eaa8db..f15b616ff 100644 --- a/elements/openstack-clients/element-deps +++ b/elements/openstack-clients/element-deps @@ -1,2 +1,3 @@ source-repositories pip-and-virtualenv +pip-manifest diff --git a/elements/os-apply-config/element-deps b/elements/os-apply-config/element-deps index 33d7e82b2..b8e381391 100644 --- a/elements/os-apply-config/element-deps +++ b/elements/os-apply-config/element-deps @@ -1 +1,2 @@ os-refresh-config +pip-manifest diff --git a/elements/os-apply-config/install.d/os-apply-config-source-install/10-os-config-applier b/elements/os-apply-config/install.d/os-apply-config-source-install/10-os-config-applier index da2c1225f..0a1068c19 100755 --- a/elements/os-apply-config/install.d/os-apply-config-source-install/10-os-config-applier +++ b/elements/os-apply-config/install.d/os-apply-config-source-install/10-os-config-applier @@ -1,16 +1,25 @@ #!/bin/bash set -eux +manifest=$(get-pip-manifest os-apply-config) + virtualenv --setuptools /opt/stack/venvs/os-apply-config -# bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which -# doesn't manage correctly HTTPS sockets when downloading pbr from -# https://pypi.python.org/simple/ if using http_proxy and https_proxy -# envvars -/opt/stack/venvs/os-apply-config/bin/pip install -U 'setuptools>=1.0' -# bug #1293812 : Avoid easy_install triggering on pbr. -/opt/stack/venvs/os-apply-config/bin/pip install -U 'pbr>=0.6,<1.0' -/opt/stack/venvs/os-apply-config/bin/pip install -U os-apply-config +if [ -n "$manifest" ]; then + use-pip-manifest $manifest +else + # bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which + # doesn't manage correctly HTTPS sockets when downloading pbr from + # https://pypi.python.org/simple/ if using http_proxy and https_proxy + # envvars + /opt/stack/venvs/os-apply-config/bin/pip install -U 'setuptools>=1.0' + # bug #1293812 : Avoid easy_install triggering on pbr. + /opt/stack/venvs/os-apply-config/bin/pip install -U 'pbr>=0.6,<1.0' + /opt/stack/venvs/os-apply-config/bin/pip install -U os-apply-config +fi + +# Write the manifest of what was installed +write-pip-manifest os-apply-config ln -s /opt/stack/venvs/os-apply-config/bin/os-apply-config /usr/local/bin/os-apply-config ln -s /opt/stack/venvs/os-apply-config/bin/os-config-applier /usr/local/bin/os-config-applier diff --git a/elements/os-collect-config/element-deps b/elements/os-collect-config/element-deps index bfb3b733a..a635a3a03 100644 --- a/elements/os-collect-config/element-deps +++ b/elements/os-collect-config/element-deps @@ -1,3 +1,4 @@ source-repositories os-refresh-config os-apply-config +pip-manifest diff --git a/elements/os-collect-config/install.d/os-collect-config-source-install/10-os-collect-config b/elements/os-collect-config/install.d/os-collect-config-source-install/10-os-collect-config index 2de3f689d..3ded73b35 100755 --- a/elements/os-collect-config/install.d/os-collect-config-source-install/10-os-collect-config +++ b/elements/os-collect-config/install.d/os-collect-config-source-install/10-os-collect-config @@ -1,16 +1,25 @@ #!/bin/bash set -eux +manifest=$(get-pip-manifest os-collect-config) + install-packages build-essential libz-dev libxslt-dev libxml2-dev python-dev virtualenv --setuptools /opt/stack/venvs/os-collect-config -# Need setuptools>=1.0 to manage connections when -# downloading from pypi using http_proxy and https_proxy -/opt/stack/venvs/os-collect-config/bin/pip install -U 'setuptools>=1.0' -# bug #1293812 : Avoid easy_install triggering on pbr. -/opt/stack/venvs/os-collect-config/bin/pip install -U 'pbr>=0.6,<1.0' -/opt/stack/venvs/os-collect-config/bin/pip install -U os-collect-config +if [ -n "$manifest" ]; then + use-pip-manifest $manifest +else + # Need setuptools>=1.0 to manage connections when + # downloading from pypi using http_proxy and https_proxy + /opt/stack/venvs/os-collect-config/bin/pip install -U 'setuptools>=1.0' + # bug #1293812 : Avoid easy_install triggering on pbr. + /opt/stack/venvs/os-collect-config/bin/pip install -U 'pbr>=0.6,<1.0' + /opt/stack/venvs/os-collect-config/bin/pip install -U os-collect-config +fi + +# Write the manifest of what was installed +write-pip-manifest os-collect-config ln -s /opt/stack/venvs/os-collect-config/bin/os-collect-config /usr/local/bin/os-collect-config diff --git a/elements/os-refresh-config/element-deps b/elements/os-refresh-config/element-deps index 9eff0a75b..a124d5280 100644 --- a/elements/os-refresh-config/element-deps +++ b/elements/os-refresh-config/element-deps @@ -2,3 +2,4 @@ pip-and-virtualenv source-repositories os-collect-config os-apply-config +pip-manifest diff --git a/elements/os-refresh-config/install.d/os-refresh-config-source-install/10-os-refresh-config b/elements/os-refresh-config/install.d/os-refresh-config-source-install/10-os-refresh-config index e81979260..18c9f334f 100755 --- a/elements/os-refresh-config/install.d/os-refresh-config-source-install/10-os-refresh-config +++ b/elements/os-refresh-config/install.d/os-refresh-config-source-install/10-os-refresh-config @@ -4,17 +4,26 @@ set -eux +manifest=$(get-pip-manifest os-refresh-config) + # pip and virtualenv is installed by the pip-and-virtualenv element virtualenv --setuptools /opt/stack/venvs/os-refresh-config -# Need setuptools>=1.0 to manage connections when -# downloading from pypi using http_proxy and https_proxy -/opt/stack/venvs/os-refresh-config/bin/pip install -U pip -/opt/stack/venvs/os-refresh-config/bin/pip install -U 'setuptools>=1.0' -# bug #1293812 : Avoid easy_install triggering on pbr. -/opt/stack/venvs/os-refresh-config/bin/pip install -U 'pbr>=0.5.21,<1.0' -/opt/stack/venvs/os-refresh-config/bin/pip install -U os-refresh-config +if [ -n "$manifest" ]; then + use-pip-manifest $manifest +else + # Need setuptools>=1.0 to manage connections when + # downloading from pypi using http_proxy and https_proxy + /opt/stack/venvs/os-refresh-config/bin/pip install -U pip + /opt/stack/venvs/os-refresh-config/bin/pip install -U 'setuptools>=1.0' + # bug #1293812 : Avoid easy_install triggering on pbr. + /opt/stack/venvs/os-refresh-config/bin/pip install -U 'pbr>=0.5.21,<1.0' + /opt/stack/venvs/os-refresh-config/bin/pip install -U os-refresh-config +fi + +# Write the manifest of what was installed +write-pip-manifest os-refresh-config ln -s /opt/stack/venvs/os-refresh-config/bin/os-refresh-config /usr/local/bin/os-refresh-config diff --git a/elements/os-svc-install/bin/os-svc-install b/elements/os-svc-install/bin/os-svc-install index b6ab2d95c..df74d1622 100755 --- a/elements/os-svc-install/bin/os-svc-install +++ b/elements/os-svc-install/bin/os-svc-install @@ -5,6 +5,8 @@ function python_install() { local svc_root=$1 local install_dir=$2 local system_site_packages=${3:-"False"} + local name=$(basename $install_dir) + local svc_manifest=$(get-pip-manifest $name) SITE_PCKGS="--no-site-packages" if [ $system_site_packages == "True" ]; then @@ -17,31 +19,39 @@ function python_install() { source $install_dir/bin/activate set -u - if [ -e $svc_root/requirements.txt ]; then - reqs=$svc_root/requirements.txt - elif [ -e $svc_root/tools/pip-requires ]; then - reqs=$svc_root/tools/pip-requires + # If given an exact deps list, use it, and upgrade to the local git service + if [ -n "$svc_manifest" ]; then + use-pip-manifest $svc_manifest else - reqs="" + if [ -e $svc_root/requirements.txt ]; then + reqs=$svc_root/requirements.txt + elif [ -e $svc_root/tools/pip-requires ]; then + reqs=$svc_root/tools/pip-requires + else + reqs="" + fi + + # bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which + # doesn't manage correctly HTTPS sockets when downloading pbr from + # https://pypi.python.org/simple/ if using http_proxy and + # https_proxy envvars + pip install -U 'setuptools>=1.0' + + # bug #1293812 : Avoid easy_install triggering on pbr. + pip install -U 'pbr>=0.5.21,<1.0' + + if [ -n "$reqs" ] ; then + pip install -r $reqs + # FIXME: pip requires doesn't include MySQL-python + pip install MySQL-python + fi fi - - # bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which - # doesn't manage correctly HTTPS sockets when downloading pbr from - # https://pypi.python.org/simple/ if using http_proxy and - # https_proxy envvars - pip install -U 'setuptools>=1.0' - - # bug #1293812 : Avoid easy_install triggering on pbr. - pip install -U 'pbr>=0.5.21,<1.0' - - if [ -n "$reqs" ] ; then - pip install -r $reqs - # FIXME: pip requires doesn't include MySQL-python - pip install MySQL-python - fi - + # Always replay this, as we cannot use the entry this would generate in the manifest pip install $svc_root + # Write the manifest of what was installed + write-pip-manifest $name + set +u deactivate set -u diff --git a/elements/os-svc-install/element-deps b/elements/os-svc-install/element-deps index 7c8922a0b..9770f9ba6 100644 --- a/elements/os-svc-install/element-deps +++ b/elements/os-svc-install/element-deps @@ -1 +1,2 @@ pip-and-virtualenv +pip-manifest diff --git a/elements/pip-manifest/README.md b/elements/pip-manifest/README.md new file mode 100644 index 000000000..351968e29 --- /dev/null +++ b/elements/pip-manifest/README.md @@ -0,0 +1,63 @@ +Utility element to enable repeatable pip installs +================================================= + +Set this element as a dependency to make the utility scripts available, +and then use them as appropriate. + +## Usage +This element makes a number of scripts available for use in image building, and +performs actions to copy specified manifests into the image for use in building. + +It also copies manifests generated during the build back to the build environment +during cleanup. + +## bin +Utility scripts for use in other elements to create and reuse pip manifests, +as detailed in the usage section. + +### get-pip-manifest +Echoes the name of the pip manifest file if one has been copied in, or just +returns. The caller passes the name associated with their element, which +should be descriptive of the element, to get the correct value. + +For example, the nova element calls `get-pip-manifest nova`. + +The name of the element is transformed to conform with bash variable naming +rules, so any charaters that are not [A-Za-z0-9] are replaced with '\_'. + +### use-pip-manifest +Uses the given manifest to perform the pip installs necessary. + +Note that any development versions listed in the manifest are not reinstalled. + +The reason for this is that development versions are expected to have been +installed from a source other than pypi or the mirror in use, and so development +versions will not be reinstallable without extra information. + +The exact details with respect to this are for the relevant consuming element to +determine. + +### write-pip-manifest +Calls pip freeze and writes the versions of all of the packages currently +installed to a manifest file. + +The format of the manifest is the standard python requirements format that is +generated by the "pip freeze" command. + +## extra-data.d + +### 75-inject-pip-manifests +Copies any pip manifest specified in DIB\_PIP\_MANIFEST\_\* environment variables +into the image chroot environment. + +## install.d + +### 01-pip-manifest +Installs the scripts in this element into the image for later use by other elements. + +## cleanup.d + +### 01-copy-pip-manifests +Runs outside of the chroot and copies created manifests back to the +build environment for re-use in repeating the image build. + diff --git a/elements/pip-manifest/bin/get-pip-manifest b/elements/pip-manifest/bin/get-pip-manifest new file mode 100755 index 000000000..bf52eb658 --- /dev/null +++ b/elements/pip-manifest/bin/get-pip-manifest @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -eux + +name=${1:?"Usage: ${0} "} +build_manifest=/etc/dib-pip-manifests/dib-pip-build-manifest-${name//[^A-Za-z0-9]/_} + +if [[ -f "${build_manifest}" ]]; then + echo "${build_manifest}" +fi diff --git a/elements/pip-manifest/bin/use-pip-manifest b/elements/pip-manifest/bin/use-pip-manifest new file mode 100755 index 000000000..fbcb6953a --- /dev/null +++ b/elements/pip-manifest/bin/use-pip-manifest @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -eux + +manifest=${1:?"Usage: ${0} "} + +# Comment out the dev versions listed, as they may not be reinstallable +sed -i "s/^\(.*[.]dev.*\)$/# \1/g" $manifest + +pip install --no-deps -r $manifest diff --git a/elements/pip-manifest/bin/write-pip-manifest b/elements/pip-manifest/bin/write-pip-manifest new file mode 100755 index 000000000..8e38ca300 --- /dev/null +++ b/elements/pip-manifest/bin/write-pip-manifest @@ -0,0 +1,22 @@ +#!/bin/bash +# +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -eux + +name=${1:?"Usage: ${0} "} + +pip freeze | tee /etc/dib-pip-manifests/dib-pip-manifest-${name//[^A-Za-z0-9]/_} diff --git a/elements/pip-manifest/cleanup.d/01-copy-pip-manifests b/elements/pip-manifest/cleanup.d/01-copy-pip-manifests new file mode 100755 index 000000000..77314005c --- /dev/null +++ b/elements/pip-manifest/cleanup.d/01-copy-pip-manifests @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -eux + +if [ -d $TMP_MOUNT_PATH/etc/dib-pip-manifests ]; then + mkdir -p ${IMAGE_NAME}-manifests + cp --no-preserve=ownership -rv $TMP_MOUNT_PATH/etc/dib-pip-manifests \ + ${IMAGE_NAME}-manifests/ +fi diff --git a/elements/pip-manifest/extra-data.d/75-inject-pip-manifests b/elements/pip-manifest/extra-data.d/75-inject-pip-manifests new file mode 100755 index 000000000..20f80bd68 --- /dev/null +++ b/elements/pip-manifest/extra-data.d/75-inject-pip-manifests @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -eux + +dib_pip_manifest_dir=$TMP_MOUNT_PATH/etc/dib-pip-manifests +sudo mkdir $dib_pip_manifest_dir + +# Find all of the pip manifests and copy them into the image for use in install.d +for manifest in ${!DIB_PIP_MANIFEST_*} +do + sudo cp ${!manifest} $dib_pip_manifest_dir/dib-pip-build-manifest-${manifest##DIB_PIP_MANIFEST_} +done diff --git a/elements/pip-manifest/install.d/01-pip-manifest b/elements/pip-manifest/install.d/01-pip-manifest new file mode 100755 index 000000000..0906858c5 --- /dev/null +++ b/elements/pip-manifest/install.d/01-pip-manifest @@ -0,0 +1,22 @@ +#!/bin/bash +# +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -eux + +install -m 0755 -o root -g root $(dirname $0)/../bin/get-pip-manifest /usr/local/bin/get-pip-manifest +install -m 0755 -o root -g root $(dirname $0)/../bin/write-pip-manifest /usr/local/bin/write-pip-manifest +install -m 0755 -o root -g root $(dirname $0)/../bin/use-pip-manifest /usr/local/bin/use-pip-manifest diff --git a/elements/swift/element-deps b/elements/swift/element-deps new file mode 100644 index 000000000..2167c86fb --- /dev/null +++ b/elements/swift/element-deps @@ -0,0 +1 @@ +os-svc-install diff --git a/elements/tempest/element-deps b/elements/tempest/element-deps new file mode 100644 index 000000000..2167c86fb --- /dev/null +++ b/elements/tempest/element-deps @@ -0,0 +1 @@ +os-svc-install