diff --git a/elements/openstack-clients/bin/install-openstack-client b/elements/openstack-clients/bin/install-openstack-client index e0fcd3759..bb6f64658 100755 --- a/elements/openstack-clients/bin/install-openstack-client +++ b/elements/openstack-clients/bin/install-openstack-client @@ -3,35 +3,45 @@ set -eux client=$1 - repo=python-${client}client +client_manifest=$(get-pip-manifest ${repo}) +venv_base=/opt/stack/venvs # We would like to use --system-site-packages here but if requirements.txt # contains libraries that are installed globally with versions that don't # satisfy our requirements.txt, we end up using the incorrect global library. # Because the global site-packages appears first in sys.path # TODO : Add this back in when we are using virtualenv >= 1.11 -virtualenv /opt/stack/venvs/$repo +virtualenv $venv_base/$repo set +u -source /opt/stack/venvs/$repo/bin/activate +source $venv_base/$repo/bin/activate set -u -# Need setuptools>=1.0 to manage connections when -# downloading from pypi using http_proxy and https_proxy -pip install -U 'setuptools>=1.0' - -# bug #1293812 : Avoid easy_install triggering on pbr. -pip install -U 'pbr>=0.5.21,<1.0' - - pushd /opt/stack/$repo -if [ -e requirements.txt ]; then - pip install -r requirements.txt -elif [ -e tools/pip-requires ]; then - pip install -r tools/pip-requires +if [ -n "$client_manifest" ]; then + use-pip-manifest $client_manifest +else + # Need setuptools>=1.0 to manage connections when + # downloading from pypi using http_proxy and https_proxy + pip install -U 'setuptools>=1.0' + + # bug #1293812 : Avoid easy_install triggering on pbr. + pip install -U 'pbr>=0.5.21,<1.0' + + if [ -e requirements.txt ]; then + pip install -r requirements.txt + elif [ -e tools/pip-requires ]; then + pip install -r tools/pip-requires + fi fi + +# Always replay this, as we cannot use the entry this would generate in the manifest python setup.py develop -ln -s /opt/stack/venvs/$repo/bin/$client /usr/local/bin/$client + +# Write the manifest of what was installed +write-pip-manifest $repo + +ln -s $venv_base/$repo/bin/$client /usr/local/bin/$client popd set +u diff --git a/elements/openstack-clients/element-deps b/elements/openstack-clients/element-deps index 887eaa8db..f15b616ff 100644 --- a/elements/openstack-clients/element-deps +++ b/elements/openstack-clients/element-deps @@ -1,2 +1,3 @@ source-repositories pip-and-virtualenv +pip-manifest diff --git a/elements/os-apply-config/element-deps b/elements/os-apply-config/element-deps index 33d7e82b2..b8e381391 100644 --- a/elements/os-apply-config/element-deps +++ b/elements/os-apply-config/element-deps @@ -1 +1,2 @@ os-refresh-config +pip-manifest diff --git a/elements/os-apply-config/install.d/os-apply-config-source-install/10-os-config-applier b/elements/os-apply-config/install.d/os-apply-config-source-install/10-os-config-applier index da2c1225f..0a1068c19 100755 --- a/elements/os-apply-config/install.d/os-apply-config-source-install/10-os-config-applier +++ b/elements/os-apply-config/install.d/os-apply-config-source-install/10-os-config-applier @@ -1,16 +1,25 @@ #!/bin/bash set -eux +manifest=$(get-pip-manifest os-apply-config) + virtualenv --setuptools /opt/stack/venvs/os-apply-config -# bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which -# doesn't manage correctly HTTPS sockets when downloading pbr from -# https://pypi.python.org/simple/ if using http_proxy and https_proxy -# envvars -/opt/stack/venvs/os-apply-config/bin/pip install -U 'setuptools>=1.0' -# bug #1293812 : Avoid easy_install triggering on pbr. -/opt/stack/venvs/os-apply-config/bin/pip install -U 'pbr>=0.6,<1.0' -/opt/stack/venvs/os-apply-config/bin/pip install -U os-apply-config +if [ -n "$manifest" ]; then + use-pip-manifest $manifest +else + # bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which + # doesn't manage correctly HTTPS sockets when downloading pbr from + # https://pypi.python.org/simple/ if using http_proxy and https_proxy + # envvars + /opt/stack/venvs/os-apply-config/bin/pip install -U 'setuptools>=1.0' + # bug #1293812 : Avoid easy_install triggering on pbr. + /opt/stack/venvs/os-apply-config/bin/pip install -U 'pbr>=0.6,<1.0' + /opt/stack/venvs/os-apply-config/bin/pip install -U os-apply-config +fi + +# Write the manifest of what was installed +write-pip-manifest os-apply-config ln -s /opt/stack/venvs/os-apply-config/bin/os-apply-config /usr/local/bin/os-apply-config ln -s /opt/stack/venvs/os-apply-config/bin/os-config-applier /usr/local/bin/os-config-applier diff --git a/elements/os-collect-config/element-deps b/elements/os-collect-config/element-deps index bfb3b733a..a635a3a03 100644 --- a/elements/os-collect-config/element-deps +++ b/elements/os-collect-config/element-deps @@ -1,3 +1,4 @@ source-repositories os-refresh-config os-apply-config +pip-manifest diff --git a/elements/os-collect-config/install.d/os-collect-config-source-install/10-os-collect-config b/elements/os-collect-config/install.d/os-collect-config-source-install/10-os-collect-config index 2de3f689d..3ded73b35 100755 --- a/elements/os-collect-config/install.d/os-collect-config-source-install/10-os-collect-config +++ b/elements/os-collect-config/install.d/os-collect-config-source-install/10-os-collect-config @@ -1,16 +1,25 @@ #!/bin/bash set -eux +manifest=$(get-pip-manifest os-collect-config) + install-packages build-essential libz-dev libxslt-dev libxml2-dev python-dev virtualenv --setuptools /opt/stack/venvs/os-collect-config -# Need setuptools>=1.0 to manage connections when -# downloading from pypi using http_proxy and https_proxy -/opt/stack/venvs/os-collect-config/bin/pip install -U 'setuptools>=1.0' -# bug #1293812 : Avoid easy_install triggering on pbr. -/opt/stack/venvs/os-collect-config/bin/pip install -U 'pbr>=0.6,<1.0' -/opt/stack/venvs/os-collect-config/bin/pip install -U os-collect-config +if [ -n "$manifest" ]; then + use-pip-manifest $manifest +else + # Need setuptools>=1.0 to manage connections when + # downloading from pypi using http_proxy and https_proxy + /opt/stack/venvs/os-collect-config/bin/pip install -U 'setuptools>=1.0' + # bug #1293812 : Avoid easy_install triggering on pbr. + /opt/stack/venvs/os-collect-config/bin/pip install -U 'pbr>=0.6,<1.0' + /opt/stack/venvs/os-collect-config/bin/pip install -U os-collect-config +fi + +# Write the manifest of what was installed +write-pip-manifest os-collect-config ln -s /opt/stack/venvs/os-collect-config/bin/os-collect-config /usr/local/bin/os-collect-config diff --git a/elements/os-refresh-config/element-deps b/elements/os-refresh-config/element-deps index 9eff0a75b..a124d5280 100644 --- a/elements/os-refresh-config/element-deps +++ b/elements/os-refresh-config/element-deps @@ -2,3 +2,4 @@ pip-and-virtualenv source-repositories os-collect-config os-apply-config +pip-manifest diff --git a/elements/os-refresh-config/install.d/os-refresh-config-source-install/10-os-refresh-config b/elements/os-refresh-config/install.d/os-refresh-config-source-install/10-os-refresh-config index e81979260..18c9f334f 100755 --- a/elements/os-refresh-config/install.d/os-refresh-config-source-install/10-os-refresh-config +++ b/elements/os-refresh-config/install.d/os-refresh-config-source-install/10-os-refresh-config @@ -4,17 +4,26 @@ set -eux +manifest=$(get-pip-manifest os-refresh-config) + # pip and virtualenv is installed by the pip-and-virtualenv element virtualenv --setuptools /opt/stack/venvs/os-refresh-config -# Need setuptools>=1.0 to manage connections when -# downloading from pypi using http_proxy and https_proxy -/opt/stack/venvs/os-refresh-config/bin/pip install -U pip -/opt/stack/venvs/os-refresh-config/bin/pip install -U 'setuptools>=1.0' -# bug #1293812 : Avoid easy_install triggering on pbr. -/opt/stack/venvs/os-refresh-config/bin/pip install -U 'pbr>=0.5.21,<1.0' -/opt/stack/venvs/os-refresh-config/bin/pip install -U os-refresh-config +if [ -n "$manifest" ]; then + use-pip-manifest $manifest +else + # Need setuptools>=1.0 to manage connections when + # downloading from pypi using http_proxy and https_proxy + /opt/stack/venvs/os-refresh-config/bin/pip install -U pip + /opt/stack/venvs/os-refresh-config/bin/pip install -U 'setuptools>=1.0' + # bug #1293812 : Avoid easy_install triggering on pbr. + /opt/stack/venvs/os-refresh-config/bin/pip install -U 'pbr>=0.5.21,<1.0' + /opt/stack/venvs/os-refresh-config/bin/pip install -U os-refresh-config +fi + +# Write the manifest of what was installed +write-pip-manifest os-refresh-config ln -s /opt/stack/venvs/os-refresh-config/bin/os-refresh-config /usr/local/bin/os-refresh-config diff --git a/elements/os-svc-install/bin/os-svc-install b/elements/os-svc-install/bin/os-svc-install index b6ab2d95c..df74d1622 100755 --- a/elements/os-svc-install/bin/os-svc-install +++ b/elements/os-svc-install/bin/os-svc-install @@ -5,6 +5,8 @@ function python_install() { local svc_root=$1 local install_dir=$2 local system_site_packages=${3:-"False"} + local name=$(basename $install_dir) + local svc_manifest=$(get-pip-manifest $name) SITE_PCKGS="--no-site-packages" if [ $system_site_packages == "True" ]; then @@ -17,31 +19,39 @@ function python_install() { source $install_dir/bin/activate set -u - if [ -e $svc_root/requirements.txt ]; then - reqs=$svc_root/requirements.txt - elif [ -e $svc_root/tools/pip-requires ]; then - reqs=$svc_root/tools/pip-requires + # If given an exact deps list, use it, and upgrade to the local git service + if [ -n "$svc_manifest" ]; then + use-pip-manifest $svc_manifest else - reqs="" + if [ -e $svc_root/requirements.txt ]; then + reqs=$svc_root/requirements.txt + elif [ -e $svc_root/tools/pip-requires ]; then + reqs=$svc_root/tools/pip-requires + else + reqs="" + fi + + # bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which + # doesn't manage correctly HTTPS sockets when downloading pbr from + # https://pypi.python.org/simple/ if using http_proxy and + # https_proxy envvars + pip install -U 'setuptools>=1.0' + + # bug #1293812 : Avoid easy_install triggering on pbr. + pip install -U 'pbr>=0.5.21,<1.0' + + if [ -n "$reqs" ] ; then + pip install -r $reqs + # FIXME: pip requires doesn't include MySQL-python + pip install MySQL-python + fi fi - - # bug #1201253 : virtualenv-1.10.1 embeds setuptools-0.9.8, which - # doesn't manage correctly HTTPS sockets when downloading pbr from - # https://pypi.python.org/simple/ if using http_proxy and - # https_proxy envvars - pip install -U 'setuptools>=1.0' - - # bug #1293812 : Avoid easy_install triggering on pbr. - pip install -U 'pbr>=0.5.21,<1.0' - - if [ -n "$reqs" ] ; then - pip install -r $reqs - # FIXME: pip requires doesn't include MySQL-python - pip install MySQL-python - fi - + # Always replay this, as we cannot use the entry this would generate in the manifest pip install $svc_root + # Write the manifest of what was installed + write-pip-manifest $name + set +u deactivate set -u diff --git a/elements/os-svc-install/element-deps b/elements/os-svc-install/element-deps index 7c8922a0b..9770f9ba6 100644 --- a/elements/os-svc-install/element-deps +++ b/elements/os-svc-install/element-deps @@ -1 +1,2 @@ pip-and-virtualenv +pip-manifest diff --git a/elements/pip-manifest/README.md b/elements/pip-manifest/README.md new file mode 100644 index 000000000..351968e29 --- /dev/null +++ b/elements/pip-manifest/README.md @@ -0,0 +1,63 @@ +Utility element to enable repeatable pip installs +================================================= + +Set this element as a dependency to make the utility scripts available, +and then use them as appropriate. + +## Usage +This element makes a number of scripts available for use in image building, and +performs actions to copy specified manifests into the image for use in building. + +It also copies manifests generated during the build back to the build environment +during cleanup. + +## bin +Utility scripts for use in other elements to create and reuse pip manifests, +as detailed in the usage section. + +### get-pip-manifest +Echoes the name of the pip manifest file if one has been copied in, or just +returns. The caller passes the name associated with their element, which +should be descriptive of the element, to get the correct value. + +For example, the nova element calls `get-pip-manifest nova`. + +The name of the element is transformed to conform with bash variable naming +rules, so any charaters that are not [A-Za-z0-9] are replaced with '\_'. + +### use-pip-manifest +Uses the given manifest to perform the pip installs necessary. + +Note that any development versions listed in the manifest are not reinstalled. + +The reason for this is that development versions are expected to have been +installed from a source other than pypi or the mirror in use, and so development +versions will not be reinstallable without extra information. + +The exact details with respect to this are for the relevant consuming element to +determine. + +### write-pip-manifest +Calls pip freeze and writes the versions of all of the packages currently +installed to a manifest file. + +The format of the manifest is the standard python requirements format that is +generated by the "pip freeze" command. + +## extra-data.d + +### 75-inject-pip-manifests +Copies any pip manifest specified in DIB\_PIP\_MANIFEST\_\* environment variables +into the image chroot environment. + +## install.d + +### 01-pip-manifest +Installs the scripts in this element into the image for later use by other elements. + +## cleanup.d + +### 01-copy-pip-manifests +Runs outside of the chroot and copies created manifests back to the +build environment for re-use in repeating the image build. + diff --git a/elements/pip-manifest/bin/get-pip-manifest b/elements/pip-manifest/bin/get-pip-manifest new file mode 100755 index 000000000..bf52eb658 --- /dev/null +++ b/elements/pip-manifest/bin/get-pip-manifest @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -eux + +name=${1:?"Usage: ${0} "} +build_manifest=/etc/dib-pip-manifests/dib-pip-build-manifest-${name//[^A-Za-z0-9]/_} + +if [[ -f "${build_manifest}" ]]; then + echo "${build_manifest}" +fi diff --git a/elements/pip-manifest/bin/use-pip-manifest b/elements/pip-manifest/bin/use-pip-manifest new file mode 100755 index 000000000..fbcb6953a --- /dev/null +++ b/elements/pip-manifest/bin/use-pip-manifest @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -eux + +manifest=${1:?"Usage: ${0} "} + +# Comment out the dev versions listed, as they may not be reinstallable +sed -i "s/^\(.*[.]dev.*\)$/# \1/g" $manifest + +pip install --no-deps -r $manifest diff --git a/elements/pip-manifest/bin/write-pip-manifest b/elements/pip-manifest/bin/write-pip-manifest new file mode 100755 index 000000000..8e38ca300 --- /dev/null +++ b/elements/pip-manifest/bin/write-pip-manifest @@ -0,0 +1,22 @@ +#!/bin/bash +# +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -eux + +name=${1:?"Usage: ${0} "} + +pip freeze | tee /etc/dib-pip-manifests/dib-pip-manifest-${name//[^A-Za-z0-9]/_} diff --git a/elements/pip-manifest/cleanup.d/01-copy-pip-manifests b/elements/pip-manifest/cleanup.d/01-copy-pip-manifests new file mode 100755 index 000000000..77314005c --- /dev/null +++ b/elements/pip-manifest/cleanup.d/01-copy-pip-manifests @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -eux + +if [ -d $TMP_MOUNT_PATH/etc/dib-pip-manifests ]; then + mkdir -p ${IMAGE_NAME}-manifests + cp --no-preserve=ownership -rv $TMP_MOUNT_PATH/etc/dib-pip-manifests \ + ${IMAGE_NAME}-manifests/ +fi diff --git a/elements/pip-manifest/extra-data.d/75-inject-pip-manifests b/elements/pip-manifest/extra-data.d/75-inject-pip-manifests new file mode 100755 index 000000000..20f80bd68 --- /dev/null +++ b/elements/pip-manifest/extra-data.d/75-inject-pip-manifests @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -eux + +dib_pip_manifest_dir=$TMP_MOUNT_PATH/etc/dib-pip-manifests +sudo mkdir $dib_pip_manifest_dir + +# Find all of the pip manifests and copy them into the image for use in install.d +for manifest in ${!DIB_PIP_MANIFEST_*} +do + sudo cp ${!manifest} $dib_pip_manifest_dir/dib-pip-build-manifest-${manifest##DIB_PIP_MANIFEST_} +done diff --git a/elements/pip-manifest/install.d/01-pip-manifest b/elements/pip-manifest/install.d/01-pip-manifest new file mode 100755 index 000000000..0906858c5 --- /dev/null +++ b/elements/pip-manifest/install.d/01-pip-manifest @@ -0,0 +1,22 @@ +#!/bin/bash +# +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -eux + +install -m 0755 -o root -g root $(dirname $0)/../bin/get-pip-manifest /usr/local/bin/get-pip-manifest +install -m 0755 -o root -g root $(dirname $0)/../bin/write-pip-manifest /usr/local/bin/write-pip-manifest +install -m 0755 -o root -g root $(dirname $0)/../bin/use-pip-manifest /usr/local/bin/use-pip-manifest diff --git a/elements/swift/element-deps b/elements/swift/element-deps new file mode 100644 index 000000000..2167c86fb --- /dev/null +++ b/elements/swift/element-deps @@ -0,0 +1 @@ +os-svc-install diff --git a/elements/tempest/element-deps b/elements/tempest/element-deps new file mode 100644 index 000000000..2167c86fb --- /dev/null +++ b/elements/tempest/element-deps @@ -0,0 +1 @@ +os-svc-install