openstack
/
diskimage-builder
Code Issues Proposed changes

Merge "enable caching for gentoo builds"

This commit is contained in:
Zuul 2018-10-18 05:51:03 +00:00 committed by Gerrit Code Review
parent 2d327c2506 89f95a5351
commit 763493e1f1
9 changed files with 120 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
diskimage_builder/elements/gentoo/README.rst
View File

@ -30,12 +30,13 @@ Notes:
default/linux/amd64/17.0/no-multilib
default/linux/amd64/17.0/hardened
default/linux/amd64/17.0/no-multilib/hardened
default/linux/amd64/17.0/systemd
* You can set the GENTOO_PORTAGE_CLEANUP environment variable to False to
prevent cleaning up portage repositories (including overlays). This makes
the image bigger.
* You can set the `GENTOO_PORTAGE_CLEANUP` environment variable to False to
disable the clean up of portage repositories (including overlays). This
will make the image bigger if caching is also disabled.
* Gentoo supports many diferent versions of python, in order to select one
* Gentoo supports many different versions of python, in order to select one
you may use the `GENTOO_PYTHON_TARGETS` environment variable to select
the versions of python you want on your image. The format of this variable
is a string as follows `"python2_7 python3_5"`.
@ -48,3 +49,10 @@ Notes:
* You can enable overlays using the `GENTOO_OVERLAYS` variable. In it you
should put a space separated list of overlays. The overlays must be in the
official overlay list and must be git based.
* `GENTOO_EMERGE_ENV` is a bash array containing default environment
variables for package install, you can override it with another bash array.
* `GENTOO_EMERGE_DEFAULT_OPTS` can be set to control the default options
passed to emerge for all package actions, this includes operations like
depclean and preserved-rebuild.

10
diskimage_builder/elements/gentoo/bin/install-packages
View File

@ -35,12 +35,12 @@ function show_options {
function install_gentoo_packages {
RUN_ONCE_SHM='1'
if mountpoint -q /dev/shm; then
emerge $@
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@
elif [[ -k /dev/shm ]]; then
emerge $@
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@
else
fix_shm
emerge $@
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@
unfix_shm
fi
}
@ -58,9 +58,9 @@ while true; do
if [[ ! -f /usr/portage/profiles ]]; then
emerge-webrsync -q
fi
install_gentoo_packages -uDNv --with-bdeps=y --complete-graph=y --jobs=2 @world
install_gentoo_packages --update --deep --newuse --complete-graph=y @world
install_gentoo_packages --verbose=n --depclean
install_gentoo_packages -v --usepkg=n @preserved-rebuild
install_gentoo_packages --usepkg=n @preserved-rebuild
etc-update --automode -5
eselect news read new
exit 0;

35
diskimage_builder/elements/gentoo/cleanup.d/05-unmount Executable file
View File

@ -0,0 +1,35 @@
#!/bin/bash
if [[ ${DIB_DEBUG_TRACE:-0} -gt 0 ]]; then
set -x
fi
set -eu
set -o pipefail
[[ -n "${TMP_MOUNT_PATH}" ]]
sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage/distfiles
sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage/packages
sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage
sudo rm -Rf "${TMP_MOUNT_PATH}"/root/.ccache/* "${TMP_MOUNT_PATH}"/usr/src/* "${TMP_MOUNT_PATH}"/var/cache/edb/dep/* "${TMP_MOUNT_PATH}"/var/cache/genkernel/* "${TMP_MOUNT_PATH}"/var/empty/* "${TMP_MOUNT_PATH}"/var/state/* "${TMP_MOUNT_PATH}"/var/tmp/*
sudo rm -Rf "${TMP_MOUNT_PATH}"/etc/*- "${TMP_MOUNT_PATH}"/etc/*.old "${TMP_MOUNT_PATH}"/etc/ssh/ssh_host_* "${TMP_MOUNT_PATH}"/root/.*history "${TMP_MOUNT_PATH}"/root/.lesshst "${TMP_MOUNT_PATH}"/root/.ssh/known_hosts "${TMP_MOUNT_PATH}"/root/.viminfo "${TMP_MOUNT_PATH}"/usr/share/genkernel "${TMP_MOUNT_PATH}"/usr/lib64/python*/site-packages/gentoolkit/test/eclean/testdistfiles.tar.gz
if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then
# remove portage files
sudo rm -Rf "${TMP_MOUNT_PATH}"/usr/portage/* "${TMP_MOUNT_PATH}"/usr/portage/.git "${TMP_MOUNT_PATH}"/var/cache/portage/distfiles
fi
# shrink a bit
for i in $(sudo -- find "${TMP_MOUNT_PATH}"/var/log -type f); do sudo -- sh -c "echo > $i"; done
sudo find "${TMP_MOUNT_PATH}"/usr/share/man/ -mindepth 1 -maxdepth 1 -path "${TMP_MOUNT_PATH}/usr/share/man/man*" -prune -o -exec rm -rf {} \;
# make it so we don't have to reinstall grub
if [[ -a ${TMP_MOUNT_PATH}/usr/sbin/grub2-install ]]; then
sudo mkdir -p "${TMP_MOUNT_PATH}"/tmp/grub
sudo touch "${TMP_MOUNT_PATH}"/tmp/grub/install
fi
# systemd id
if [[ "${GENTOO_PROFILE}" == *"systemd"* ]]; then
sudo cp /dev/null "${TMP_MOUNT_PATH}"/etc/machine-id
fi

10
diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash
View File

@ -5,3 +5,13 @@ export GENTOO_PORTAGE_CLEANUP=${GENTOO_PORTAGE_CLEANUP:-'True'}
export GENTOO_PYTHON_TARGETS=${GENTOO_PYTHON_TARGETS:-'python2_7 python3_5'}
export GENTOO_PYTHON_ACTIVE_VERSION=${GENTOO_PYTHON_ACTIVE_VERSION:-'python3.5'}
export GENTOO_OVERLAYS=${GENTOO_OVERLAYS:-''}
export GENTOO_EMERGE_DEFAULT_OPTS=${GENTOO_EMERGE_DEFAULT_OPTS:-"--binpkg-respect-use --rebuilt-binaries=y --usepkg=y --with-bdeps=y --binpkg-changed-deps=y --quiet --jobs=2"}
# set the default bash array if GENTOO_EMERGE_ENV is not defined as an array
if ! declare -p GENTOO_EMERGE_ENV 2> /dev/null | grep -q '^declare \-a'; then
export GENTOO_EMERGE_ENV=('USE="-build"' 'FEATURES="binpkg-multi-instance buildpkg parallel-fetch parallel-install"')
fi
# itterate over the array, exporting each 'line'
for (( i=0; i<${#GENTOO_EMERGE_ENV[@]}; i++ )); do
eval export "${GENTOO_EMERGE_ENV[i]}"
done

31
diskimage_builder/elements/gentoo/finalise.d/99-cleanup
View File

@ -14,14 +14,17 @@ else
fix_shm
fi
# sync if needed
[[ ! -e /usr/portage/profiles ]] && emerge-webrsync
# make world consistant
USE="-build" emerge -q --complete-graph=y --jobs=2 --update --newuse --deep --with-bdeps=y @world
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --complete-graph=y --update --newuse --deep @world
# rebuild packages that might need it
USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
# remove unneeded packages
USE="-build" emerge --verbose=n --depclean
emerge --verbose=n --depclean
# rebuild packages that might have somehow depended on the unneeded packages
USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
# update config files
etc-update --automode -5
@ -31,8 +34,6 @@ emerge --verbose=n --depclean
emaint all -f
eselect news read all
rm -Rf /root/.ccache/* /usr/src/* /var/cache/edb/dep/* /var/cache/genkernel/* /var/empty/* /var/run/* /var/state/* /var/tmp/*
rm -Rf /etc/*- /etc/*.old /etc/ssh/ssh_host_* /root/.*history /root/.lesshst /root/.ssh/known_hosts /root/.viminfo /usr/share/genkernel /usr/lib64/python*/site-packages/gentoolkit/test/eclean/testdistfiles.tar.gz
if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then
# remove the overlays
if [[ ${GENTOO_OVERLAYS} != '' ]]; then
@ -40,23 +41,9 @@ if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then
layman -d "${OVERLAY}"
done
fi
# remove portage files
rm -Rf /usr/portage/* /usr/portage/.git /var/cache/portage/distfiles
fi
# shrink a bit
for i in $(find /var/log -type f); do echo > $i; done
find /usr/share/man/ -mindepth 1 -maxdepth 1 -path "/usr/share/man/man*" -prune -o -exec rm -rf {} \;
# make it so we don't have to reinstall grub
if [[ -a /usr/sbin/grub2-install ]]; then
mkdir -p /tmp/grub
touch /tmp/grub/install
fi
unfix_shm
# systemd id
if [[ "${GENTOO_PROFILE}" == *"systemd"* ]]; then
cp /dev/null /etc/machine-id
fi
# remove within the chroot since /var/run is a symlink
rm -Rf /var/run/* /run/*

17
diskimage_builder/elements/gentoo/pre-finalise.d/01-gentoo-cache Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
GENTOO_PORTAGE_DISTFILES_DIR="${DIB_IMAGE_CACHE}/gentoo/distfiles"
GENTOO_PORTAGE_PACKAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/packages/${GENTOO_PROFILE}"
GENTOO_PORTAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/portage"
sudo mount --bind "${GENTOO_PORTAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage
mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/distfiles
sudo mount --bind "${GENTOO_PORTAGE_DISTFILES_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/distfiles
mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/packages
sudo mount --bind "${GENTOO_PORTAGE_PACKAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/packages

4
diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-03-enable-overlays
View File

@ -26,9 +26,9 @@ if [[ ${GENTOO_OVERLAYS} != '' ]]; then
# depends on that version of openssl.
echo '=dev-python/cryptography-2.1.4-r2 ~amd64' >> /etc/portage/package.keywords/layman
echo '=dev-libs/openssl-1.0.2o-r6 ~amd64' >> /etc/portage/package.keywords/layman
emerge -q --oneshot --jobs=2 openssl openssh
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot openssl openssh
# install layman
USE="-build" emerge --deep -q --jobs=2 --ignore-built-slot-operator-deps=y layman
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --deep --ignore-built-slot-operator-deps=y layman
# sync the initial overlay list
layman -S
# enable the various overlays, ignore failures (overlay my already be enabled)

20
diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-04-install-desired-python
View File

@ -26,11 +26,11 @@ if [[ "${GENTOO_PYTHON_TARGETS}" == *"python3_6"* ]]; then
echo '~dev-python/setuptools-36.0.1 ~amd64' >> /etc/portage/package.keywords/python
echo '~dev-python/pyxattr-0.6.0 ~amd64' >> /etc/portage/package.keywords/python
echo '~sys-apps/kmod-24 ~amd64' >> /etc/portage/package.keywords/python
[[ ! -e /usr/lib64/libpython3.6m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.6
[[ ! -e /usr/lib64/libpython3.6m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.6
fi
[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_5"* ]] && [[ ! -e /usr/lib64/libpython3.5m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.5
[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_4"* ]] && [[ ! -e /usr/lib64/libpython3.4m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.4
[[ "${GENTOO_PYTHON_TARGETS}" == *"python2_7"* ]] && [[ ! -e /usr/lib64/libpython2.7.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:2.7
[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_5"* ]] && [[ ! -e /usr/lib64/libpython3.5m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.5
[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_4"* ]] && [[ ! -e /usr/lib64/libpython3.4m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.4
[[ "${GENTOO_PYTHON_TARGETS}" == *"python2_7"* ]] && [[ ! -e /usr/lib64/libpython2.7.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:2.7
# disable python in git if we have to, it only supports python-2.7
if [[ "${GENTOO_PYTHON_TARGETS}" != *"python2_7"* ]]; then
echo 'dev-vcs/git -python' >> /etc/portage/package.use/git
@ -39,9 +39,9 @@ fi
# this is done one by one with nodeps because it can sometimes have dependency loops
PYTHON_PACKAGES="dev-python/six dev-python/packaging dev-python/pyparsing dev-python/appdirs dev-python/setuptools dev-python/certifi"
for PACKAGE in ${PYTHON_PACKAGES}; do
USE="-build" emerge -q --oneshot --jobs=2 --with-bdeps=y --update --newuse --deep --nodeps "${PACKAGE}"
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --update --newuse --deep --nodeps "${PACKAGE}"
done
USE="-build" emerge -q --oneshot --jobs=2 --with-bdeps=y --update --newuse --deep --complete-graph=y sys-apps/portage dev-python/pyxattr
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --update --newuse --deep --complete-graph=y sys-apps/portage dev-python/pyxattr
# set the active python version
eselect python set ${GENTOO_PYTHON_ACTIVE_VERSION}
# allow these uninstalls to fail as they may not be installed to begin with
@ -53,12 +53,12 @@ set +e
set -e
# make world consistant
USE="-build" emerge -q --complete-graph=y --jobs=2 --update --newuse --deep --with-bdeps=y @world
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --complete-graph=y --update --newuse --deep @world
# rebuild packages that might need it
USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
# remove unneeded packages
USE="-build" emerge --verbose=n --depclean
emerge --verbose=n --depclean
# rebuild packages that might have somehow depended on the unneeded packages
USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
unfix_shm

20
diskimage_builder/elements/gentoo/root.d/50-gentoo-cache Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
[ -n "${TARGET_ROOT}" ]
GENTOO_PORTAGE_DISTFILES_DIR="${DIB_IMAGE_CACHE}/gentoo/distfiles"
GENTOO_PORTAGE_PACKAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/packages/${GENTOO_PROFILE}"
GENTOO_PORTAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/portage"
mkdir -p "${GENTOO_PORTAGE_DIR}" "${GENTOO_PORTAGE_PACKAGE_DIR}" "${GENTOO_PORTAGE_DISTFILES_DIR}"
sudo mount --bind "${GENTOO_PORTAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage
mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/distfiles
sudo mount --bind "${GENTOO_PORTAGE_DISTFILES_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/distfiles
mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/packages
sudo mount --bind "${GENTOO_PORTAGE_PACKAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/packages