From 89f95a5351078c0d2272657d236570249da381b4 Mon Sep 17 00:00:00 2001 From: Matthew Thode Date: Mon, 8 Oct 2018 00:39:35 -0500 Subject: [PATCH] enable caching for gentoo builds To facilitate this I've created two new environment variables to set the environment and default options for package actions. eval is needed for the export as it preserves quotes. Change-Id: Ib03651ee8dacd48cd1c135afd57cd31101356056 Signed-off-by: Matthew Thode --- diskimage_builder/elements/gentoo/README.rst | 16 ++++++--- .../elements/gentoo/bin/install-packages | 10 +++--- .../elements/gentoo/cleanup.d/05-unmount | 35 +++++++++++++++++++ .../environment.d/00-gentoo-envars.bash | 10 ++++++ .../elements/gentoo/finalise.d/99-cleanup | 31 +++++----------- .../gentoo/pre-finalise.d/01-gentoo-cache | 17 +++++++++ .../02-gentoo-03-enable-overlays | 4 +-- .../02-gentoo-04-install-desired-python | 20 +++++------ .../elements/gentoo/root.d/50-gentoo-cache | 20 +++++++++++ 9 files changed, 120 insertions(+), 43 deletions(-) create mode 100755 diskimage_builder/elements/gentoo/cleanup.d/05-unmount create mode 100755 diskimage_builder/elements/gentoo/pre-finalise.d/01-gentoo-cache create mode 100755 diskimage_builder/elements/gentoo/root.d/50-gentoo-cache diff --git a/diskimage_builder/elements/gentoo/README.rst b/diskimage_builder/elements/gentoo/README.rst index 8c7dcdde7..89124a655 100644 --- a/diskimage_builder/elements/gentoo/README.rst +++ b/diskimage_builder/elements/gentoo/README.rst @@ -30,12 +30,13 @@ Notes: default/linux/amd64/17.0/no-multilib default/linux/amd64/17.0/hardened default/linux/amd64/17.0/no-multilib/hardened + default/linux/amd64/17.0/systemd -* You can set the GENTOO_PORTAGE_CLEANUP environment variable to False to - prevent cleaning up portage repositories (including overlays). This makes - the image bigger. +* You can set the `GENTOO_PORTAGE_CLEANUP` environment variable to False to + disable the clean up of portage repositories (including overlays). This + will make the image bigger if caching is also disabled. -* Gentoo supports many diferent versions of python, in order to select one +* Gentoo supports many different versions of python, in order to select one you may use the `GENTOO_PYTHON_TARGETS` environment variable to select the versions of python you want on your image. The format of this variable is a string as follows `"python2_7 python3_5"`. @@ -48,3 +49,10 @@ Notes: * You can enable overlays using the `GENTOO_OVERLAYS` variable. In it you should put a space separated list of overlays. The overlays must be in the official overlay list and must be git based. + +* `GENTOO_EMERGE_ENV` is a bash array containing default environment + variables for package install, you can override it with another bash array. + +* `GENTOO_EMERGE_DEFAULT_OPTS` can be set to control the default options + passed to emerge for all package actions, this includes operations like + depclean and preserved-rebuild. diff --git a/diskimage_builder/elements/gentoo/bin/install-packages b/diskimage_builder/elements/gentoo/bin/install-packages index b98ec16a5..7b25e3065 100755 --- a/diskimage_builder/elements/gentoo/bin/install-packages +++ b/diskimage_builder/elements/gentoo/bin/install-packages @@ -35,12 +35,12 @@ function show_options { function install_gentoo_packages { RUN_ONCE_SHM='1' if mountpoint -q /dev/shm; then - emerge $@ + emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@ elif [[ -k /dev/shm ]]; then - emerge $@ + emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@ else fix_shm - emerge $@ + emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@ unfix_shm fi } @@ -58,9 +58,9 @@ while true; do if [[ ! -f /usr/portage/profiles ]]; then emerge-webrsync -q fi - install_gentoo_packages -uDNv --with-bdeps=y --complete-graph=y --jobs=2 @world + install_gentoo_packages --update --deep --newuse --complete-graph=y @world install_gentoo_packages --verbose=n --depclean - install_gentoo_packages -v --usepkg=n @preserved-rebuild + install_gentoo_packages --usepkg=n @preserved-rebuild etc-update --automode -5 eselect news read new exit 0; diff --git a/diskimage_builder/elements/gentoo/cleanup.d/05-unmount b/diskimage_builder/elements/gentoo/cleanup.d/05-unmount new file mode 100755 index 000000000..9b720380a --- /dev/null +++ b/diskimage_builder/elements/gentoo/cleanup.d/05-unmount @@ -0,0 +1,35 @@ +#!/bin/bash + +if [[ ${DIB_DEBUG_TRACE:-0} -gt 0 ]]; then + set -x +fi +set -eu +set -o pipefail + +[[ -n "${TMP_MOUNT_PATH}" ]] + +sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage/distfiles +sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage/packages +sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage + +sudo rm -Rf "${TMP_MOUNT_PATH}"/root/.ccache/* "${TMP_MOUNT_PATH}"/usr/src/* "${TMP_MOUNT_PATH}"/var/cache/edb/dep/* "${TMP_MOUNT_PATH}"/var/cache/genkernel/* "${TMP_MOUNT_PATH}"/var/empty/* "${TMP_MOUNT_PATH}"/var/state/* "${TMP_MOUNT_PATH}"/var/tmp/* +sudo rm -Rf "${TMP_MOUNT_PATH}"/etc/*- "${TMP_MOUNT_PATH}"/etc/*.old "${TMP_MOUNT_PATH}"/etc/ssh/ssh_host_* "${TMP_MOUNT_PATH}"/root/.*history "${TMP_MOUNT_PATH}"/root/.lesshst "${TMP_MOUNT_PATH}"/root/.ssh/known_hosts "${TMP_MOUNT_PATH}"/root/.viminfo "${TMP_MOUNT_PATH}"/usr/share/genkernel "${TMP_MOUNT_PATH}"/usr/lib64/python*/site-packages/gentoolkit/test/eclean/testdistfiles.tar.gz +if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then + # remove portage files + sudo rm -Rf "${TMP_MOUNT_PATH}"/usr/portage/* "${TMP_MOUNT_PATH}"/usr/portage/.git "${TMP_MOUNT_PATH}"/var/cache/portage/distfiles +fi + +# shrink a bit +for i in $(sudo -- find "${TMP_MOUNT_PATH}"/var/log -type f); do sudo -- sh -c "echo > $i"; done +sudo find "${TMP_MOUNT_PATH}"/usr/share/man/ -mindepth 1 -maxdepth 1 -path "${TMP_MOUNT_PATH}/usr/share/man/man*" -prune -o -exec rm -rf {} \; + +# make it so we don't have to reinstall grub +if [[ -a ${TMP_MOUNT_PATH}/usr/sbin/grub2-install ]]; then + sudo mkdir -p "${TMP_MOUNT_PATH}"/tmp/grub + sudo touch "${TMP_MOUNT_PATH}"/tmp/grub/install +fi + +# systemd id +if [[ "${GENTOO_PROFILE}" == *"systemd"* ]]; then + sudo cp /dev/null "${TMP_MOUNT_PATH}"/etc/machine-id +fi diff --git a/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash b/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash index 3b55eab57..84350f7fe 100644 --- a/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash +++ b/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash @@ -5,3 +5,13 @@ export GENTOO_PORTAGE_CLEANUP=${GENTOO_PORTAGE_CLEANUP:-'True'} export GENTOO_PYTHON_TARGETS=${GENTOO_PYTHON_TARGETS:-'python2_7 python3_5'} export GENTOO_PYTHON_ACTIVE_VERSION=${GENTOO_PYTHON_ACTIVE_VERSION:-'python3.5'} export GENTOO_OVERLAYS=${GENTOO_OVERLAYS:-''} +export GENTOO_EMERGE_DEFAULT_OPTS=${GENTOO_EMERGE_DEFAULT_OPTS:-"--binpkg-respect-use --rebuilt-binaries=y --usepkg=y --with-bdeps=y --binpkg-changed-deps=y --quiet --jobs=2"} + +# set the default bash array if GENTOO_EMERGE_ENV is not defined as an array +if ! declare -p GENTOO_EMERGE_ENV 2> /dev/null | grep -q '^declare \-a'; then + export GENTOO_EMERGE_ENV=('USE="-build"' 'FEATURES="binpkg-multi-instance buildpkg parallel-fetch parallel-install"') +fi +# itterate over the array, exporting each 'line' +for (( i=0; i<${#GENTOO_EMERGE_ENV[@]}; i++ )); do + eval export "${GENTOO_EMERGE_ENV[i]}" +done diff --git a/diskimage_builder/elements/gentoo/finalise.d/99-cleanup b/diskimage_builder/elements/gentoo/finalise.d/99-cleanup index 32a89bf4c..1ecf4f063 100755 --- a/diskimage_builder/elements/gentoo/finalise.d/99-cleanup +++ b/diskimage_builder/elements/gentoo/finalise.d/99-cleanup @@ -14,14 +14,17 @@ else fix_shm fi +# sync if needed +[[ ! -e /usr/portage/profiles ]] && emerge-webrsync + # make world consistant -USE="-build" emerge -q --complete-graph=y --jobs=2 --update --newuse --deep --with-bdeps=y @world +emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --complete-graph=y --update --newuse --deep @world # rebuild packages that might need it -USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild +emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild # remove unneeded packages -USE="-build" emerge --verbose=n --depclean +emerge --verbose=n --depclean # rebuild packages that might have somehow depended on the unneeded packages -USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild +emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild # update config files etc-update --automode -5 @@ -31,8 +34,6 @@ emerge --verbose=n --depclean emaint all -f eselect news read all -rm -Rf /root/.ccache/* /usr/src/* /var/cache/edb/dep/* /var/cache/genkernel/* /var/empty/* /var/run/* /var/state/* /var/tmp/* -rm -Rf /etc/*- /etc/*.old /etc/ssh/ssh_host_* /root/.*history /root/.lesshst /root/.ssh/known_hosts /root/.viminfo /usr/share/genkernel /usr/lib64/python*/site-packages/gentoolkit/test/eclean/testdistfiles.tar.gz if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then # remove the overlays if [[ ${GENTOO_OVERLAYS} != '' ]]; then @@ -40,23 +41,9 @@ if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then layman -d "${OVERLAY}" done fi - # remove portage files - rm -Rf /usr/portage/* /usr/portage/.git /var/cache/portage/distfiles -fi - -# shrink a bit -for i in $(find /var/log -type f); do echo > $i; done -find /usr/share/man/ -mindepth 1 -maxdepth 1 -path "/usr/share/man/man*" -prune -o -exec rm -rf {} \; - -# make it so we don't have to reinstall grub -if [[ -a /usr/sbin/grub2-install ]]; then - mkdir -p /tmp/grub - touch /tmp/grub/install fi unfix_shm -# systemd id -if [[ "${GENTOO_PROFILE}" == *"systemd"* ]]; then - cp /dev/null /etc/machine-id -fi +# remove within the chroot since /var/run is a symlink +rm -Rf /var/run/* /run/* diff --git a/diskimage_builder/elements/gentoo/pre-finalise.d/01-gentoo-cache b/diskimage_builder/elements/gentoo/pre-finalise.d/01-gentoo-cache new file mode 100755 index 000000000..32887405c --- /dev/null +++ b/diskimage_builder/elements/gentoo/pre-finalise.d/01-gentoo-cache @@ -0,0 +1,17 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +GENTOO_PORTAGE_DISTFILES_DIR="${DIB_IMAGE_CACHE}/gentoo/distfiles" +GENTOO_PORTAGE_PACKAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/packages/${GENTOO_PROFILE}" +GENTOO_PORTAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/portage" + +sudo mount --bind "${GENTOO_PORTAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage +mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/distfiles +sudo mount --bind "${GENTOO_PORTAGE_DISTFILES_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/distfiles +mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/packages +sudo mount --bind "${GENTOO_PORTAGE_PACKAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/packages diff --git a/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-03-enable-overlays b/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-03-enable-overlays index e27757221..537083ebc 100755 --- a/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-03-enable-overlays +++ b/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-03-enable-overlays @@ -26,9 +26,9 @@ if [[ ${GENTOO_OVERLAYS} != '' ]]; then # depends on that version of openssl. echo '=dev-python/cryptography-2.1.4-r2 ~amd64' >> /etc/portage/package.keywords/layman echo '=dev-libs/openssl-1.0.2o-r6 ~amd64' >> /etc/portage/package.keywords/layman - emerge -q --oneshot --jobs=2 openssl openssh + emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot openssl openssh # install layman - USE="-build" emerge --deep -q --jobs=2 --ignore-built-slot-operator-deps=y layman + emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --deep --ignore-built-slot-operator-deps=y layman # sync the initial overlay list layman -S # enable the various overlays, ignore failures (overlay my already be enabled) diff --git a/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-04-install-desired-python b/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-04-install-desired-python index e8b84f037..a689600c0 100755 --- a/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-04-install-desired-python +++ b/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-04-install-desired-python @@ -26,11 +26,11 @@ if [[ "${GENTOO_PYTHON_TARGETS}" == *"python3_6"* ]]; then echo '~dev-python/setuptools-36.0.1 ~amd64' >> /etc/portage/package.keywords/python echo '~dev-python/pyxattr-0.6.0 ~amd64' >> /etc/portage/package.keywords/python echo '~sys-apps/kmod-24 ~amd64' >> /etc/portage/package.keywords/python - [[ ! -e /usr/lib64/libpython3.6m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.6 + [[ ! -e /usr/lib64/libpython3.6m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.6 fi -[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_5"* ]] && [[ ! -e /usr/lib64/libpython3.5m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.5 -[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_4"* ]] && [[ ! -e /usr/lib64/libpython3.4m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.4 -[[ "${GENTOO_PYTHON_TARGETS}" == *"python2_7"* ]] && [[ ! -e /usr/lib64/libpython2.7.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:2.7 +[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_5"* ]] && [[ ! -e /usr/lib64/libpython3.5m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.5 +[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_4"* ]] && [[ ! -e /usr/lib64/libpython3.4m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.4 +[[ "${GENTOO_PYTHON_TARGETS}" == *"python2_7"* ]] && [[ ! -e /usr/lib64/libpython2.7.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:2.7 # disable python in git if we have to, it only supports python-2.7 if [[ "${GENTOO_PYTHON_TARGETS}" != *"python2_7"* ]]; then echo 'dev-vcs/git -python' >> /etc/portage/package.use/git @@ -39,9 +39,9 @@ fi # this is done one by one with nodeps because it can sometimes have dependency loops PYTHON_PACKAGES="dev-python/six dev-python/packaging dev-python/pyparsing dev-python/appdirs dev-python/setuptools dev-python/certifi" for PACKAGE in ${PYTHON_PACKAGES}; do - USE="-build" emerge -q --oneshot --jobs=2 --with-bdeps=y --update --newuse --deep --nodeps "${PACKAGE}" + emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --update --newuse --deep --nodeps "${PACKAGE}" done -USE="-build" emerge -q --oneshot --jobs=2 --with-bdeps=y --update --newuse --deep --complete-graph=y sys-apps/portage dev-python/pyxattr +emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --update --newuse --deep --complete-graph=y sys-apps/portage dev-python/pyxattr # set the active python version eselect python set ${GENTOO_PYTHON_ACTIVE_VERSION} # allow these uninstalls to fail as they may not be installed to begin with @@ -53,12 +53,12 @@ set +e set -e # make world consistant -USE="-build" emerge -q --complete-graph=y --jobs=2 --update --newuse --deep --with-bdeps=y @world +emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --complete-graph=y --update --newuse --deep @world # rebuild packages that might need it -USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild +emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild # remove unneeded packages -USE="-build" emerge --verbose=n --depclean +emerge --verbose=n --depclean # rebuild packages that might have somehow depended on the unneeded packages -USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild +emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild unfix_shm diff --git a/diskimage_builder/elements/gentoo/root.d/50-gentoo-cache b/diskimage_builder/elements/gentoo/root.d/50-gentoo-cache new file mode 100755 index 000000000..84143016e --- /dev/null +++ b/diskimage_builder/elements/gentoo/root.d/50-gentoo-cache @@ -0,0 +1,20 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +[ -n "${TARGET_ROOT}" ] + +GENTOO_PORTAGE_DISTFILES_DIR="${DIB_IMAGE_CACHE}/gentoo/distfiles" +GENTOO_PORTAGE_PACKAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/packages/${GENTOO_PROFILE}" +GENTOO_PORTAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/portage" +mkdir -p "${GENTOO_PORTAGE_DIR}" "${GENTOO_PORTAGE_PACKAGE_DIR}" "${GENTOO_PORTAGE_DISTFILES_DIR}" + +sudo mount --bind "${GENTOO_PORTAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage +mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/distfiles +sudo mount --bind "${GENTOO_PORTAGE_DISTFILES_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/distfiles +mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/packages +sudo mount --bind "${GENTOO_PORTAGE_PACKAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/packages