enable caching for gentoo builds

To facilitate this I've created two new environment variables to set the environment and default options for package actions. eval is needed for the export as it preserves quotes. Change-Id: Ib03651ee8dacd48cd1c135afd57cd31101356056 Signed-off-by: Matthew Thode <mthode@mthode.org>
2018-10-08 00:39:35 -05:00 · 2018-10-08 00:39:35 -05:00 · 89f95a5351
parent b86af3dc6a
commit 89f95a5351
9 changed files with 120 additions and 43 deletions
--- a/diskimage_builder/elements/gentoo/README.rst
+++ b/diskimage_builder/elements/gentoo/README.rst
@ -30,12 +30,13 @@ Notes:
    default/linux/amd64/17.0/no-multilib
    default/linux/amd64/17.0/hardened
    default/linux/amd64/17.0/no-multilib/hardened
+    default/linux/amd64/17.0/systemd

-* You can set the GENTOO_PORTAGE_CLEANUP environment variable to False to
-  prevent cleaning up portage repositories (including overlays).  This makes
-  the image bigger.
+* You can set the `GENTOO_PORTAGE_CLEANUP` environment variable to False to
+  disable the clean up of portage repositories (including overlays).  This
+  will make the image bigger if caching is also disabled.

-* Gentoo supports many diferent versions of python, in order to select one
+* Gentoo supports many different versions of python, in order to select one
  you may use the `GENTOO_PYTHON_TARGETS` environment variable to select
  the versions of python you want on your image.  The format of this variable
  is a string as follows `"python2_7 python3_5"`.
@ -48,3 +49,10 @@ Notes:
 * You can enable overlays using the `GENTOO_OVERLAYS` variable.  In it you
  should put a space separated list of overlays.  The overlays must be in the
  official overlay list and must be git based.
+
+* `GENTOO_EMERGE_ENV` is a bash array containing default environment
+  variables for package install, you can override it with another bash array.
+
+* `GENTOO_EMERGE_DEFAULT_OPTS` can be set to control the default options
+  passed to emerge for all package actions, this includes operations like
+  depclean and preserved-rebuild.
--- a/diskimage_builder/elements/gentoo/bin/install-packages
+++ b/diskimage_builder/elements/gentoo/bin/install-packages
@ -35,12 +35,12 @@ function show_options {
 function install_gentoo_packages {
    RUN_ONCE_SHM='1'
    if mountpoint -q /dev/shm; then
-        emerge $@
+        emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@
    elif [[ -k /dev/shm ]]; then
-        emerge $@
+        emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@
    else
        fix_shm
-        emerge $@
+        emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@
        unfix_shm
    fi
 }
@ -58,9 +58,9 @@ while true; do
            if [[ ! -f /usr/portage/profiles ]]; then
                emerge-webrsync -q
            fi
-            install_gentoo_packages -uDNv --with-bdeps=y --complete-graph=y --jobs=2 @world
+            install_gentoo_packages --update --deep --newuse --complete-graph=y @world
            install_gentoo_packages --verbose=n --depclean
-            install_gentoo_packages -v --usepkg=n @preserved-rebuild
+            install_gentoo_packages --usepkg=n @preserved-rebuild
            etc-update --automode -5
            eselect news read new
            exit 0;
--- a/diskimage_builder/elements/gentoo/cleanup.d/05-unmount
+++ b/diskimage_builder/elements/gentoo/cleanup.d/05-unmount
@ -0,0 +1,35 @@
+#!/bin/bash
+
+if [[ ${DIB_DEBUG_TRACE:-0} -gt 0 ]]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+[[ -n "${TMP_MOUNT_PATH}" ]]
+
+sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage/distfiles
+sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage/packages
+sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage
+
+sudo rm -Rf "${TMP_MOUNT_PATH}"/root/.ccache/* "${TMP_MOUNT_PATH}"/usr/src/* "${TMP_MOUNT_PATH}"/var/cache/edb/dep/* "${TMP_MOUNT_PATH}"/var/cache/genkernel/* "${TMP_MOUNT_PATH}"/var/empty/* "${TMP_MOUNT_PATH}"/var/state/* "${TMP_MOUNT_PATH}"/var/tmp/*
+sudo rm -Rf "${TMP_MOUNT_PATH}"/etc/*- "${TMP_MOUNT_PATH}"/etc/*.old "${TMP_MOUNT_PATH}"/etc/ssh/ssh_host_* "${TMP_MOUNT_PATH}"/root/.*history "${TMP_MOUNT_PATH}"/root/.lesshst "${TMP_MOUNT_PATH}"/root/.ssh/known_hosts "${TMP_MOUNT_PATH}"/root/.viminfo "${TMP_MOUNT_PATH}"/usr/share/genkernel "${TMP_MOUNT_PATH}"/usr/lib64/python*/site-packages/gentoolkit/test/eclean/testdistfiles.tar.gz
+if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then
+    # remove portage files
+    sudo rm -Rf "${TMP_MOUNT_PATH}"/usr/portage/* "${TMP_MOUNT_PATH}"/usr/portage/.git "${TMP_MOUNT_PATH}"/var/cache/portage/distfiles
+fi
+
+# shrink a bit
+for i in $(sudo -- find "${TMP_MOUNT_PATH}"/var/log -type f); do sudo -- sh -c "echo > $i"; done
+sudo find "${TMP_MOUNT_PATH}"/usr/share/man/ -mindepth 1  -maxdepth 1 -path "${TMP_MOUNT_PATH}/usr/share/man/man*" -prune -o -exec rm -rf {} \;
+
+# make it so we don't have to reinstall grub
+if [[ -a ${TMP_MOUNT_PATH}/usr/sbin/grub2-install ]]; then
+    sudo mkdir -p "${TMP_MOUNT_PATH}"/tmp/grub
+    sudo touch "${TMP_MOUNT_PATH}"/tmp/grub/install
+fi
+
+# systemd id
+if [[ "${GENTOO_PROFILE}" == *"systemd"* ]]; then
+    sudo cp /dev/null "${TMP_MOUNT_PATH}"/etc/machine-id
+fi
--- a/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash
+++ b/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash
@ -5,3 +5,13 @@ export GENTOO_PORTAGE_CLEANUP=${GENTOO_PORTAGE_CLEANUP:-'True'}
 export GENTOO_PYTHON_TARGETS=${GENTOO_PYTHON_TARGETS:-'python2_7 python3_5'}
 export GENTOO_PYTHON_ACTIVE_VERSION=${GENTOO_PYTHON_ACTIVE_VERSION:-'python3.5'}
 export GENTOO_OVERLAYS=${GENTOO_OVERLAYS:-''}
+export GENTOO_EMERGE_DEFAULT_OPTS=${GENTOO_EMERGE_DEFAULT_OPTS:-"--binpkg-respect-use --rebuilt-binaries=y --usepkg=y --with-bdeps=y --binpkg-changed-deps=y --quiet --jobs=2"}
+
+# set the default bash array if GENTOO_EMERGE_ENV is not defined as an array
+if ! declare -p GENTOO_EMERGE_ENV  2> /dev/null | grep -q '^declare \-a'; then
+    export GENTOO_EMERGE_ENV=('USE="-build"' 'FEATURES="binpkg-multi-instance buildpkg parallel-fetch parallel-install"')
+fi
+# itterate over the array, exporting each 'line'
+for (( i=0; i<${#GENTOO_EMERGE_ENV[@]}; i++ )); do
+    eval export "${GENTOO_EMERGE_ENV[i]}"
+done
--- a/diskimage_builder/elements/gentoo/finalise.d/99-cleanup
+++ b/diskimage_builder/elements/gentoo/finalise.d/99-cleanup
@ -14,14 +14,17 @@ else
    fix_shm
 fi

+# sync if needed
+[[ ! -e /usr/portage/profiles ]] && emerge-webrsync
+
 # make world consistant
-USE="-build" emerge -q --complete-graph=y --jobs=2 --update --newuse --deep --with-bdeps=y @world
+emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --complete-graph=y --update --newuse --deep @world
 # rebuild packages that might need it
-USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
+emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
 # remove unneeded packages
-USE="-build" emerge --verbose=n --depclean
+emerge --verbose=n --depclean
 # rebuild packages that might have somehow depended on the unneeded packages
-USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
+emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild

 # update config files
 etc-update --automode -5
@ -31,8 +34,6 @@ emerge --verbose=n --depclean
 emaint all -f
 eselect news read all

-rm -Rf /root/.ccache/* /usr/src/* /var/cache/edb/dep/* /var/cache/genkernel/* /var/empty/* /var/run/* /var/state/* /var/tmp/*
-rm -Rf /etc/*- /etc/*.old /etc/ssh/ssh_host_* /root/.*history /root/.lesshst /root/.ssh/known_hosts /root/.viminfo /usr/share/genkernel /usr/lib64/python*/site-packages/gentoolkit/test/eclean/testdistfiles.tar.gz
 if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then
    # remove the overlays
    if [[ ${GENTOO_OVERLAYS} != '' ]]; then
@ -40,23 +41,9 @@ if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then
            layman -d "${OVERLAY}"
        done
    fi
-    # remove portage files
-    rm -Rf /usr/portage/* /usr/portage/.git /var/cache/portage/distfiles
-fi
-
-# shrink a bit
-for i in $(find /var/log -type f); do echo > $i; done
-find /usr/share/man/ -mindepth 1  -maxdepth 1 -path "/usr/share/man/man*" -prune -o -exec rm -rf {} \;
-
-# make it so we don't have to reinstall grub
-if [[ -a /usr/sbin/grub2-install ]]; then
-    mkdir -p /tmp/grub
-    touch /tmp/grub/install
 fi

 unfix_shm

-# systemd id
-if [[ "${GENTOO_PROFILE}" == *"systemd"* ]]; then
-    cp /dev/null /etc/machine-id
-fi
+# remove within the chroot since /var/run is a symlink
+rm -Rf /var/run/* /run/*
--- a/diskimage_builder/elements/gentoo/pre-finalise.d/01-gentoo-cache
+++ b/diskimage_builder/elements/gentoo/pre-finalise.d/01-gentoo-cache
@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+GENTOO_PORTAGE_DISTFILES_DIR="${DIB_IMAGE_CACHE}/gentoo/distfiles"
+GENTOO_PORTAGE_PACKAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/packages/${GENTOO_PROFILE}"
+GENTOO_PORTAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/portage"
+
+sudo mount --bind "${GENTOO_PORTAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage
+mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/distfiles
+sudo mount --bind "${GENTOO_PORTAGE_DISTFILES_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/distfiles
+mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/packages
+sudo mount --bind "${GENTOO_PORTAGE_PACKAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/packages
--- a/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-03-enable-overlays
+++ b/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-03-enable-overlays
@ -26,9 +26,9 @@ if [[ ${GENTOO_OVERLAYS} != '' ]]; then
    # depends on that version of openssl.
    echo '=dev-python/cryptography-2.1.4-r2 ~amd64' >> /etc/portage/package.keywords/layman
    echo '=dev-libs/openssl-1.0.2o-r6 ~amd64' >> /etc/portage/package.keywords/layman
-    emerge -q --oneshot --jobs=2 openssl openssh
+    emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot openssl openssh
    # install layman
-    USE="-build" emerge --deep -q --jobs=2 --ignore-built-slot-operator-deps=y layman
+    emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --deep --ignore-built-slot-operator-deps=y layman
    # sync the initial overlay list
    layman -S
    # enable the various overlays, ignore failures (overlay my already be enabled)
--- a/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-04-install-desired-python
+++ b/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-04-install-desired-python
@ -26,11 +26,11 @@ if [[ "${GENTOO_PYTHON_TARGETS}" == *"python3_6"* ]]; then
    echo '~dev-python/setuptools-36.0.1 ~amd64' >> /etc/portage/package.keywords/python
    echo '~dev-python/pyxattr-0.6.0 ~amd64' >> /etc/portage/package.keywords/python
    echo '~sys-apps/kmod-24 ~amd64' >> /etc/portage/package.keywords/python
-    [[ ! -e /usr/lib64/libpython3.6m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.6
+    [[ ! -e /usr/lib64/libpython3.6m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.6
 fi
-[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_5"* ]] && [[ ! -e /usr/lib64/libpython3.5m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.5
-[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_4"* ]] && [[ ! -e /usr/lib64/libpython3.4m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.4
-[[ "${GENTOO_PYTHON_TARGETS}" == *"python2_7"* ]] && [[ ! -e /usr/lib64/libpython2.7.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:2.7
+[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_5"* ]] && [[ ! -e /usr/lib64/libpython3.5m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.5
+[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_4"* ]] && [[ ! -e /usr/lib64/libpython3.4m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.4
+[[ "${GENTOO_PYTHON_TARGETS}" == *"python2_7"* ]] && [[ ! -e /usr/lib64/libpython2.7.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:2.7
 # disable python in git if we have to, it only supports python-2.7
 if [[ "${GENTOO_PYTHON_TARGETS}" != *"python2_7"* ]]; then
    echo 'dev-vcs/git -python' >> /etc/portage/package.use/git
@ -39,9 +39,9 @@ fi
 # this is done one by one with nodeps because it can sometimes have dependency loops
 PYTHON_PACKAGES="dev-python/six dev-python/packaging dev-python/pyparsing dev-python/appdirs dev-python/setuptools dev-python/certifi"
 for PACKAGE in ${PYTHON_PACKAGES}; do
-    USE="-build" emerge -q --oneshot --jobs=2 --with-bdeps=y --update --newuse --deep --nodeps "${PACKAGE}"
+    emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --update --newuse --deep --nodeps "${PACKAGE}"
 done
-USE="-build" emerge -q --oneshot --jobs=2 --with-bdeps=y --update --newuse --deep --complete-graph=y sys-apps/portage dev-python/pyxattr
+emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --update --newuse --deep --complete-graph=y sys-apps/portage dev-python/pyxattr
 # set the active python version
 eselect python set ${GENTOO_PYTHON_ACTIVE_VERSION}
 # allow these uninstalls to fail as they may not be installed to begin with
@ -53,12 +53,12 @@ set +e
 set -e

 # make world consistant
-USE="-build" emerge -q --complete-graph=y --jobs=2 --update --newuse --deep --with-bdeps=y @world
+emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --complete-graph=y --update --newuse --deep @world
 # rebuild packages that might need it
-USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
+emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
 # remove unneeded packages
-USE="-build" emerge --verbose=n --depclean
+emerge --verbose=n --depclean
 # rebuild packages that might have somehow depended on the unneeded packages
-USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
+emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild

 unfix_shm
--- a/diskimage_builder/elements/gentoo/root.d/50-gentoo-cache
+++ b/diskimage_builder/elements/gentoo/root.d/50-gentoo-cache
@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+[ -n "${TARGET_ROOT}" ]
+
+GENTOO_PORTAGE_DISTFILES_DIR="${DIB_IMAGE_CACHE}/gentoo/distfiles"
+GENTOO_PORTAGE_PACKAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/packages/${GENTOO_PROFILE}"
+GENTOO_PORTAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/portage"
+mkdir -p "${GENTOO_PORTAGE_DIR}" "${GENTOO_PORTAGE_PACKAGE_DIR}" "${GENTOO_PORTAGE_DISTFILES_DIR}"
+
+sudo mount --bind "${GENTOO_PORTAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage
+mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/distfiles
+sudo mount --bind "${GENTOO_PORTAGE_DISTFILES_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/distfiles
+mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/packages
+sudo mount --bind "${GENTOO_PORTAGE_PACKAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/packages