enable caching for gentoo builds

To facilitate this I've created two new environment variables to set
the environment and default options for package actions.

eval is needed for the export as it preserves quotes.

Change-Id: Ib03651ee8dacd48cd1c135afd57cd31101356056
Signed-off-by: Matthew Thode <mthode@mthode.org>

diskimage_builder/elements/gentoo/README.rst

@@ -30,12 +30,13 @@ Notes:
     default/linux/amd64/17.0/no-multilib
     default/linux/amd64/17.0/hardened
     default/linux/amd64/17.0/no-multilib/hardened
+    default/linux/amd64/17.0/systemd
 
-* You can set the GENTOO_PORTAGE_CLEANUP environment variable to False to
+* You can set the `GENTOO_PORTAGE_CLEANUP` environment variable to False to
-  prevent cleaning up portage repositories (including overlays).  This makes
+  disable the clean up of portage repositories (including overlays).  This
-  the image bigger.
+  will make the image bigger if caching is also disabled.
 
-* Gentoo supports many diferent versions of python, in order to select one
+* Gentoo supports many different versions of python, in order to select one
   you may use the `GENTOO_PYTHON_TARGETS` environment variable to select
   the versions of python you want on your image.  The format of this variable
   is a string as follows `"python2_7 python3_5"`.
@@ -48,3 +49,10 @@ Notes:
 * You can enable overlays using the `GENTOO_OVERLAYS` variable.  In it you
   should put a space separated list of overlays.  The overlays must be in the
   official overlay list and must be git based.
+
+* `GENTOO_EMERGE_ENV` is a bash array containing default environment
+  variables for package install, you can override it with another bash array.
+
+* `GENTOO_EMERGE_DEFAULT_OPTS` can be set to control the default options
+  passed to emerge for all package actions, this includes operations like
+  depclean and preserved-rebuild.

diskimage_builder/elements/gentoo/bin/install-packages

@@ -35,12 +35,12 @@ function show_options {
 function install_gentoo_packages {
     RUN_ONCE_SHM='1'
     if mountpoint -q /dev/shm; then
-        emerge $@
+        emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@
     elif [[ -k /dev/shm ]]; then
-        emerge $@
+        emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@
     else
         fix_shm
-        emerge $@
+        emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@
         unfix_shm
     fi
 }
@@ -58,9 +58,9 @@ while true; do
             if [[ ! -f /usr/portage/profiles ]]; then
                 emerge-webrsync -q
             fi
-            install_gentoo_packages -uDNv --with-bdeps=y --complete-graph=y --jobs=2 @world
+            install_gentoo_packages --update --deep --newuse --complete-graph=y @world
             install_gentoo_packages --verbose=n --depclean
-            install_gentoo_packages -v --usepkg=n @preserved-rebuild
+            install_gentoo_packages --usepkg=n @preserved-rebuild
             etc-update --automode -5
             eselect news read new
             exit 0;

diskimage_builder/elements/gentoo/cleanup.d/05-unmount

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+if [[ ${DIB_DEBUG_TRACE:-0} -gt 0 ]]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+[[ -n "${TMP_MOUNT_PATH}" ]]
+
+sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage/distfiles
+sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage/packages
+sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage
+
+sudo rm -Rf "${TMP_MOUNT_PATH}"/root/.ccache/* "${TMP_MOUNT_PATH}"/usr/src/* "${TMP_MOUNT_PATH}"/var/cache/edb/dep/* "${TMP_MOUNT_PATH}"/var/cache/genkernel/* "${TMP_MOUNT_PATH}"/var/empty/* "${TMP_MOUNT_PATH}"/var/state/* "${TMP_MOUNT_PATH}"/var/tmp/*
+sudo rm -Rf "${TMP_MOUNT_PATH}"/etc/*- "${TMP_MOUNT_PATH}"/etc/*.old "${TMP_MOUNT_PATH}"/etc/ssh/ssh_host_* "${TMP_MOUNT_PATH}"/root/.*history "${TMP_MOUNT_PATH}"/root/.lesshst "${TMP_MOUNT_PATH}"/root/.ssh/known_hosts "${TMP_MOUNT_PATH}"/root/.viminfo "${TMP_MOUNT_PATH}"/usr/share/genkernel "${TMP_MOUNT_PATH}"/usr/lib64/python*/site-packages/gentoolkit/test/eclean/testdistfiles.tar.gz
+if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then
+    # remove portage files
+    sudo rm -Rf "${TMP_MOUNT_PATH}"/usr/portage/* "${TMP_MOUNT_PATH}"/usr/portage/.git "${TMP_MOUNT_PATH}"/var/cache/portage/distfiles
+fi
+
+# shrink a bit
+for i in $(sudo -- find "${TMP_MOUNT_PATH}"/var/log -type f); do sudo -- sh -c "echo > $i"; done
+sudo find "${TMP_MOUNT_PATH}"/usr/share/man/ -mindepth 1  -maxdepth 1 -path "${TMP_MOUNT_PATH}/usr/share/man/man*" -prune -o -exec rm -rf {} \;
+
+# make it so we don't have to reinstall grub
+if [[ -a ${TMP_MOUNT_PATH}/usr/sbin/grub2-install ]]; then
+    sudo mkdir -p "${TMP_MOUNT_PATH}"/tmp/grub
+    sudo touch "${TMP_MOUNT_PATH}"/tmp/grub/install
+fi
+
+# systemd id
+if [[ "${GENTOO_PROFILE}" == *"systemd"* ]]; then
+    sudo cp /dev/null "${TMP_MOUNT_PATH}"/etc/machine-id
+fi

diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash

@@ -5,3 +5,13 @@ export GENTOO_PORTAGE_CLEANUP=${GENTOO_PORTAGE_CLEANUP:-'True'}
 export GENTOO_PYTHON_TARGETS=${GENTOO_PYTHON_TARGETS:-'python2_7 python3_5'}
 export GENTOO_PYTHON_ACTIVE_VERSION=${GENTOO_PYTHON_ACTIVE_VERSION:-'python3.5'}
 export GENTOO_OVERLAYS=${GENTOO_OVERLAYS:-''}
+export GENTOO_EMERGE_DEFAULT_OPTS=${GENTOO_EMERGE_DEFAULT_OPTS:-"--binpkg-respect-use --rebuilt-binaries=y --usepkg=y --with-bdeps=y --binpkg-changed-deps=y --quiet --jobs=2"}
+
+# set the default bash array if GENTOO_EMERGE_ENV is not defined as an array
+if ! declare -p GENTOO_EMERGE_ENV  2> /dev/null | grep -q '^declare \-a'; then
+    export GENTOO_EMERGE_ENV=('USE="-build"' 'FEATURES="binpkg-multi-instance buildpkg parallel-fetch parallel-install"')
+fi
+# itterate over the array, exporting each 'line'
+for (( i=0; i<${#GENTOO_EMERGE_ENV[@]}; i++ )); do
+    eval export "${GENTOO_EMERGE_ENV[i]}"
+done

diskimage_builder/elements/gentoo/finalise.d/99-cleanup

@@ -14,14 +14,17 @@ else
     fix_shm
 fi
 
+# sync if needed
+[[ ! -e /usr/portage/profiles ]] && emerge-webrsync
+
 # make world consistant
-USE="-build" emerge -q --complete-graph=y --jobs=2 --update --newuse --deep --with-bdeps=y @world
+emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --complete-graph=y --update --newuse --deep @world
 # rebuild packages that might need it
-USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
+emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
 # remove unneeded packages
-USE="-build" emerge --verbose=n --depclean
+emerge --verbose=n --depclean
 # rebuild packages that might have somehow depended on the unneeded packages
-USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
+emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
 
 # update config files
 etc-update --automode -5
@@ -31,8 +34,6 @@ emerge --verbose=n --depclean
 emaint all -f
 eselect news read all
 
-rm -Rf /root/.ccache/* /usr/src/* /var/cache/edb/dep/* /var/cache/genkernel/* /var/empty/* /var/run/* /var/state/* /var/tmp/*
-rm -Rf /etc/*- /etc/*.old /etc/ssh/ssh_host_* /root/.*history /root/.lesshst /root/.ssh/known_hosts /root/.viminfo /usr/share/genkernel /usr/lib64/python*/site-packages/gentoolkit/test/eclean/testdistfiles.tar.gz
 if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then
     # remove the overlays
     if [[ ${GENTOO_OVERLAYS} != '' ]]; then
@@ -40,23 +41,9 @@ if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then
             layman -d "${OVERLAY}"
         done
     fi
-    # remove portage files
-    rm -Rf /usr/portage/* /usr/portage/.git /var/cache/portage/distfiles
-fi
-
-# shrink a bit
-for i in $(find /var/log -type f); do echo > $i; done
-find /usr/share/man/ -mindepth 1  -maxdepth 1 -path "/usr/share/man/man*" -prune -o -exec rm -rf {} \;
-
-# make it so we don't have to reinstall grub
-if [[ -a /usr/sbin/grub2-install ]]; then
-    mkdir -p /tmp/grub
-    touch /tmp/grub/install
 fi
 
 unfix_shm
 
-# systemd id
+# remove within the chroot since /var/run is a symlink
-if [[ "${GENTOO_PROFILE}" == *"systemd"* ]]; then
+rm -Rf /var/run/* /run/*
-    cp /dev/null /etc/machine-id
-fi

diskimage_builder/elements/gentoo/pre-finalise.d/01-gentoo-cache

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+GENTOO_PORTAGE_DISTFILES_DIR="${DIB_IMAGE_CACHE}/gentoo/distfiles"
+GENTOO_PORTAGE_PACKAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/packages/${GENTOO_PROFILE}"
+GENTOO_PORTAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/portage"
+
+sudo mount --bind "${GENTOO_PORTAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage
+mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/distfiles
+sudo mount --bind "${GENTOO_PORTAGE_DISTFILES_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/distfiles
+mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/packages
+sudo mount --bind "${GENTOO_PORTAGE_PACKAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/packages

diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-03-enable-overlays

@@ -26,9 +26,9 @@ if [[ ${GENTOO_OVERLAYS} != '' ]]; then
     # depends on that version of openssl.
     echo '=dev-python/cryptography-2.1.4-r2 ~amd64' >> /etc/portage/package.keywords/layman
     echo '=dev-libs/openssl-1.0.2o-r6 ~amd64' >> /etc/portage/package.keywords/layman
-    emerge -q --oneshot --jobs=2 openssl openssh
+    emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot openssl openssh
     # install layman
-    USE="-build" emerge --deep -q --jobs=2 --ignore-built-slot-operator-deps=y layman
+    emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --deep --ignore-built-slot-operator-deps=y layman
     # sync the initial overlay list
     layman -S
     # enable the various overlays, ignore failures (overlay my already be enabled)

diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-04-install-desired-python

@@ -26,11 +26,11 @@ if [[ "${GENTOO_PYTHON_TARGETS}" == *"python3_6"* ]]; then
     echo '~dev-python/setuptools-36.0.1 ~amd64' >> /etc/portage/package.keywords/python
     echo '~dev-python/pyxattr-0.6.0 ~amd64' >> /etc/portage/package.keywords/python
     echo '~sys-apps/kmod-24 ~amd64' >> /etc/portage/package.keywords/python
-    [[ ! -e /usr/lib64/libpython3.6m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.6
+    [[ ! -e /usr/lib64/libpython3.6m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.6
 fi
-[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_5"* ]] && [[ ! -e /usr/lib64/libpython3.5m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.5
+[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_5"* ]] && [[ ! -e /usr/lib64/libpython3.5m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.5
-[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_4"* ]] && [[ ! -e /usr/lib64/libpython3.4m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.4
+[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_4"* ]] && [[ ! -e /usr/lib64/libpython3.4m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.4
-[[ "${GENTOO_PYTHON_TARGETS}" == *"python2_7"* ]] && [[ ! -e /usr/lib64/libpython2.7.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:2.7
+[[ "${GENTOO_PYTHON_TARGETS}" == *"python2_7"* ]] && [[ ! -e /usr/lib64/libpython2.7.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:2.7
 # disable python in git if we have to, it only supports python-2.7
 if [[ "${GENTOO_PYTHON_TARGETS}" != *"python2_7"* ]]; then
     echo 'dev-vcs/git -python' >> /etc/portage/package.use/git
@@ -39,9 +39,9 @@ fi
 # this is done one by one with nodeps because it can sometimes have dependency loops
 PYTHON_PACKAGES="dev-python/six dev-python/packaging dev-python/pyparsing dev-python/appdirs dev-python/setuptools dev-python/certifi"
 for PACKAGE in ${PYTHON_PACKAGES}; do
-    USE="-build" emerge -q --oneshot --jobs=2 --with-bdeps=y --update --newuse --deep --nodeps "${PACKAGE}"
+    emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --update --newuse --deep --nodeps "${PACKAGE}"
 done
-USE="-build" emerge -q --oneshot --jobs=2 --with-bdeps=y --update --newuse --deep --complete-graph=y sys-apps/portage dev-python/pyxattr
+emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --update --newuse --deep --complete-graph=y sys-apps/portage dev-python/pyxattr
 # set the active python version
 eselect python set ${GENTOO_PYTHON_ACTIVE_VERSION}
 # allow these uninstalls to fail as they may not be installed to begin with
@@ -53,12 +53,12 @@ set +e
 set -e
 
 # make world consistant
-USE="-build" emerge -q --complete-graph=y --jobs=2 --update --newuse --deep --with-bdeps=y @world
+emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --complete-graph=y --update --newuse --deep @world
 # rebuild packages that might need it
-USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
+emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
 # remove unneeded packages
-USE="-build" emerge --verbose=n --depclean
+emerge --verbose=n --depclean
 # rebuild packages that might have somehow depended on the unneeded packages
-USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
+emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
 
 unfix_shm

diskimage_builder/elements/gentoo/root.d/50-gentoo-cache

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+[ -n "${TARGET_ROOT}" ]
+
+GENTOO_PORTAGE_DISTFILES_DIR="${DIB_IMAGE_CACHE}/gentoo/distfiles"
+GENTOO_PORTAGE_PACKAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/packages/${GENTOO_PROFILE}"
+GENTOO_PORTAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/portage"
+mkdir -p "${GENTOO_PORTAGE_DIR}" "${GENTOO_PORTAGE_PACKAGE_DIR}" "${GENTOO_PORTAGE_DISTFILES_DIR}"
+
+sudo mount --bind "${GENTOO_PORTAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage
+mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/distfiles
+sudo mount --bind "${GENTOO_PORTAGE_DISTFILES_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/distfiles
+mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/packages
+sudo mount --bind "${GENTOO_PORTAGE_PACKAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/packages