enable caching for gentoo builds

To facilitate this I've created two new environment variables to set
the environment and default options for package actions.

eval is needed for the export as it preserves quotes.

Change-Id: Ib03651ee8dacd48cd1c135afd57cd31101356056
Signed-off-by: Matthew Thode <mthode@mthode.org>
This commit is contained in:
Matthew Thode 2018-10-08 00:39:35 -05:00
parent b86af3dc6a
commit 89f95a5351
No known key found for this signature in database
GPG Key ID: 64A37BEAAE19A4E8
9 changed files with 120 additions and 43 deletions

View File

@ -30,12 +30,13 @@ Notes:
default/linux/amd64/17.0/no-multilib
default/linux/amd64/17.0/hardened
default/linux/amd64/17.0/no-multilib/hardened
default/linux/amd64/17.0/systemd
* You can set the GENTOO_PORTAGE_CLEANUP environment variable to False to
prevent cleaning up portage repositories (including overlays). This makes
the image bigger.
* You can set the `GENTOO_PORTAGE_CLEANUP` environment variable to False to
disable the clean up of portage repositories (including overlays). This
will make the image bigger if caching is also disabled.
* Gentoo supports many diferent versions of python, in order to select one
* Gentoo supports many different versions of python, in order to select one
you may use the `GENTOO_PYTHON_TARGETS` environment variable to select
the versions of python you want on your image. The format of this variable
is a string as follows `"python2_7 python3_5"`.
@ -48,3 +49,10 @@ Notes:
* You can enable overlays using the `GENTOO_OVERLAYS` variable. In it you
should put a space separated list of overlays. The overlays must be in the
official overlay list and must be git based.
* `GENTOO_EMERGE_ENV` is a bash array containing default environment
variables for package install, you can override it with another bash array.
* `GENTOO_EMERGE_DEFAULT_OPTS` can be set to control the default options
passed to emerge for all package actions, this includes operations like
depclean and preserved-rebuild.

View File

@ -35,12 +35,12 @@ function show_options {
function install_gentoo_packages {
RUN_ONCE_SHM='1'
if mountpoint -q /dev/shm; then
emerge $@
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@
elif [[ -k /dev/shm ]]; then
emerge $@
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@
else
fix_shm
emerge $@
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@
unfix_shm
fi
}
@ -58,9 +58,9 @@ while true; do
if [[ ! -f /usr/portage/profiles ]]; then
emerge-webrsync -q
fi
install_gentoo_packages -uDNv --with-bdeps=y --complete-graph=y --jobs=2 @world
install_gentoo_packages --update --deep --newuse --complete-graph=y @world
install_gentoo_packages --verbose=n --depclean
install_gentoo_packages -v --usepkg=n @preserved-rebuild
install_gentoo_packages --usepkg=n @preserved-rebuild
etc-update --automode -5
eselect news read new
exit 0;

View File

@ -0,0 +1,35 @@
#!/bin/bash
if [[ ${DIB_DEBUG_TRACE:-0} -gt 0 ]]; then
set -x
fi
set -eu
set -o pipefail
[[ -n "${TMP_MOUNT_PATH}" ]]
sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage/distfiles
sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage/packages
sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage
sudo rm -Rf "${TMP_MOUNT_PATH}"/root/.ccache/* "${TMP_MOUNT_PATH}"/usr/src/* "${TMP_MOUNT_PATH}"/var/cache/edb/dep/* "${TMP_MOUNT_PATH}"/var/cache/genkernel/* "${TMP_MOUNT_PATH}"/var/empty/* "${TMP_MOUNT_PATH}"/var/state/* "${TMP_MOUNT_PATH}"/var/tmp/*
sudo rm -Rf "${TMP_MOUNT_PATH}"/etc/*- "${TMP_MOUNT_PATH}"/etc/*.old "${TMP_MOUNT_PATH}"/etc/ssh/ssh_host_* "${TMP_MOUNT_PATH}"/root/.*history "${TMP_MOUNT_PATH}"/root/.lesshst "${TMP_MOUNT_PATH}"/root/.ssh/known_hosts "${TMP_MOUNT_PATH}"/root/.viminfo "${TMP_MOUNT_PATH}"/usr/share/genkernel "${TMP_MOUNT_PATH}"/usr/lib64/python*/site-packages/gentoolkit/test/eclean/testdistfiles.tar.gz
if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then
# remove portage files
sudo rm -Rf "${TMP_MOUNT_PATH}"/usr/portage/* "${TMP_MOUNT_PATH}"/usr/portage/.git "${TMP_MOUNT_PATH}"/var/cache/portage/distfiles
fi
# shrink a bit
for i in $(sudo -- find "${TMP_MOUNT_PATH}"/var/log -type f); do sudo -- sh -c "echo > $i"; done
sudo find "${TMP_MOUNT_PATH}"/usr/share/man/ -mindepth 1 -maxdepth 1 -path "${TMP_MOUNT_PATH}/usr/share/man/man*" -prune -o -exec rm -rf {} \;
# make it so we don't have to reinstall grub
if [[ -a ${TMP_MOUNT_PATH}/usr/sbin/grub2-install ]]; then
sudo mkdir -p "${TMP_MOUNT_PATH}"/tmp/grub
sudo touch "${TMP_MOUNT_PATH}"/tmp/grub/install
fi
# systemd id
if [[ "${GENTOO_PROFILE}" == *"systemd"* ]]; then
sudo cp /dev/null "${TMP_MOUNT_PATH}"/etc/machine-id
fi

View File

@ -5,3 +5,13 @@ export GENTOO_PORTAGE_CLEANUP=${GENTOO_PORTAGE_CLEANUP:-'True'}
export GENTOO_PYTHON_TARGETS=${GENTOO_PYTHON_TARGETS:-'python2_7 python3_5'}
export GENTOO_PYTHON_ACTIVE_VERSION=${GENTOO_PYTHON_ACTIVE_VERSION:-'python3.5'}
export GENTOO_OVERLAYS=${GENTOO_OVERLAYS:-''}
export GENTOO_EMERGE_DEFAULT_OPTS=${GENTOO_EMERGE_DEFAULT_OPTS:-"--binpkg-respect-use --rebuilt-binaries=y --usepkg=y --with-bdeps=y --binpkg-changed-deps=y --quiet --jobs=2"}
# set the default bash array if GENTOO_EMERGE_ENV is not defined as an array
if ! declare -p GENTOO_EMERGE_ENV 2> /dev/null | grep -q '^declare \-a'; then
export GENTOO_EMERGE_ENV=('USE="-build"' 'FEATURES="binpkg-multi-instance buildpkg parallel-fetch parallel-install"')
fi
# itterate over the array, exporting each 'line'
for (( i=0; i<${#GENTOO_EMERGE_ENV[@]}; i++ )); do
eval export "${GENTOO_EMERGE_ENV[i]}"
done

View File

@ -14,14 +14,17 @@ else
fix_shm
fi
# sync if needed
[[ ! -e /usr/portage/profiles ]] && emerge-webrsync
# make world consistant
USE="-build" emerge -q --complete-graph=y --jobs=2 --update --newuse --deep --with-bdeps=y @world
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --complete-graph=y --update --newuse --deep @world
# rebuild packages that might need it
USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
# remove unneeded packages
USE="-build" emerge --verbose=n --depclean
emerge --verbose=n --depclean
# rebuild packages that might have somehow depended on the unneeded packages
USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
# update config files
etc-update --automode -5
@ -31,8 +34,6 @@ emerge --verbose=n --depclean
emaint all -f
eselect news read all
rm -Rf /root/.ccache/* /usr/src/* /var/cache/edb/dep/* /var/cache/genkernel/* /var/empty/* /var/run/* /var/state/* /var/tmp/*
rm -Rf /etc/*- /etc/*.old /etc/ssh/ssh_host_* /root/.*history /root/.lesshst /root/.ssh/known_hosts /root/.viminfo /usr/share/genkernel /usr/lib64/python*/site-packages/gentoolkit/test/eclean/testdistfiles.tar.gz
if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then
# remove the overlays
if [[ ${GENTOO_OVERLAYS} != '' ]]; then
@ -40,23 +41,9 @@ if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then
layman -d "${OVERLAY}"
done
fi
# remove portage files
rm -Rf /usr/portage/* /usr/portage/.git /var/cache/portage/distfiles
fi
# shrink a bit
for i in $(find /var/log -type f); do echo > $i; done
find /usr/share/man/ -mindepth 1 -maxdepth 1 -path "/usr/share/man/man*" -prune -o -exec rm -rf {} \;
# make it so we don't have to reinstall grub
if [[ -a /usr/sbin/grub2-install ]]; then
mkdir -p /tmp/grub
touch /tmp/grub/install
fi
unfix_shm
# systemd id
if [[ "${GENTOO_PROFILE}" == *"systemd"* ]]; then
cp /dev/null /etc/machine-id
fi
# remove within the chroot since /var/run is a symlink
rm -Rf /var/run/* /run/*

View File

@ -0,0 +1,17 @@
#!/bin/bash
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
GENTOO_PORTAGE_DISTFILES_DIR="${DIB_IMAGE_CACHE}/gentoo/distfiles"
GENTOO_PORTAGE_PACKAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/packages/${GENTOO_PROFILE}"
GENTOO_PORTAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/portage"
sudo mount --bind "${GENTOO_PORTAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage
mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/distfiles
sudo mount --bind "${GENTOO_PORTAGE_DISTFILES_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/distfiles
mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/packages
sudo mount --bind "${GENTOO_PORTAGE_PACKAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/packages

View File

@ -26,9 +26,9 @@ if [[ ${GENTOO_OVERLAYS} != '' ]]; then
# depends on that version of openssl.
echo '=dev-python/cryptography-2.1.4-r2 ~amd64' >> /etc/portage/package.keywords/layman
echo '=dev-libs/openssl-1.0.2o-r6 ~amd64' >> /etc/portage/package.keywords/layman
emerge -q --oneshot --jobs=2 openssl openssh
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot openssl openssh
# install layman
USE="-build" emerge --deep -q --jobs=2 --ignore-built-slot-operator-deps=y layman
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --deep --ignore-built-slot-operator-deps=y layman
# sync the initial overlay list
layman -S
# enable the various overlays, ignore failures (overlay my already be enabled)

View File

@ -26,11 +26,11 @@ if [[ "${GENTOO_PYTHON_TARGETS}" == *"python3_6"* ]]; then
echo '~dev-python/setuptools-36.0.1 ~amd64' >> /etc/portage/package.keywords/python
echo '~dev-python/pyxattr-0.6.0 ~amd64' >> /etc/portage/package.keywords/python
echo '~sys-apps/kmod-24 ~amd64' >> /etc/portage/package.keywords/python
[[ ! -e /usr/lib64/libpython3.6m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.6
[[ ! -e /usr/lib64/libpython3.6m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.6
fi
[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_5"* ]] && [[ ! -e /usr/lib64/libpython3.5m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.5
[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_4"* ]] && [[ ! -e /usr/lib64/libpython3.4m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.4
[[ "${GENTOO_PYTHON_TARGETS}" == *"python2_7"* ]] && [[ ! -e /usr/lib64/libpython2.7.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:2.7
[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_5"* ]] && [[ ! -e /usr/lib64/libpython3.5m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.5
[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_4"* ]] && [[ ! -e /usr/lib64/libpython3.4m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.4
[[ "${GENTOO_PYTHON_TARGETS}" == *"python2_7"* ]] && [[ ! -e /usr/lib64/libpython2.7.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:2.7
# disable python in git if we have to, it only supports python-2.7
if [[ "${GENTOO_PYTHON_TARGETS}" != *"python2_7"* ]]; then
echo 'dev-vcs/git -python' >> /etc/portage/package.use/git
@ -39,9 +39,9 @@ fi
# this is done one by one with nodeps because it can sometimes have dependency loops
PYTHON_PACKAGES="dev-python/six dev-python/packaging dev-python/pyparsing dev-python/appdirs dev-python/setuptools dev-python/certifi"
for PACKAGE in ${PYTHON_PACKAGES}; do
USE="-build" emerge -q --oneshot --jobs=2 --with-bdeps=y --update --newuse --deep --nodeps "${PACKAGE}"
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --update --newuse --deep --nodeps "${PACKAGE}"
done
USE="-build" emerge -q --oneshot --jobs=2 --with-bdeps=y --update --newuse --deep --complete-graph=y sys-apps/portage dev-python/pyxattr
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --update --newuse --deep --complete-graph=y sys-apps/portage dev-python/pyxattr
# set the active python version
eselect python set ${GENTOO_PYTHON_ACTIVE_VERSION}
# allow these uninstalls to fail as they may not be installed to begin with
@ -53,12 +53,12 @@ set +e
set -e
# make world consistant
USE="-build" emerge -q --complete-graph=y --jobs=2 --update --newuse --deep --with-bdeps=y @world
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --complete-graph=y --update --newuse --deep @world
# rebuild packages that might need it
USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
# remove unneeded packages
USE="-build" emerge --verbose=n --depclean
emerge --verbose=n --depclean
# rebuild packages that might have somehow depended on the unneeded packages
USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
unfix_shm

View File

@ -0,0 +1,20 @@
#!/bin/bash
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
[ -n "${TARGET_ROOT}" ]
GENTOO_PORTAGE_DISTFILES_DIR="${DIB_IMAGE_CACHE}/gentoo/distfiles"
GENTOO_PORTAGE_PACKAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/packages/${GENTOO_PROFILE}"
GENTOO_PORTAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/portage"
mkdir -p "${GENTOO_PORTAGE_DIR}" "${GENTOO_PORTAGE_PACKAGE_DIR}" "${GENTOO_PORTAGE_DISTFILES_DIR}"
sudo mount --bind "${GENTOO_PORTAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage
mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/distfiles
sudo mount --bind "${GENTOO_PORTAGE_DISTFILES_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/distfiles
mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/packages
sudo mount --bind "${GENTOO_PORTAGE_PACKAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/packages