enable caching for gentoo builds
To facilitate this I've created two new environment variables to set the environment and default options for package actions. eval is needed for the export as it preserves quotes. Change-Id: Ib03651ee8dacd48cd1c135afd57cd31101356056 Signed-off-by: Matthew Thode <mthode@mthode.org>
This commit is contained in:
parent
b86af3dc6a
commit
89f95a5351
@ -30,12 +30,13 @@ Notes:
|
||||
default/linux/amd64/17.0/no-multilib
|
||||
default/linux/amd64/17.0/hardened
|
||||
default/linux/amd64/17.0/no-multilib/hardened
|
||||
default/linux/amd64/17.0/systemd
|
||||
|
||||
* You can set the GENTOO_PORTAGE_CLEANUP environment variable to False to
|
||||
prevent cleaning up portage repositories (including overlays). This makes
|
||||
the image bigger.
|
||||
* You can set the `GENTOO_PORTAGE_CLEANUP` environment variable to False to
|
||||
disable the clean up of portage repositories (including overlays). This
|
||||
will make the image bigger if caching is also disabled.
|
||||
|
||||
* Gentoo supports many diferent versions of python, in order to select one
|
||||
* Gentoo supports many different versions of python, in order to select one
|
||||
you may use the `GENTOO_PYTHON_TARGETS` environment variable to select
|
||||
the versions of python you want on your image. The format of this variable
|
||||
is a string as follows `"python2_7 python3_5"`.
|
||||
@ -48,3 +49,10 @@ Notes:
|
||||
* You can enable overlays using the `GENTOO_OVERLAYS` variable. In it you
|
||||
should put a space separated list of overlays. The overlays must be in the
|
||||
official overlay list and must be git based.
|
||||
|
||||
* `GENTOO_EMERGE_ENV` is a bash array containing default environment
|
||||
variables for package install, you can override it with another bash array.
|
||||
|
||||
* `GENTOO_EMERGE_DEFAULT_OPTS` can be set to control the default options
|
||||
passed to emerge for all package actions, this includes operations like
|
||||
depclean and preserved-rebuild.
|
||||
|
@ -35,12 +35,12 @@ function show_options {
|
||||
function install_gentoo_packages {
|
||||
RUN_ONCE_SHM='1'
|
||||
if mountpoint -q /dev/shm; then
|
||||
emerge $@
|
||||
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@
|
||||
elif [[ -k /dev/shm ]]; then
|
||||
emerge $@
|
||||
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@
|
||||
else
|
||||
fix_shm
|
||||
emerge $@
|
||||
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} $@
|
||||
unfix_shm
|
||||
fi
|
||||
}
|
||||
@ -58,9 +58,9 @@ while true; do
|
||||
if [[ ! -f /usr/portage/profiles ]]; then
|
||||
emerge-webrsync -q
|
||||
fi
|
||||
install_gentoo_packages -uDNv --with-bdeps=y --complete-graph=y --jobs=2 @world
|
||||
install_gentoo_packages --update --deep --newuse --complete-graph=y @world
|
||||
install_gentoo_packages --verbose=n --depclean
|
||||
install_gentoo_packages -v --usepkg=n @preserved-rebuild
|
||||
install_gentoo_packages --usepkg=n @preserved-rebuild
|
||||
etc-update --automode -5
|
||||
eselect news read new
|
||||
exit 0;
|
||||
|
35
diskimage_builder/elements/gentoo/cleanup.d/05-unmount
Executable file
35
diskimage_builder/elements/gentoo/cleanup.d/05-unmount
Executable file
@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [[ ${DIB_DEBUG_TRACE:-0} -gt 0 ]]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
[[ -n "${TMP_MOUNT_PATH}" ]]
|
||||
|
||||
sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage/distfiles
|
||||
sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage/packages
|
||||
sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage
|
||||
|
||||
sudo rm -Rf "${TMP_MOUNT_PATH}"/root/.ccache/* "${TMP_MOUNT_PATH}"/usr/src/* "${TMP_MOUNT_PATH}"/var/cache/edb/dep/* "${TMP_MOUNT_PATH}"/var/cache/genkernel/* "${TMP_MOUNT_PATH}"/var/empty/* "${TMP_MOUNT_PATH}"/var/state/* "${TMP_MOUNT_PATH}"/var/tmp/*
|
||||
sudo rm -Rf "${TMP_MOUNT_PATH}"/etc/*- "${TMP_MOUNT_PATH}"/etc/*.old "${TMP_MOUNT_PATH}"/etc/ssh/ssh_host_* "${TMP_MOUNT_PATH}"/root/.*history "${TMP_MOUNT_PATH}"/root/.lesshst "${TMP_MOUNT_PATH}"/root/.ssh/known_hosts "${TMP_MOUNT_PATH}"/root/.viminfo "${TMP_MOUNT_PATH}"/usr/share/genkernel "${TMP_MOUNT_PATH}"/usr/lib64/python*/site-packages/gentoolkit/test/eclean/testdistfiles.tar.gz
|
||||
if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then
|
||||
# remove portage files
|
||||
sudo rm -Rf "${TMP_MOUNT_PATH}"/usr/portage/* "${TMP_MOUNT_PATH}"/usr/portage/.git "${TMP_MOUNT_PATH}"/var/cache/portage/distfiles
|
||||
fi
|
||||
|
||||
# shrink a bit
|
||||
for i in $(sudo -- find "${TMP_MOUNT_PATH}"/var/log -type f); do sudo -- sh -c "echo > $i"; done
|
||||
sudo find "${TMP_MOUNT_PATH}"/usr/share/man/ -mindepth 1 -maxdepth 1 -path "${TMP_MOUNT_PATH}/usr/share/man/man*" -prune -o -exec rm -rf {} \;
|
||||
|
||||
# make it so we don't have to reinstall grub
|
||||
if [[ -a ${TMP_MOUNT_PATH}/usr/sbin/grub2-install ]]; then
|
||||
sudo mkdir -p "${TMP_MOUNT_PATH}"/tmp/grub
|
||||
sudo touch "${TMP_MOUNT_PATH}"/tmp/grub/install
|
||||
fi
|
||||
|
||||
# systemd id
|
||||
if [[ "${GENTOO_PROFILE}" == *"systemd"* ]]; then
|
||||
sudo cp /dev/null "${TMP_MOUNT_PATH}"/etc/machine-id
|
||||
fi
|
@ -5,3 +5,13 @@ export GENTOO_PORTAGE_CLEANUP=${GENTOO_PORTAGE_CLEANUP:-'True'}
|
||||
export GENTOO_PYTHON_TARGETS=${GENTOO_PYTHON_TARGETS:-'python2_7 python3_5'}
|
||||
export GENTOO_PYTHON_ACTIVE_VERSION=${GENTOO_PYTHON_ACTIVE_VERSION:-'python3.5'}
|
||||
export GENTOO_OVERLAYS=${GENTOO_OVERLAYS:-''}
|
||||
export GENTOO_EMERGE_DEFAULT_OPTS=${GENTOO_EMERGE_DEFAULT_OPTS:-"--binpkg-respect-use --rebuilt-binaries=y --usepkg=y --with-bdeps=y --binpkg-changed-deps=y --quiet --jobs=2"}
|
||||
|
||||
# set the default bash array if GENTOO_EMERGE_ENV is not defined as an array
|
||||
if ! declare -p GENTOO_EMERGE_ENV 2> /dev/null | grep -q '^declare \-a'; then
|
||||
export GENTOO_EMERGE_ENV=('USE="-build"' 'FEATURES="binpkg-multi-instance buildpkg parallel-fetch parallel-install"')
|
||||
fi
|
||||
# itterate over the array, exporting each 'line'
|
||||
for (( i=0; i<${#GENTOO_EMERGE_ENV[@]}; i++ )); do
|
||||
eval export "${GENTOO_EMERGE_ENV[i]}"
|
||||
done
|
||||
|
@ -14,14 +14,17 @@ else
|
||||
fix_shm
|
||||
fi
|
||||
|
||||
# sync if needed
|
||||
[[ ! -e /usr/portage/profiles ]] && emerge-webrsync
|
||||
|
||||
# make world consistant
|
||||
USE="-build" emerge -q --complete-graph=y --jobs=2 --update --newuse --deep --with-bdeps=y @world
|
||||
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --complete-graph=y --update --newuse --deep @world
|
||||
# rebuild packages that might need it
|
||||
USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
|
||||
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
|
||||
# remove unneeded packages
|
||||
USE="-build" emerge --verbose=n --depclean
|
||||
emerge --verbose=n --depclean
|
||||
# rebuild packages that might have somehow depended on the unneeded packages
|
||||
USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
|
||||
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
|
||||
|
||||
# update config files
|
||||
etc-update --automode -5
|
||||
@ -31,8 +34,6 @@ emerge --verbose=n --depclean
|
||||
emaint all -f
|
||||
eselect news read all
|
||||
|
||||
rm -Rf /root/.ccache/* /usr/src/* /var/cache/edb/dep/* /var/cache/genkernel/* /var/empty/* /var/run/* /var/state/* /var/tmp/*
|
||||
rm -Rf /etc/*- /etc/*.old /etc/ssh/ssh_host_* /root/.*history /root/.lesshst /root/.ssh/known_hosts /root/.viminfo /usr/share/genkernel /usr/lib64/python*/site-packages/gentoolkit/test/eclean/testdistfiles.tar.gz
|
||||
if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then
|
||||
# remove the overlays
|
||||
if [[ ${GENTOO_OVERLAYS} != '' ]]; then
|
||||
@ -40,23 +41,9 @@ if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then
|
||||
layman -d "${OVERLAY}"
|
||||
done
|
||||
fi
|
||||
# remove portage files
|
||||
rm -Rf /usr/portage/* /usr/portage/.git /var/cache/portage/distfiles
|
||||
fi
|
||||
|
||||
# shrink a bit
|
||||
for i in $(find /var/log -type f); do echo > $i; done
|
||||
find /usr/share/man/ -mindepth 1 -maxdepth 1 -path "/usr/share/man/man*" -prune -o -exec rm -rf {} \;
|
||||
|
||||
# make it so we don't have to reinstall grub
|
||||
if [[ -a /usr/sbin/grub2-install ]]; then
|
||||
mkdir -p /tmp/grub
|
||||
touch /tmp/grub/install
|
||||
fi
|
||||
|
||||
unfix_shm
|
||||
|
||||
# systemd id
|
||||
if [[ "${GENTOO_PROFILE}" == *"systemd"* ]]; then
|
||||
cp /dev/null /etc/machine-id
|
||||
fi
|
||||
# remove within the chroot since /var/run is a symlink
|
||||
rm -Rf /var/run/* /run/*
|
||||
|
17
diskimage_builder/elements/gentoo/pre-finalise.d/01-gentoo-cache
Executable file
17
diskimage_builder/elements/gentoo/pre-finalise.d/01-gentoo-cache
Executable file
@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
GENTOO_PORTAGE_DISTFILES_DIR="${DIB_IMAGE_CACHE}/gentoo/distfiles"
|
||||
GENTOO_PORTAGE_PACKAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/packages/${GENTOO_PROFILE}"
|
||||
GENTOO_PORTAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/portage"
|
||||
|
||||
sudo mount --bind "${GENTOO_PORTAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage
|
||||
mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/distfiles
|
||||
sudo mount --bind "${GENTOO_PORTAGE_DISTFILES_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/distfiles
|
||||
mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/packages
|
||||
sudo mount --bind "${GENTOO_PORTAGE_PACKAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/packages
|
@ -26,9 +26,9 @@ if [[ ${GENTOO_OVERLAYS} != '' ]]; then
|
||||
# depends on that version of openssl.
|
||||
echo '=dev-python/cryptography-2.1.4-r2 ~amd64' >> /etc/portage/package.keywords/layman
|
||||
echo '=dev-libs/openssl-1.0.2o-r6 ~amd64' >> /etc/portage/package.keywords/layman
|
||||
emerge -q --oneshot --jobs=2 openssl openssh
|
||||
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot openssl openssh
|
||||
# install layman
|
||||
USE="-build" emerge --deep -q --jobs=2 --ignore-built-slot-operator-deps=y layman
|
||||
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --deep --ignore-built-slot-operator-deps=y layman
|
||||
# sync the initial overlay list
|
||||
layman -S
|
||||
# enable the various overlays, ignore failures (overlay my already be enabled)
|
||||
|
@ -26,11 +26,11 @@ if [[ "${GENTOO_PYTHON_TARGETS}" == *"python3_6"* ]]; then
|
||||
echo '~dev-python/setuptools-36.0.1 ~amd64' >> /etc/portage/package.keywords/python
|
||||
echo '~dev-python/pyxattr-0.6.0 ~amd64' >> /etc/portage/package.keywords/python
|
||||
echo '~sys-apps/kmod-24 ~amd64' >> /etc/portage/package.keywords/python
|
||||
[[ ! -e /usr/lib64/libpython3.6m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.6
|
||||
[[ ! -e /usr/lib64/libpython3.6m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.6
|
||||
fi
|
||||
[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_5"* ]] && [[ ! -e /usr/lib64/libpython3.5m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.5
|
||||
[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_4"* ]] && [[ ! -e /usr/lib64/libpython3.4m.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:3.4
|
||||
[[ "${GENTOO_PYTHON_TARGETS}" == *"python2_7"* ]] && [[ ! -e /usr/lib64/libpython2.7.so ]] && USE="-build" emerge -1q --jobs=2 dev-lang/python:2.7
|
||||
[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_5"* ]] && [[ ! -e /usr/lib64/libpython3.5m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.5
|
||||
[[ "${GENTOO_PYTHON_TARGETS}" == *"python3_4"* ]] && [[ ! -e /usr/lib64/libpython3.4m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.4
|
||||
[[ "${GENTOO_PYTHON_TARGETS}" == *"python2_7"* ]] && [[ ! -e /usr/lib64/libpython2.7.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:2.7
|
||||
# disable python in git if we have to, it only supports python-2.7
|
||||
if [[ "${GENTOO_PYTHON_TARGETS}" != *"python2_7"* ]]; then
|
||||
echo 'dev-vcs/git -python' >> /etc/portage/package.use/git
|
||||
@ -39,9 +39,9 @@ fi
|
||||
# this is done one by one with nodeps because it can sometimes have dependency loops
|
||||
PYTHON_PACKAGES="dev-python/six dev-python/packaging dev-python/pyparsing dev-python/appdirs dev-python/setuptools dev-python/certifi"
|
||||
for PACKAGE in ${PYTHON_PACKAGES}; do
|
||||
USE="-build" emerge -q --oneshot --jobs=2 --with-bdeps=y --update --newuse --deep --nodeps "${PACKAGE}"
|
||||
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --update --newuse --deep --nodeps "${PACKAGE}"
|
||||
done
|
||||
USE="-build" emerge -q --oneshot --jobs=2 --with-bdeps=y --update --newuse --deep --complete-graph=y sys-apps/portage dev-python/pyxattr
|
||||
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --update --newuse --deep --complete-graph=y sys-apps/portage dev-python/pyxattr
|
||||
# set the active python version
|
||||
eselect python set ${GENTOO_PYTHON_ACTIVE_VERSION}
|
||||
# allow these uninstalls to fail as they may not be installed to begin with
|
||||
@ -53,12 +53,12 @@ set +e
|
||||
set -e
|
||||
|
||||
# make world consistant
|
||||
USE="-build" emerge -q --complete-graph=y --jobs=2 --update --newuse --deep --with-bdeps=y @world
|
||||
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --complete-graph=y --update --newuse --deep @world
|
||||
# rebuild packages that might need it
|
||||
USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
|
||||
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
|
||||
# remove unneeded packages
|
||||
USE="-build" emerge --verbose=n --depclean
|
||||
emerge --verbose=n --depclean
|
||||
# rebuild packages that might have somehow depended on the unneeded packages
|
||||
USE="-build" emerge -q --jobs=2 --usepkg=n @preserved-rebuild
|
||||
emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --usepkg=n @preserved-rebuild
|
||||
|
||||
unfix_shm
|
||||
|
20
diskimage_builder/elements/gentoo/root.d/50-gentoo-cache
Executable file
20
diskimage_builder/elements/gentoo/root.d/50-gentoo-cache
Executable file
@ -0,0 +1,20 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
[ -n "${TARGET_ROOT}" ]
|
||||
|
||||
GENTOO_PORTAGE_DISTFILES_DIR="${DIB_IMAGE_CACHE}/gentoo/distfiles"
|
||||
GENTOO_PORTAGE_PACKAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/packages/${GENTOO_PROFILE}"
|
||||
GENTOO_PORTAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/portage"
|
||||
mkdir -p "${GENTOO_PORTAGE_DIR}" "${GENTOO_PORTAGE_PACKAGE_DIR}" "${GENTOO_PORTAGE_DISTFILES_DIR}"
|
||||
|
||||
sudo mount --bind "${GENTOO_PORTAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage
|
||||
mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/distfiles
|
||||
sudo mount --bind "${GENTOO_PORTAGE_DISTFILES_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/distfiles
|
||||
mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/packages
|
||||
sudo mount --bind "${GENTOO_PORTAGE_PACKAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/packages
|
Loading…
Reference in New Issue
Block a user