diskimage-builder/lib/common-functions
Derek Higgins 3fada2566a Build raw image in separate tmpfs
If the tmpfs being used to hold the image filesystem is close to being
full then housing the raw disk image in the same place can be
problematic as it tends to grow faster then the filesystem shrinks when
data is being moved into it.

Putting them both into separate tmpfs's will allow there to be an
overlap where they jointly use more then the size limit for a singe tmpfs.

Change-Id: Ia17ca357d3b865d70a2d3e13e0479d008ca5f924
Closes-Bug: #1289582
2014-05-01 11:46:29 +01:00

297 lines
9.1 KiB
Bash

# vim: syntax=sh ts=4 sts=4 sw=2:et
# Copyright 2012 Hewlett-Packard Development Company, L.P.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
function tmpfs_check() {
[ "$DIB_NO_TMPFS" == "0" ] || return 1
[ -r /proc/meminfo ] || return 1
total_kB=$(awk '/^MemTotal/ { print $2 }' /proc/meminfo)
# tmpfs uses by default 50% of the available RAM, so the RAM should be at least
# the double of the minimum tmpfs size required
RAM_NEEDED=$[ $DIB_MIN_TMPFS * 2 ]
[ $total_kB -lt $(($RAM_NEEDED*1024*1024)) ] || return 0
echo "Not enough RAM to use tmpfs for build. ($total_kB < ${RAM_NEEDED}G)"
return 1
}
function mk_build_dir () {
TMP_BUILD_DIR=$(mktemp -t -d --tmpdir=${TMP_DIR:-/tmp} image.XXXXXXXX)
TMP_IMAGE_DIR=$(mktemp -t -d --tmpdir=${TMP_DIR:-/tmp} image.XXXXXXXX)
[ $? -eq 0 ] || die "Failed to create tmp directory"
export TMP_BUILD_DIR
if tmpfs_check ; then
sudo mount -t tmpfs tmpfs $TMP_BUILD_DIR
sudo mount -t tmpfs tmpfs $TMP_IMAGE_DIR
fi
sudo chown $(id -u):$(id -g) $TMP_BUILD_DIR $TMP_IMAGE_DIR
trap cleanup EXIT
echo Building in $TMP_BUILD_DIR
export TMP_IMAGE_PATH=$TMP_IMAGE_DIR/image.raw
export TMP_HOOKS_PATH=$TMP_BUILD_DIR/hooks
}
function finish_image () {
mv $TMP_IMAGE_PATH $1
cleanup_dirs
# All done!
trap EXIT
echo "Image file $1 created..."
}
function save_image () {
# TODO: this really should rename the old file
if [ -f $1 ] ; then
echo "Old Image file Found REMOVING"
rm -f $1
fi
cp $TMP_IMAGE_PATH $1
cleanup_dirs
# All done!
trap EXIT
echo "Image file $1 created..."
}
function element_pre_check() {
local element_dir=$1
[ -d $element_dir ] || return 1
if [ -d $element_dir/first-boot.d ] ; then
first_boot_deprecated_message $element_dir
fi
}
_DISPLAYED_FIRST_BOOT_DEPRECATED=0
function first_boot_deprecated_message() {
local element_dir=$1
echo "WARNING: first-boot.d found in $element_dir"
if [ $_DISPLAYED_FIRST_BOOT_DEPRECATED -eq 1 ] ; then
return
fi
echo "***********************************************************"
echo "* *"
echo "* *"
echo "* *"
echo "* *"
echo "* WARNING: first-boot.d is DEPRECATED, it will be removed *"
echo "* from diskimage-builder in a future release. *"
echo "* *"
echo "* *"
echo "* *"
echo "* *"
echo "***********************************************************"
echo Pausing 10 seconds....
sleep 10
_DISPLAYED_FIRST_BOOT_DEPRECATED=1
}
function generate_hooks () {
mkdir -p $TMP_HOOKS_PATH
for _ELEMENT in $IMAGE_ELEMENT ; do
for dir in $(echo $ELEMENTS_PATH | tr ":" " ") ; do
element_pre_check $dir/$_ELEMENT || continue
cp -t $TMP_HOOKS_PATH -a $dir/$_ELEMENT/* ;
break
done
done
}
# Call the supplied break-in routine if the named point is listed in the break
# list.
# $1 the break point.
# $2.. what to call if a break is needed
function check_break () {
if echo "${break:-}" | egrep -e "(,|^)$1(,|$)" -q; then
echo "Starting debug shell. Exit to resume building." >&2
echo At stage $1 >&2
shift
"$@"
echo "Resuming" >&2
fi
}
# Check that a real element has been chosen (prevents foot-guns)
function check_element () {
[ -d $TMP_HOOKS_PATH ] || generate_hooks
}
# Run a hook, looking for a regex in its stdout, and eval the matched lines.
# $1 is the hook to run
# $2 is the regex to look for
function eval_run_d () {
local TEMP=`run_d $1`
echo "$TEMP"
if [ `echo "$TEMP" | grep -s "$2"` ]; then
TEMP=`echo "$TEMP" | grep "$2"`
eval "$TEMP"
fi
}
# Usage: map_nbd $image
# Returns nbd device path
function map_nbd {
(lsmod | grep '^nbd ' >/dev/null) || sudo modprobe nbd max_part=16
if [[ $(qemu-nbd --help | grep cache) == *writeback* ]] ; then
CACHE="--cache=writeback"
else
echo "Warning: qemu-nbd without --cache=writeback is /slow/."
CACHE=""
fi
NBD_DEV=
for i in `seq 0 15`; do
if [ ! -e /sys/block/nbd$i/pid ]; then
NBD_DEV=/dev/nbd$i
# Connect to nbd and wait till it is ready
sudo qemu-nbd -c $NBD_DEV $CACHE $1
if ! timeout 60 sh -c "while ! [ -e /sys/block/nbd$i/pid ]; do sleep 1; done"; then
echo "Couldn't connect $NBD_DEV"
exit 1
fi
break
fi
done
if [ -z "$NBD_DEV" ]; then
echo "No free NBD slots"
exit 1
fi
}
# Delete and unmount the working dir used in extracting kernel/initrd
function unmount_qcow_image () {
sudo umount $WORK_DIR || true
sudo qemu-nbd -d $NBD_DEV >/dev/null || true
sudo rm -rf $WORK_DIR
trap - SIGHUP SIGINT SIGTERM EXIT
}
function mount_qcow_image() {
trap unmount_qcow_image SIGHUP SIGINT SIGTERM EXIT
WORK_DIR=$(mktemp -d)
map_nbd $1
if [ -e "${NBD_DEV}p1" ]; then
sudo mount ${NBD_DEV}p1 $WORK_DIR
else
sudo mount ${NBD_DEV} $WORK_DIR
fi
}
function cleanup_dirs () {
sudo rm -rf $TMP_BUILD_DIR/built
sudo rm -rf $TMP_BUILD_DIR/mnt
sudo umount -f $TMP_BUILD_DIR $TMP_IMAGE_DIR || true
rm -rf $TMP_BUILD_DIR $TMP_IMAGE_DIR
}
# Run a directory of hooks outside the target.
function run_d() {
check_element
check_break before-$1 bash
if [ -d ${TMP_HOOKS_PATH}/$1.d ] ; then
${SCRIPT_HOME}/dib-run-parts ${TMP_HOOKS_PATH}/$1.d
fi
check_break after-$1 bash
}
function detach_loopback() {
local loopdev=$1
# Remove the map if it exists
# If setup on a rhel or derivative the map was created with kpartx not losetup
# and subsequently needs to be removed.
loopdev_name=$(echo $loopdev | sed 's/\/dev\///g')
mapper_name=$(sudo dmsetup ls | grep $loopdev_name | awk '{ print $1 }')
if [ "$mapper_name" ]; then
sudo dmsetup remove $mapper_name
fi
# loopback dev may be tied up a bit by udev events triggered by partition events
for try in $(seq 10 -1 1) ; do
if sudo losetup -d $loopdev ; then
return 0
fi
echo $loopdev may be busy, sleeping up to $try more seconds...
sleep 1
done
echo Gave up trying to detach $loopdev
return 1
}
function arg_to_elements() {
for arg do IMAGE_ELEMENT="$IMAGE_ELEMENT $arg" ; done
if [ "$SKIP_BASE" != "1" ]; then
IMAGE_ELEMENT="base $IMAGE_ELEMENT"
fi
if [ "$IS_RAMDISK" == "1" ]; then
IMAGE_ELEMENT="ramdisk $IMAGE_ELEMENT"
fi
echo "Building elements: $IMAGE_ELEMENT"
IMAGE_ELEMENT=$($SCRIPT_HOME/element-info --expand-dependencies $IMAGE_ELEMENT)
echo "Expanded element dependencies to: $IMAGE_ELEMENT"
}
function create_base () {
mkdir $TMP_BUILD_DIR/mnt
export TMP_MOUNT_PATH=$TMP_BUILD_DIR/mnt
# Copy data in to the root.
TARGET_ROOT=$TMP_MOUNT_PATH run_d root
if [ -z "$(ls $TMP_MOUNT_PATH | grep -v '^lost+found\|tmp$')" ] ; then
# No root element copied in. Note the test above allows
# root.d elements to put things in /tmp
echo "Please include at least one distribution root element."
exit 1
fi
# Configure Image
# Setup resolv.conf so we can chroot to install some packages
if [ -L $TMP_MOUNT_PATH/etc/resolv.conf ] || [ -f $TMP_MOUNT_PATH/etc/resolv.conf ] ; then
sudo mv $TMP_MOUNT_PATH/etc/resolv.conf $TMP_MOUNT_PATH/etc/resolv.conf.ORIG
fi
# Recreate resolv.conf
sudo touch $TMP_MOUNT_PATH/etc/resolv.conf
sudo chmod 777 $TMP_MOUNT_PATH/etc/resolv.conf
# use system configured resolv.conf if available to support internal proxy resolving
if [ -e /etc/resolv.conf ]
then
cat /etc/resolv.conf > $TMP_MOUNT_PATH/etc/resolv.conf
else
echo nameserver 8.8.8.8 > $TMP_MOUNT_PATH/etc/resolv.conf
fi
mount_proc_dev_sys
}
function mount_proc_dev_sys () {
# supporting kernel file systems
sudo mount -t proc none $TMP_MOUNT_PATH/proc
sudo mount --bind /dev $TMP_MOUNT_PATH/dev
sudo mount --bind /dev/pts $TMP_MOUNT_PATH/dev/pts
sudo mount -t sysfs none $TMP_MOUNT_PATH/sys
}
function unmount_dir () {
local pattern="$1" mnts=""
if [ -n "$pattern" ]; then
mnts=`awk '{print $2}' < /proc/mounts | grep "^$pattern" | sort -r`
fi
if [ -n "$mnts" ]; then
sudo umount -fl $mnts || true
fi
}