diskimage-builder/diskimage_builder/lib/common-functions

357 lines
11 KiB
Plaintext
Raw Normal View History

#!/bin/bash
2012-11-15 03:20:32 +00:00
# Copyright 2012 Hewlett-Packard Development Company, L.P.
# All Rights Reserved.
#
2012-11-15 03:20:32 +00:00
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
2012-11-15 03:20:32 +00:00
# http://www.apache.org/licenses/LICENSE-2.0
#
2012-11-15 03:20:32 +00:00
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
function tmpfs_check() {
local echo_message=${1:-1}
[ "$DIB_NO_TMPFS" == "0" ] || return 1
[ -r /proc/meminfo ] || return 1
total_kB=$(awk '/^MemTotal/ { print $2 }' /proc/meminfo)
# tmpfs uses by default 50% of the available RAM, so the RAM should be at least
# the double of the minimum tmpfs size required
RAM_NEEDED=$(($DIB_MIN_TMPFS * 2))
[ $total_kB -lt $(($RAM_NEEDED*1024*1024)) ] || return 0
if [ $echo_message == '1' ]; then
echo "WARNING: Not enough RAM to use tmpfs for build. Using ${TMP_DIR:-/tmp}. ($total_kB < ${RAM_NEEDED}G)"
fi
return 1
}
function mk_build_dir () {
TMP_BUILD_DIR=$(mktemp -t -d --tmpdir=${TMP_DIR:-/tmp} dib_build.XXXXXXXX)
TMP_IMAGE_DIR=$(mktemp -t -d --tmpdir=${TMP_DIR:-/tmp} dib_image.XXXXXXXX)
[ $? -eq 0 ] || die "Failed to create tmp directory"
export TMP_BUILD_DIR
if tmpfs_check ; then
sudo mount -t tmpfs tmpfs $TMP_BUILD_DIR
sudo mount -t tmpfs tmpfs $TMP_IMAGE_DIR
sudo chown $(id -u):$(id -g) $TMP_BUILD_DIR $TMP_IMAGE_DIR
fi
trap trap_cleanup EXIT
echo Building in $TMP_BUILD_DIR
export TMP_IMAGE_DIR
export OUT_IMAGE_PATH=$TMP_IMAGE_PATH
export TMP_HOOKS_PATH=$TMP_BUILD_DIR/hooks
}
function finish_image () {
if [ -f $1 -a ${OVERWRITE_OLD_IMAGE:-0} -eq 0 ]; then
old_image="${1%.*}"-$(date +%Y.%m.%d-%H.%M.%S).${1##*.}
echo "Old image found. Renaming it to $old_image"
mv "$1" "$old_image"
if [ -f "$1.md5" ]; then
mv "$1.md5" "$old_image.md5"
fi
if [ -f "$1.sha256" ]; then
mv "$1.sha256" "$old_image.sha256"
fi
fi
mv $OUT_IMAGE_PATH $1
if [ "$DIB_CHECKSUM" == "1" ]; then
# NOTE(pabelanger): Read image into memory once and generate both checksum
# files.
md5sum $1 > $1.md5 & sha256sum $1 > $1.sha256 & wait
fi
echo "Image file $1 created..."
}
function save_image () {
finish_image $1
}
function copy_hooks_not_overwrite () {
_DIR=$(basename $1)
test -d $TMP_HOOKS_PATH/$_DIR || mkdir $TMP_HOOKS_PATH/$_DIR
for _HOOK in $(ls $1); do
if [ ! -f $TMP_HOOKS_PATH/$_DIR/$_HOOK ]; then
echo "Copying hooks $1/$_HOOK"
cp -t $TMP_HOOKS_PATH/$_DIR -a $1/$_HOOK
else
echo "There is a duplicated hook in your elements: $_ELEMENT/$_DIR/$_HOOK"
exit 1
fi
done
}
function generate_hooks () {
local dir
local file
eval declare -A image_elements=($(get_image_element_array))
mkdir -p $TMP_HOOKS_PATH
for i in "${!image_elements[@]}"; do
local element=$i
local element_dir=${image_elements[$i]}
for dir in $(find $element_dir \
-follow -mindepth 1 -maxdepth 1 \
-type d -not -name tests); do
copy_hooks_not_overwrite $dir
done
for file in $(find $element_dir \
-follow -maxdepth 1 -type f); do
cp -t $TMP_HOOKS_PATH -a $file
done
done
}
# Call the supplied break-in routine if the named point is listed in the break
# list.
# $1 the break point.
# $2.. what to call if a break is needed
function check_break () {
if echo "${break:-}" | egrep -e "(,|^)$1(,|$)" -q; then
echo "Starting debug shell. Exit to resume building." >&2
echo At stage $1 >&2
shift
"$@"
echo "Resuming" >&2
fi
}
# Check that a real element has been chosen (prevents foot-guns)
function check_element () {
[ -d $TMP_HOOKS_PATH ] || generate_hooks
}
# Run a hook, looking for a regex in its stdout, and eval the matched lines.
# $1 is the hook to run
# $2 is the regex to look for
function eval_run_d () {
local run_output=$(mktemp)
trap "rm -f $run_output; check_break after-error ${break_cmd:-bash}" ERR
run_d $1 $run_output
if grep -q "$2" $run_output; then
local temp=$(grep "$2" $run_output)
eval "$temp"
fi
rm $run_output
trap - ERR
}
# Get any process that appears to be running in $TMP_BUILD_DIR
function _get_chroot_processes () {
# Deselect kernel threads, and use a python script to avoid
# forking lots and lots of readlink / grep processes on a busy
# system.
ps --ppid 2 -p 2 --deselect -o pid= | xargs python -c '
import os
import sys
for pid in sys.argv[2:]:
try:
root = os.readlink("/proc/%s/root" % pid)
except:
continue
if sys.argv[1] in root:
print("%s" % pid)
' $TMP_BUILD_DIR
}
function kill_chroot_processes () {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local pidname
if [ -z "${1}" ]; then
echo "ERROR: no chroot directory specified"
exit 1
fi
for pid in $(_get_chroot_processes); do
# If there are open files from the chroot, just kill the process using
# these files. This is racy, but good enough
pidname=$(cat $piddir/comm 2>/dev/null || echo "unknown")
echo "Killing chroot process: '${pidname}($pid)'"
sudo kill $pid
done
$xtrace
}
function cleanup_build_dir () {
if ! timeout 5 sh -c " while ! sudo rm -rf $TMP_BUILD_DIR/built; do sleep 1; done"; then
echo "ERROR: unable to cleanly remove $TMP_BUILD_DIR/built"
exit 1
fi
sudo rm -rf $TMP_BUILD_DIR/mnt
kill_chroot_processes $TMP_BUILD_DIR
if tmpfs_check 0; then
# If kill_chroot_processes did not succeed then we have to wait for
# init to reap the orphaned chroot processes
if ! timeout 120 sh -c "while ! sudo umount -f $TMP_BUILD_DIR; do sleep 1; done"; then
echo "ERROR: failed to umount the $TMP_BUILD_DIR tmpfs mount point"
exit 1
fi
fi
rm -rf --one-file-system $TMP_BUILD_DIR
}
function cleanup_image_dir () {
kill_chroot_processes $TMP_IMAGE_DIR
if tmpfs_check 0; then
if ! timeout 120 sh -c "while ! sudo umount -f $TMP_IMAGE_DIR; do sleep 1; done"; then
echo "ERROR: failed to umount the $TMP_IMAGE_DIR tmpfs mount point"
exit 1
fi
fi
rm -rf --one-file-system $TMP_IMAGE_DIR
}
# Run a directory of hooks outside the target (that is, no chrooting).
function run_d() {
check_element
check_break before-$1 ${break_cmd:-bash}
if [ -d ${TMP_HOOKS_PATH}/$1.d ] ; then
echo "Running hooks from ${TMP_HOOKS_PATH}/$1.d"
if [ -n "$2" ]; then
dib-run-parts ${TMP_HOOKS_PATH}/$1.d | tee $2
if [[ ${PIPESTATUS[0]} != 0 ]]; then
return 1
fi
else
dib-run-parts ${TMP_HOOKS_PATH}/$1.d
fi
fi
check_break after-$1 bash
}
function detach_loopback() {
local loopdev=$1
# Remove the map if it exists
# If setup on a rhel or derivative the map was created with kpartx not losetup
# and subsequently needs to be removed.
loopdev_name=$(echo $loopdev | sed 's/\/dev\///g')
if sudo dmsetup ls | grep $loopdev_name; then
mapper_name=$(sudo dmsetup ls | grep $loopdev_name | awk '{ print $1 }')
sudo dmsetup --noudevsync remove $mapper_name
fi
return 0
}
function arg_to_elements() {
for arg do IMAGE_ELEMENT="$IMAGE_ELEMENT $arg" ; done
if [ "$SKIP_BASE" != "1" ]; then
IMAGE_ELEMENT="base $IMAGE_ELEMENT"
fi
if [ "$IS_RAMDISK" == "1" ]; then
IMAGE_ELEMENT="$RAMDISK_ELEMENT $IMAGE_ELEMENT"
fi
echo "Building elements: $IMAGE_ELEMENT"
export IMAGE_ELEMENT
# element-info will output bash code to create
# * IMAGE_ELEMENT
# legacy list of elements
#
# * IMAGE_ELEMENT_YAML
# YAML dictionary with key=element, value=path
#
# import os
# import yaml
# yaml.load(os.getenv('IMAGE_ELEMENT_YAML')
#
# * function get_image_element_array
# Function to create Bash associative-array. Since bash can not
# export array variables, we provide a function to populate the
# variables.
#
# # we need the eval, it expands the string for the array create
# eval declare -A image_elements=($(get_image_element_array))
# for i in ${!image_elements[@]}; do
# element=$i
# path=${image_elements[$i]
# done
elinfo_out="$(element-info --env $IMAGE_ELEMENT)"
if [ $? -ne 0 ]; then
echo "ERROR: element-info failed to expand elements."
exit 1
fi
eval "$elinfo_out"
echo "Expanded element dependencies to: $IMAGE_ELEMENT"
}
function create_base () {
mkdir $TMP_BUILD_DIR/mnt
export TMP_MOUNT_PATH=$TMP_BUILD_DIR/mnt
# Copy data in to the root.
TARGET_ROOT=$TMP_MOUNT_PATH run_d root
Move /tmp/ccache setup to base element Commit c7d80dd (Cleanup mount points automatically) removed the unmount of $TMP_MOUNT_PATH/tmp/ccache in run_d_in_target() and moved the "rm /tmp/ccache" to elements/base/finalise.d/02-remove-ccache. There are two problems with this: 1) Not unmounting at the end of run_d_in_target() results in tmp/ccache being bind mounted muliple times on top of itself (three times, if you just run `disk-image-create base`). It is eventually unmounted, but somehow the auto unmount code is confused, and tries to unmount it one more time than it was mounted, which results in an error like "umount: /tmp/image.THQkZxQa/mnt/tmp/ccache: not mounted". This doesn't actually break anything, but it's a little messy. 2) "rm /tmp/ccache" in elements/base/finalise.d/02-remove-ccache never succeeds in removing /tmp/ccache, because that hook is invoked by run_d_in_target(), *while* /tmp/ccache is mounted. This present commit solves the above by moving the ccache setup glue out of img-functions and into the base element's root.d. This has the following implications: 1) lib/img-functions is a little cleaner. 2) /tmp/ccache is available in the chroot during the root, extra-data, pre-install, install and post-install stages. It is not available during block-device, finalise and cleanup stages as it will have been automatically unmounted by then. 3) /tmp/ccache won't be setup if you're building an image that doesn't include the base element. Change-Id: Ief4c0a6f4ec622db6c6f652776215684178d8943
2013-10-08 12:53:16 +00:00
if [ -z "$(ls $TMP_MOUNT_PATH | grep -v '^lost+found\|tmp$')" ] ; then
# No root element copied in. Note the test above allows
# root.d elements to put things in /tmp
echo "Failed to deploy the root element."
exit 1
fi
# Configure Image
# Setup resolv.conf so we can chroot to install some packages
if [ -L $TMP_MOUNT_PATH/etc/resolv.conf ] || [ -f $TMP_MOUNT_PATH/etc/resolv.conf ] ; then
sudo mv $TMP_MOUNT_PATH/etc/resolv.conf $TMP_MOUNT_PATH/etc/resolv.conf.ORIG
fi
# Recreate resolv.conf
sudo touch $TMP_MOUNT_PATH/etc/resolv.conf
sudo chmod 777 $TMP_MOUNT_PATH/etc/resolv.conf
# use system configured resolv.conf if available to support internal proxy resolving
if [ -e /etc/resolv.conf ]; then
cat /etc/resolv.conf > $TMP_MOUNT_PATH/etc/resolv.conf
else
echo nameserver 8.8.8.8 > $TMP_MOUNT_PATH/etc/resolv.conf
fi
mount_proc_dev_sys
}
function mount_proc_dev_sys () {
# supporting kernel file systems
sudo mount -t proc none $TMP_MOUNT_PATH/proc
sudo mount --bind /dev $TMP_MOUNT_PATH/dev
sudo mount --bind /dev/pts $TMP_MOUNT_PATH/dev/pts
sudo mount -t sysfs none $TMP_MOUNT_PATH/sys
}
# Recursively unmount directories under a given directory DIR
# usage:
# unmount_dir DIR
function unmount_dir {
local dir="$1"
local real_dir
local mnts
if [ ! -d $dir ]; then
echo "*** $dir is not a directory"
return 1
fi
# get rid of any symlink elements in the incoming path, because
# /proc/mounts is the real path
real_dir=$(readlink -e $dir)
mnts=$(awk '{print $2}' < /proc/mounts | grep "^$real_dir" | sort -r)
for m in $mnts; do
echo "Unmount $m"
sudo umount -fl $m || true
done
}