#!/bin/bash # # Copyright 2012 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. set -eE # Set/override locale. This ensures consistency in sorting etc. We # need to choose a lowest-common denominator locale, as this is # applied when running in the building chroot too (maybe a bug and we # should prune this?). Thus "C" --centOS 7 doesn't include C.utf-8 # (fedora does, centos 8 probably will). Note: LC_ALL to really # override this; it overrides LANG and all other LC_ vars export LC_ALL=C # Store our initial environment and command line args for later export DIB_ARGS="$@" export DIB_ENV=$(export | grep ' DIB_.*=') SCRIPTNAME=$(basename $0) if [ -z "$_LIB" ]; then echo "_LIB not set!" exit 1 fi _BASE_ELEMENT_DIR=$(${DIB_PYTHON_EXEC:-python} -c ' import diskimage_builder.paths diskimage_builder.paths.show_path("elements")') source $_LIB/die IS_RAMDISK=0 if [ "$SCRIPTNAME" == "ramdisk-image-create" ]; then IS_RAMDISK=1 fi function show_options () { echo "Usage: ${SCRIPTNAME} [OPTION]... [ELEMENT]..." echo echo "Options:" echo " -a i386|amd64|armhf|arm64 -- set the architecture of the image(default amd64)" echo " -o imagename -- set the imagename of the output image file(default image)" echo " -t qcow2,tar,tgz,squashfs,vhd,docker,aci,raw -- set the image types of the output image files (default qcow2)" echo " File types should be comma separated. VHD outputting requires the vhd-util" echo " executable be in your PATH. ACI outputting requires the ACI_MANIFEST " echo " environment variable be a path to a manifest file." echo " -x -- turn on tracing (use -x -x for very detailed tracing)." echo " -u -- uncompressed; do not compress the image - larger but faster" echo " -c -- clear environment before starting work" echo " --logfile -- save run output to given logfile (implies DIB_QUIET=1)" echo " --checksum -- generate MD5 and SHA256 checksum files for the created image" echo " --image-size size -- image size in GB for the created image" echo " --image-extra-size size -- extra image size in GB for the created image" echo " --image-cache directory -- location for cached images(default ~/.cache/image-create)" echo " --max-online-resize size -- max number of filesystem blocks to support when resizing." echo " Useful if you want a really large root partition when the image is deployed." echo " Using a very large value may run into a known bug in resize2fs." echo " Setting the value to 274877906944 will get you a 1PB root file system." echo " Making this value unnecessarily large will consume extra disk space " echo " on the root partition with extra file system inodes." echo " --min-tmpfs size -- minimum size in GB needed in tmpfs to build the image" echo " --mkfs-journal-size -- filesystem journal size in MB to pass to mkfs." echo " --mkfs-options -- option flags to be passed directly to mkfs." echo " Options should be passed as a single string value." echo " --no-tmpfs -- do not use tmpfs to speed image build" echo " --offline -- do not update cached resources" echo " --qemu-img-options -- option flags to be passed directly to qemu-img." echo " Options need to be comma separated, and follow the key=value pattern." echo " --root-label label -- label for the root filesystem. Defaults to 'cloudimg-rootfs'." echo " --ramdisk-element -- specify the main element to be used for building ramdisks." echo " Defaults to 'ramdisk'. Should be set to 'dracut-ramdisk' for platforms such" echo " as RHEL and CentOS that do not package busybox." echo " --install-type -- specify the default installation type. Defaults to 'source'. Set to 'package' to use package based installations by default." echo " --docker-target -- specify the repo and tag to use if the output type is docker. Defaults to the value of output imagename" if [ "$IS_RAMDISK" == "0" ]; then echo " -n skip the default inclusion of the 'base' element" echo " -p package[,p2...] [-p p3] -- extra packages to install in the image. Runs once, after 'install.d' phase. Can be specified multiple times" fi echo " -h|--help -- display this help and exit" echo " --version -- display version and exit" echo echo "Environment Variables:" echo " (this is not a complete list)" echo echo " * ELEMENTS_PATH: specify external locations for the elements. As for \$PATH" echo " * DIB_NO_TIMESTAMP: no timestamp prefix on output. Useful if capturing output" echo " * DIB_QUIET: 1=do not output log output to stdout; 0=always ouptut to stdout. See --logfile" echo echo "NOTE: At least one distribution root element must be specified." echo echo "NOTE: If using the VHD output format you need to have a patched version of vhd-util installed for the image" echo " to be bootable. The patch is available here: https://github.com/emonty/vhd-util/blob/master/debian/patches/citrix" echo " and a PPA with the patched tool is available here: https://launchpad.net/~openstack-ci-core/+archive/ubuntu/vhd-util" echo echo "Examples:" if [ "$IS_RAMDISK" == "0" ]; then echo " ${SCRIPTNAME} -a amd64 -o ubuntu-amd64 vm ubuntu" echo " export ELEMENTS_PATH=~/source/tripleo-image-elements/elements" echo " ${SCRIPTNAME} -a amd64 -o fedora-amd64-heat-cfntools vm fedora heat-cfntools" else echo " ${SCRIPTNAME} -a amd64 -o fedora-deploy deploy fedora" echo " ${SCRIPTNAME} -a amd64 -o ubuntu-ramdisk ramdisk ubuntu" fi } function show_version() { ${DIB_PYTHON_EXEC:-python} -c "from diskimage_builder import version; print(version.version_info.version_string())" } DIB_DEBUG_TRACE=${DIB_DEBUG_TRACE:-0} INSTALL_PACKAGES="" IMAGE_TYPES=("qcow2") COMPRESS_IMAGE="true" ROOT_LABEL="" DIB_DEFAULT_INSTALLTYPE=${DIB_DEFAULT_INSTALLTYPE:-"source"} MKFS_OPTS="" ACI_MANIFEST=${ACI_MANIFEST:-} DOCKER_TARGET="" LOGFILE="" TEMP=`getopt -o a:ho:t:xucnp: -l checksum,no-tmpfs,offline,help,version,min-tmpfs:,image-size:,image-cache:,max-online-resize:,mkfs-options:,qemu-img-options:,ramdisk-element:,root-label:,install-type:,docker-target:,logfile: -n $SCRIPTNAME -- "$@"` if [ $? -ne 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi # Note the quotes around `$TEMP': they are essential! eval set -- "$TEMP" while true ; do case "$1" in -a) export ARCH=$2; shift 2 ;; -o) export IMAGE_NAME=$2; shift 2 ;; -t) IFS="," read -a IMAGE_TYPES <<< "$2"; export IMAGE_TYPES ; shift 2 ;; -h|--help) show_options; exit 0;; --version) show_version; exit 0;; -x) shift; DIB_DEBUG_TRACE=$(( $DIB_DEBUG_TRACE + 1 ));; -u) shift; export COMPRESS_IMAGE="";; -c) shift ; export CLEAR_ENV=1;; -n) shift; export SKIP_BASE="1";; -p) IFS="," read -a _INSTALL_PACKAGES <<< "$2"; export INSTALL_PACKAGES=( ${INSTALL_PACKAGES[@]} ${_INSTALL_PACKAGES[@]} ) ; shift 2 ;; --checksum) shift; export DIB_CHECKSUM=1;; --image-size) export DIB_IMAGE_SIZE=$2; shift 2;; --image-extra-size) export DIB_IMAGE_EXTRA_SIZE=$2; shift 2;; --image-cache) export DIB_IMAGE_CACHE=$2; shift 2;; --max-online-resize) export MAX_ONLINE_RESIZE=$2; shift 2;; --mkfs-journal-size) export DIB_JOURNAL_SIZE=$2; shift 2;; --mkfs-options) MKFS_OPTS=$2; shift 2;; --min-tmpfs) export DIB_MIN_TMPFS=$2; shift 2;; --no-tmpfs) shift; export DIB_NO_TMPFS=1;; --offline) shift; export DIB_OFFLINE=1;; --qemu-img-options) QEMU_IMG_OPTIONS=$2; shift 2;; --root-label) ROOT_LABEL=$2; shift 2;; --ramdisk-element) RAMDISK_ELEMENT=$2; shift 2;; --install-type) DIB_DEFAULT_INSTALLTYPE=$2; shift 2;; --docker-target) export DOCKER_TARGET=$2; shift 2 ;; --logfile) export LOGFILE=$2; shift 2 ;; --) shift ; break ;; *) echo "Internal error!" ; exit 1 ;; esac done export DIB_DEBUG_TRACE # TODO: namespace this under ~/.cache/dib/ for consistency export DIB_IMAGE_CACHE=${DIB_IMAGE_CACHE:-~/.cache/image-create} mkdir -p $DIB_IMAGE_CACHE # We have a couple of critical sections (touching parts of the host # system or download images to common cache) that we use flock around. # Use this directory for lockfiles. export DIB_LOCKFILES=${DIB_LOCKFILES:-~/.cache/dib/lockfiles} mkdir -p $DIB_LOCKFILES if [ "$CLEAR_ENV" = "1" -a "$HOME" != "" ]; then echo "Re-execing to clear environment." echo "(note this will prevent much of the local_config element from working)" exec -c $0 "$@" fi # We send stdout & stderr through "outfilter" which does timestamping, # basic filtering and log file output. _TS_FLAG="" if [[ "${DIB_NO_TIMESTAMP:-0}" -eq 1 ]]; then _TS_FLAG="--no-timestamp" fi # A logfile with *no* DIB_QUIET specified implies we just want output # to the logfile. Explicitly setting DIB_QUIET=0 will overide this # and log both. if [[ -n "${LOGFILE}" && -z "${DIB_QUIET}" ]]; then DIB_QUIET=1 fi _QUIET_FLAG="-v" if [[ "${DIB_QUIET:-0}" -eq 1 ]]; then _QUIET_FLAG="" fi _LOGFILE_FLAG="" if [[ -n "${LOGFILE}" ]]; then echo "Output logs going to: ${LOGFILE}" _LOGFILE_FLAG="-o ${LOGFILE}" fi # Save the existing stdout to fd3 exec 3>&1 exec 1> >( ${DIB_PYTHON_EXEC:-python} $_LIB/outfilter.py ${_TS_FLAG} ${_QUIET_FLAG} ${_LOGFILE_FLAG} ) 2>&1 # Display the current file/function/line in the debug output function _ps4 { IFS=" " called=($(caller 0)) local f=$(readlink -f ${called[2]}) # As we're being run out of the python package's lib/ dir (either # virtualenv or system), we can strip everything before # "site-packages" to significantly shorten the line without really f=${f##*site-packages/} printf "%-80s " "$f:${called[1]}:${called[0]}" } export -f _ps4 export PS4='+ $(_ps4): ' source $_LIB/img-defaults source $_LIB/common-functions source $_LIB/img-functions if [ "$IS_RAMDISK" == "1" ]; then source $_LIB/ramdisk-defaults source $_LIB/ramdisk-functions fi echo "diskimage-builder version $(show_version)" # If no elements are specified theres no way we can succeed if [ -z "$*" ]; then echo "ERROR: At least one distribution root element must be specified" exit 1 fi arg_to_elements "$@" # start tracing after most boilerplate if [ ${DIB_DEBUG_TRACE} -gt 0 ]; then set -x fi if [ "${#IMAGE_TYPES[@]}" = "1" ]; then export IMAGE_NAME=${IMAGE_NAME%%\.${IMAGE_TYPES[0]}} fi # Check for required tools early on for X in ${!IMAGE_TYPES[@]}; do case "${IMAGE_TYPES[$X]}" in qcow2) if ! type qemu-img > /dev/null 2>&1; then echo "qcow2 output format specified but qemu-img executable not found." exit 1 fi ;; tgz) # Force tar to be created. IMAGE_TYPES+=('tar') ;; vhd) if ! type vhd-util > /dev/null 2>&1; then echo "vhd output format specified but no vhd-util executable found." exit 1 fi ;; squashfs) if ! type mksquashfs > /dev/null 2>&1; then echo "squashfs output format specified but no mksquashfs executable found." exit 1 fi ;; docker) if ! type docker > /dev/null 2>&1; then echo "docker output format specified but no docker executable found." exit 1 fi if [ -z "$DOCKER_TARGET" ]; then echo "Please set --docker-target." exit 1 fi ;; esac done # NOTE: fstrim is on most all recent systems. It is provided by the util-linux # package. if ! type fstrim > /dev/null 2>&1; then echo "fstrim utility is not found. This is provided by util-linux package" echo "Please check your PATH variable is set correctly" exit 1 fi # xattr support cannot be relied upon with tmpfs builds # some kernels supoprt it, some don't if [[ -n "${GENTOO_PROFILE}" ]]; then if [[ "${GENTOO_PROFILE}" =~ "hardened" ]]; then echo 'disabling tmpfs for gentoo hardened build' export DIB_NO_TMPFS=1 fi fi mk_build_dir # Create the YAML file with the final and raw configuration for # the block device layer. mkdir -p ${TMP_BUILD_DIR}/block-device BLOCK_DEVICE_CONFIG_YAML=${TMP_BUILD_DIR}/block-device/config.yaml block_device_create_config_file "${BLOCK_DEVICE_CONFIG_YAML}" # Write out the parameter file DIB_BLOCK_DEVICE_PARAMS_YAML=${TMP_BUILD_DIR}/block-device/params.yaml export DIB_BLOCK_DEVICE_PARAMS_YAML cat >${DIB_BLOCK_DEVICE_PARAMS_YAML} < $TMP_HOOKS_PATH/environment.d/11-dib-install-type.bash run_d extra-data # Run pre-install scripts. These do things that prepare the chroot for package installs run_d_in_target pre-install # Call install scripts to pull in the software users want. run_d_in_target install do_extra_package_install run_d_in_target post-install run_d post-root # ensure we do not have a lost+found directory in the root folder # that could cause copy to fail (it will be created again later # when creating the file system, if it needs such directory) if [ -e "$TMP_BUILD_DIR/mnt/lost+found" ]; then sudo rm -rf "$TMP_BUILD_DIR/mnt/lost+found" fi # Free up /mnt unmount_image mv $TMP_BUILD_DIR/mnt $TMP_BUILD_DIR/built # save xtrace state, as we always want to turn it off to avoid # spamming the logs with du output below. xtrace=$(set +o | grep xtrace) # temp file for holding du output du_output=${TMP_BUILD_DIR}/du_output.tmp if [ -n "$DIB_IMAGE_SIZE" ]; then du_size=$(echo "$DIB_IMAGE_SIZE" | awk '{printf("%d\n",$1 * 1024 *1024)}') else set +o xtrace echo "Calculating image size (this may take a minute)..." sudo du -a -c -x ${TMP_BUILD_DIR}/built > ${du_output} # the last line is the total size from "-c". if [ -n "$DIB_IMAGE_EXTRA_SIZE" ]; then # add DIB_IMAGE_EXTRA_SIZE megabytes to create a bigger image as requested du_extra_size=$(echo "$DIB_IMAGE_EXTRA_SIZE" | awk '{printf("%d\n",$1 * 1024)}') du_size_tmp=$(tail -n1 ${du_output} | cut -f1) du_size=$(echo "$du_size_tmp $du_extra_size" | awk '{print int($1 + $2)}') else # scale this by 0.6 to create a slightly bigger image du_size=$(tail -n1 ${du_output} | cut -f1 | awk '{print int($1 / 0.6)}') fi $xtrace fi if [[ "${DIB_SHOW_IMAGE_USAGE:-0}" != 0 ]]; then set +o xtrace if [ ! -f "$du_output" ]; then sudo du -a -c -x ${TMP_BUILD_DIR}/built > ${du_output} fi du_output_show="sort -nr ${du_output} | numfmt --to=iec-i --padding=7 --suffix=B --field=1 --from-unit=1024" # by default show the 10MiB and greater files & directories -- a # dir with lots of little files will still show up, but this helps # signal:noise ratio if [[ ${DIB_SHOW_IMAGE_USAGE_FULL:-0} == 0 ]]; then # numfmt will start giving a decimal place when < 10MiB du_output_show+="| egrep 'MiB|GiB|TiB|PiB' | grep -v '\..MiB'" echo "=================================" echo "Image size report (files > 10MiB)" echo "=================================" else echo "=================" echo "Image size report" echo "=================" fi eval ${du_output_show} echo echo "===== end image size report =====" echo $xtrace fi rm -f ${du_output} if [ -n "$DIB_JOURNAL_SIZE" ]; then journal_size="$DIB_JOURNAL_SIZE" else journal_size=64 fi if [ "$DIB_ROOT_FSTYPE" = "ext4" ] ; then # Very conservative to handle images being resized a lot # We set journal size to 64M so our journal is large enough when we # perform an FS resize. MKFS_OPTS="-i 4096 -J size=$journal_size $MKFS_OPTS" # Grow the image size to account for the journal, only if the user # has not asked for a specific size. if [ -z "$DIB_IMAGE_SIZE" ]; then du_size=$(( $du_size + ($journal_size * 1024) )) fi fi # EFI system partitions default to be quite large at 512mb for maximum # compatability (see notes in # 7fd52ba84180b4e749ccf4c9db8c49eafff46ea8) . We need to increase the # total size to account for this, or we run out of space creating the # final image. See if we have included the block-device-efi element, # which implies we have a large EFI partition, and then pad the final # image size. if [[ ${IMAGE_ELEMENT} =~ "block-device-efi" ]]; then echo "Expanding disk for EFI partition" du_size=$(( $du_size + (525 * 1024) )) fi # Rounding down size so that is is a multiple of 64, works around a bug in # qemu-img that may occur when compressing raw images that aren't a multiple # of 64k. https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1180021 export DIB_IMAGE_SIZE=$(echo "$du_size" | awk ' { if ($1 % 64 != 0) print $1 + 64 - ( $1 % 64); else print $1; } ') if [ -n "$MAX_ONLINE_RESIZE" ]; then MKFS_OPTS="-E resize=$MAX_ONLINE_RESIZE $MKFS_OPTS" fi export TMP_IMAGE_DIR # Try the 'old fashioned' way calling the block device # phase. If this gives no result, use the configuration based approach: eval_run_d block-device "IMAGE_BLOCK_DEVICE=" if [ -z ${IMAGE_BLOCK_DEVICE} ] ; then # For compatibily reasons in addition to the YAML configuration # there is the need to handle the old environment variables. echo "image-size: ${DIB_IMAGE_SIZE}KiB" >> ${DIB_BLOCK_DEVICE_PARAMS_YAML} if [ -n "${MKFS_OPTS}" ] ; then echo "root-fs-opts: '${MKFS_OPTS}'" >> ${DIB_BLOCK_DEVICE_PARAMS_YAML} fi # After changeing the parameters, there is the need to # re-run dib-block-device init because some value might # change based on the new set parameters. dib-block-device init # values to dib-block-device: using the YAML config and dib-block-device create # This is the device (/dev/loopX). It's where to install the # bootloader. IMAGE_BLOCK_DEVICE=$(dib-block-device getval image-block-device) export IMAGE_BLOCK_DEVICE # Similar to above, but all mounted devices. This is handy for # some bootloaders that have multi-partition layouts and want to # copy things to different places other than just # IMAGE_BLOCK_DEVICE. "eval" this into an array as needed IMAGE_BLOCK_DEVICES=$(dib-block-device getval image-block-devices) export IMAGE_BLOCK_DEVICES # Write the fstab dib-block-device writefstab fi # XXX: needed? LOOPDEV=${IMAGE_BLOCK_DEVICE} # At this point, dib-block-device has created the raw image file # (IMAGE_BLOCK_DEVICE) and mounted all the partitions under # $TMP_BUILD_DIR/mnt for us. We can now copy into the final image. # 'mv' is not usable here - especially when a top level directory # has the same name as a mount point of a partition. If so, 'mv' # will complain: # mv: inter-device move failed: '...' to '...'; \ # unable to remove target: Device or resource busy # therefore a 'cp' and 'rm' approach is used. sudo cp -ra ${TMP_BUILD_DIR}/built/* $TMP_BUILD_DIR/mnt sudo rm -fr ${TMP_BUILD_DIR}/built/* mount_proc_dev_sys run_d pre-finalise run_d_in_target finalise finalise_base for X in ${!IMAGE_TYPES[@]} ; do if [[ " tar aci " =~ "${IMAGE_TYPES[$X]}" ]]; then if [ "${IMAGE_TYPES[$X]}" = "aci" ]; then sudo tar -C ${TMP_BUILD_DIR}/mnt -cf $IMAGE_NAME.aci --exclude ./sys \ --exclude ./proc --xattrs --xattrs-include=\* \ --transform 's,^.,rootfs,S' . if [ -n "$ACI_MANIFEST" ]; then cp $ACI_MANIFEST ${TMP_BUILD_DIR}/manifest sudo tar -C ${TMP_BUILD_DIR} --append -f $IMAGE_NAME.aci manifest else echo "No ACI_MANIFEST specified. An ACI_MANIFEST must be specified for" echo " this image to be usable." fi else sudo tar -C ${TMP_BUILD_DIR}/mnt -cf $IMAGE_NAME.tar --exclude ./sys \ --exclude ./proc --xattrs --xattrs-include=\* . fi sudo chown $USER: $IMAGE_NAME.${IMAGE_TYPES[$X]} unset IMAGE_TYPES[$X] elif [ "${IMAGE_TYPES[$x]}" == "squashfs" ]; then sudo mksquashfs ${TMP_BUILD_DIR}/mnt $IMAGE_NAME.squash -comp xz \ -noappend -root-becomes ${TMP_BUILD_DIR}/mnt \ -wildcards -e "proc/*" -e "sys/*" -no-recovery elif [ "${IMAGE_TYPES[$X]}" == "docker" ]; then sudo tar -C ${TMP_BUILD_DIR}/mnt -cf - --exclude ./sys \ --exclude ./proc --xattrs --xattrs-include=\* . \ | sudo docker import - $DOCKER_TARGET unset IMAGE_TYPES[$X] fi done # Unmount and cleanup the /mnt and /build subdirectories, to save # space before converting the image to some other format. # XXX ? needed? export EXTRA_UNMOUNT="" unmount_image TMP_IMAGE_PATH=$(dib-block-device getval image-path) export TMP_IMAGE_PATH # remove all mounts dib-block-device umount dib-block-device cleanup cleanup_build_dir if [[ (! $IMAGE_ELEMENT =~ no-final-image) && "$IS_RAMDISK" == "0" ]]; then has_raw_type= for IMAGE_TYPE in ${IMAGE_TYPES[@]} ; do # We have to do raw last because it is destructive if [ "$IMAGE_TYPE" = "raw" ]; then has_raw_type=1 elif [ "$IMAGE_TYPE" != "squashfs" ]; then compress_and_save_image $IMAGE_NAME.$IMAGE_TYPE fi done if [ -n "$has_raw_type" ]; then IMAGE_TYPE="raw" compress_and_save_image $IMAGE_NAME.$IMAGE_TYPE fi fi # Remove the leftovers, i.e. the temporary image directory. cleanup_image_dir # Restore fd 1&2 from the outfilter.py redirect back to the original # saved fd. Note small hack that we can't really wait properly for # outfilter.py so put in a sleep (might be possible to use coproc for # this...?) # # TODO(ianw): probably better to cleanup the exit handler a bit for # this? We really want some helper functions that append to the exit # handler so we can register multiple things. set +o xtrace echo "Build completed successfully" exec 1>&3 2>&3 sleep 1 # All done! trap EXIT