From 15960f01cdf476d5bee928b6f780d12f98ba9acd Mon Sep 17 00:00:00 2001 From: James Slagle Date: Tue, 21 Jan 2014 09:46:06 -0500 Subject: [PATCH] Add ability to use local cloud image Adds the ability to set $DIB_LOCAL_IMAGE to use as the base cloud image for a Fedora image build. There are many repetitve tasks that are done every image build. With this change you can build an image with the fedora and vm element and then reuse the resulting image as input to future image builds. This greatly reduces future image build times. For instance 99-up-to-date is already taking almost 2 minutes (even with rpm downloads already cached) for Fedora 20, and is only going to keep taking longer. By having a local up to date Fedora cloud image, this time can be saved on each image build. There is one minor change to support this in that /tmp/grub needs to get cleaned up at the end of image builds so that the image can be reused. Plus, there is no reason for it to stick around anyway. (didn't think this was worth a seperate commit). Change-Id: Ic74d138da922ecc99c38c27f105170d90009a84a --- elements/fedora/README.md | 5 ++ .../fedora/finalise.d/99-cleanup-tmp-grub | 5 ++ .../pre-install.d/15-fedora-remove-grub | 2 +- elements/fedora/root.d/10-fedora-cloud-image | 60 +++++++++++++------ 4 files changed, 52 insertions(+), 20 deletions(-) create mode 100755 elements/fedora/finalise.d/99-cleanup-tmp-grub diff --git a/elements/fedora/README.md b/elements/fedora/README.md index be2c0762..63a9af72 100644 --- a/elements/fedora/README.md +++ b/elements/fedora/README.md @@ -6,3 +6,8 @@ Overrides: use the environment variable DIB\_CLOUD\_IMAGES * To download a non-default release of Fedora cloud images, use the environment variable DIB\_RELEASE + * Alternatively, set DIB\_LOCAL\_IMAGE to the local path of a qcow2 cloud + image. This is useful in that you can use a customized or previously built + cloud image from diskimage-builder as input. The cloud image does not have + to have been build by diskimage-builder. It should be a full disk image, + not just a filesystem image. diff --git a/elements/fedora/finalise.d/99-cleanup-tmp-grub b/elements/fedora/finalise.d/99-cleanup-tmp-grub new file mode 100755 index 00000000..a2065bf2 --- /dev/null +++ b/elements/fedora/finalise.d/99-cleanup-tmp-grub @@ -0,0 +1,5 @@ +#!/bin/bash + +set -eux + +rm -rf /tmp/grub diff --git a/elements/fedora/pre-install.d/15-fedora-remove-grub b/elements/fedora/pre-install.d/15-fedora-remove-grub index 309bb92d..dd3e9830 100755 --- a/elements/fedora/pre-install.d/15-fedora-remove-grub +++ b/elements/fedora/pre-install.d/15-fedora-remove-grub @@ -13,7 +13,7 @@ find /tmp/yum/$basearch -regex ".*/grub2-[0-9].*\.rpm" -exec rm -f {} \; install-packages -d grub2 # Copy grub2 rpm out of mounted yum cache for install during finalise -mkdir /tmp/grub +mkdir -p /tmp/grub cp $(find /tmp/yum/$basearch -regex ".*/grub2-[0-9].*\.rpm") /tmp/grub echo "rpm -i /tmp/grub/*.rpm" > /tmp/grub/install diff --git a/elements/fedora/root.d/10-fedora-cloud-image b/elements/fedora/root.d/10-fedora-cloud-image index 5f0367d4..91a6e0ce 100755 --- a/elements/fedora/root.d/10-fedora-cloud-image +++ b/elements/fedora/root.d/10-fedora-cloud-image @@ -9,32 +9,50 @@ if [ 'amd64' = "$ARCH" ] ; then ARCH="x86_64" fi -DIB_RELEASE=${DIB_RELEASE:-20} -DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES:-http://cloud.fedoraproject.org} -BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-fedora-$DIB_RELEASE.$ARCH.qcow2} -BASE_IMAGE_TAR=$DIB_RELEASE-Cloud-$ARCH-$DIB_RELEASE.tgz +DIB_LOCAL_IMAGE=${DIB_LOCAL_IMAGE:-""} + +if [ -n "$DIB_LOCAL_IMAGE" ]; then + IMAGE_LOCATION=$DIB_LOCAL_IMAGE + # No need to copy a local image into the cache directory, so just specify + # the cached path as the original path. + CACHED_IMAGE=$IMAGE_LOCATION + BASE_IMAGE_FILE=`basename $DIB_LOCAL_IMAGE` + BASE_IMAGE_TAR=$BASE_IMAGE_FILE.tgz +else + DIB_RELEASE=${DIB_RELEASE:-20} + DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES:-http://cloud.fedoraproject.org} + BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-fedora-$DIB_RELEASE.$ARCH.qcow2} + BASE_IMAGE_TAR=$DIB_RELEASE-Cloud-$ARCH-$DIB_RELEASE.tgz + IMAGE_LOCATION=$DIB_CLOUD_IMAGES/$BASE_IMAGE_FILE + CACHED_IMAGE=$DIB_IMAGE_CACHE/$BASE_IMAGE_FILE + +fi + CACHED_TAR=$DIB_IMAGE_CACHE/$BASE_IMAGE_TAR if [ -n "$DIB_OFFLINE" -a -f "$CACHED_TAR" ] ; then echo "Not checking freshness of cached $CACHED_TAR." else - echo "Fetching Base Image" - # There seems to be some bad Fedora mirrors returning http 404's for the cloud image. - # If the image fails to download due to a 404 we retry once. - set +e - $TMP_HOOKS_PATH/bin/cache-url $DIB_CLOUD_IMAGES/$BASE_IMAGE_FILE $DIB_IMAGE_CACHE/$BASE_IMAGE_FILE - RV=$? - set -e + if [ -z "$DIB_LOCAL_IMAGE" ]; then + echo "Fetching Base Image" - if [ "$RV" == "44" ] ; then - $TMP_HOOKS_PATH/bin/cache-url $DIB_CLOUD_IMAGES/$BASE_IMAGE_FILE $DIB_IMAGE_CACHE/$BASE_IMAGE_FILE - elif [ "$RV" != "0" ] ; then - exit 1 + # There seems to be some bad Fedora mirrors returning http 404's for the cloud image. + # If the image fails to download due to a 404 we retry once. + set +e + $TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE + RV=$? + set -e + + if [ "$RV" == "44" ] ; then + $TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE + elif [ "$RV" != "0" ] ; then + exit 1 + fi fi if [ ! -f $CACHED_TAR -o \ - $DIB_IMAGE_CACHE/$BASE_IMAGE_FILE -nt $CACHED_TAR ] ; then + $CACHED_IMAGE -nt $CACHED_TAR ] ; then echo "Repacking base image as tarball." WORKING=$(mktemp -d) EACTION="rm -r $WORKING" @@ -42,7 +60,7 @@ else echo "Working in $WORKING" RAW_FILE=$(basename $BASE_IMAGE_FILE) RAW_FILE=${RAW_FILE#.qcow2}.raw - qemu-img convert -f qcow2 -O raw $DIB_IMAGE_CACHE/$BASE_IMAGE_FILE $WORKING/$RAW_FILE + qemu-img convert -f qcow2 -O raw $CACHED_IMAGE $WORKING/$RAW_FILE # WARNING: The mattdm image has the root filesystem on the second # partition (p2). If he changes the image the MAGIC_BIT # might also need to change. @@ -70,12 +88,16 @@ else trap "$EACTION" EXIT # Chroot in so that we get the correct uid/gid sudo chroot $WORKING/mnt bin/tar -cz . > $WORKING/tmp.tar - mv $WORKING/tmp.tar $DIB_IMAGE_CACHE/$BASE_IMAGE_TAR + mv $WORKING/tmp.tar $CACHED_TAR + else + echo "Using cached tar from $CACHED_TAR" fi + fi # Extract the base image (use --numeric-owner to avoid UID/GID mismatch between # image tarball and host OS e.g. when building Fedora image on an openSUSE host) -sudo tar -C $TARGET_ROOT --numeric-owner -xzf $DIB_IMAGE_CACHE/$BASE_IMAGE_TAR +echo "Extracting base root image from $CACHED_TAR" +sudo tar -C $TARGET_ROOT --numeric-owner -xzf $CACHED_TAR if [ -e "$TARGET_ROOT/lost+found" ]; then sudo rmdir $TARGET_ROOT/lost+found fi