Add ability to use local cloud image

Adds the ability to set $DIB_LOCAL_IMAGE to use as the base cloud image
for a Fedora image build. There are many repetitve tasks that are done
every image build. With this change you can build an image with the
fedora and vm element and then reuse the resulting image as input to
future image builds. This greatly reduces future image build times.

For instance 99-up-to-date is already taking almost 2 minutes (even with
rpm downloads already cached) for Fedora 20, and is only going to keep
taking longer. By having a local up to date Fedora cloud image, this
time can be saved on each image build.

There is one minor change to support this in that /tmp/grub needs to get
cleaned up at the end of image builds so that the image can be reused.
Plus, there is no reason for it to stick around anyway.  (didn't think
this was worth a seperate commit).

Change-Id: Ic74d138da922ecc99c38c27f105170d90009a84a

elements/fedora/README.md

@@ -6,3 +6,8 @@ Overrides:
    use the environment variable DIB\_CLOUD\_IMAGES
  * To download a non-default release of Fedora cloud images, use the
    environment variable DIB\_RELEASE
+ * Alternatively, set DIB\_LOCAL\_IMAGE to the local path of a qcow2 cloud
+   image. This is useful in that you can use a customized or previously built
+   cloud image from diskimage-builder as input. The cloud image does not have
+   to have been build by diskimage-builder. It should be a full disk image,
+   not just a filesystem image.

elements/fedora/finalise.d/99-cleanup-tmp-grub

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -eux
+
+rm -rf /tmp/grub

elements/fedora/pre-install.d/15-fedora-remove-grub

@@ -13,7 +13,7 @@ find /tmp/yum/$basearch -regex ".*/grub2-[0-9].*\.rpm" -exec rm -f {} \;
 install-packages -d grub2
 
 # Copy grub2 rpm out of mounted yum cache for install during finalise
-mkdir /tmp/grub
+mkdir -p /tmp/grub
 cp $(find /tmp/yum/$basearch -regex ".*/grub2-[0-9].*\.rpm") /tmp/grub
 echo "rpm -i /tmp/grub/*.rpm" > /tmp/grub/install
 

elements/fedora/root.d/10-fedora-cloud-image

@@ -9,32 +9,50 @@ if [ 'amd64' = "$ARCH" ] ; then
   ARCH="x86_64"
 fi
 
-DIB_RELEASE=${DIB_RELEASE:-20}
-DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES:-http://cloud.fedoraproject.org}
-BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-fedora-$DIB_RELEASE.$ARCH.qcow2}
-BASE_IMAGE_TAR=$DIB_RELEASE-Cloud-$ARCH-$DIB_RELEASE.tgz
+DIB_LOCAL_IMAGE=${DIB_LOCAL_IMAGE:-""}
+
+if [ -n "$DIB_LOCAL_IMAGE" ]; then
+    IMAGE_LOCATION=$DIB_LOCAL_IMAGE
+    # No need to copy a local image into the cache directory, so just specify
+    # the cached path as the original path.
+    CACHED_IMAGE=$IMAGE_LOCATION
+    BASE_IMAGE_FILE=`basename $DIB_LOCAL_IMAGE`
+    BASE_IMAGE_TAR=$BASE_IMAGE_FILE.tgz
+else
+    DIB_RELEASE=${DIB_RELEASE:-20}
+    DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES:-http://cloud.fedoraproject.org}
+    BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-fedora-$DIB_RELEASE.$ARCH.qcow2}
+    BASE_IMAGE_TAR=$DIB_RELEASE-Cloud-$ARCH-$DIB_RELEASE.tgz
+    IMAGE_LOCATION=$DIB_CLOUD_IMAGES/$BASE_IMAGE_FILE
+    CACHED_IMAGE=$DIB_IMAGE_CACHE/$BASE_IMAGE_FILE
+
+fi
+
 CACHED_TAR=$DIB_IMAGE_CACHE/$BASE_IMAGE_TAR
 
 if [ -n "$DIB_OFFLINE" -a -f "$CACHED_TAR" ] ; then
     echo "Not checking freshness of cached $CACHED_TAR."
 else
-    echo "Fetching Base Image"
 
-    # There seems to be some bad Fedora mirrors returning http 404's for the cloud image.
-    # If the image fails to download due to a 404 we retry once.
-    set +e
-    $TMP_HOOKS_PATH/bin/cache-url $DIB_CLOUD_IMAGES/$BASE_IMAGE_FILE $DIB_IMAGE_CACHE/$BASE_IMAGE_FILE
-    RV=$?
-    set -e
+    if [ -z "$DIB_LOCAL_IMAGE" ]; then
+        echo "Fetching Base Image"
 
-    if [ "$RV" == "44" ] ; then
-        $TMP_HOOKS_PATH/bin/cache-url $DIB_CLOUD_IMAGES/$BASE_IMAGE_FILE $DIB_IMAGE_CACHE/$BASE_IMAGE_FILE
-    elif [ "$RV" != "0" ] ; then
-        exit 1
+        # There seems to be some bad Fedora mirrors returning http 404's for the cloud image.
+        # If the image fails to download due to a 404 we retry once.
+        set +e
+        $TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE
+        RV=$?
+        set -e
+
+        if [ "$RV" == "44" ] ; then
+            $TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE
+        elif [ "$RV" != "0" ] ; then
+            exit 1
+        fi
     fi
 
     if [ ! -f $CACHED_TAR -o \
-        $DIB_IMAGE_CACHE/$BASE_IMAGE_FILE -nt $CACHED_TAR ] ; then
+        $CACHED_IMAGE -nt $CACHED_TAR ] ; then
         echo "Repacking base image as tarball."
         WORKING=$(mktemp -d)
         EACTION="rm -r $WORKING"
@@ -42,7 +60,7 @@ else
         echo "Working in $WORKING"
         RAW_FILE=$(basename $BASE_IMAGE_FILE)
         RAW_FILE=${RAW_FILE#.qcow2}.raw
-        qemu-img convert -f qcow2 -O raw $DIB_IMAGE_CACHE/$BASE_IMAGE_FILE $WORKING/$RAW_FILE
+        qemu-img convert -f qcow2 -O raw $CACHED_IMAGE $WORKING/$RAW_FILE
         # WARNING: The mattdm image has the root filesystem on the second
         #          partition (p2). If he changes the image the MAGIC_BIT
         #          might also need to change.
@@ -70,12 +88,16 @@ else
         trap "$EACTION" EXIT
         # Chroot in so that we get the correct uid/gid
         sudo chroot $WORKING/mnt bin/tar -cz . > $WORKING/tmp.tar
-        mv $WORKING/tmp.tar $DIB_IMAGE_CACHE/$BASE_IMAGE_TAR
+        mv $WORKING/tmp.tar $CACHED_TAR
+    else
+        echo "Using cached tar from $CACHED_TAR"
     fi
+
 fi
 # Extract the base image (use --numeric-owner to avoid UID/GID mismatch between
 # image tarball and host OS e.g. when building Fedora image on an openSUSE host)
-sudo tar -C $TARGET_ROOT --numeric-owner -xzf $DIB_IMAGE_CACHE/$BASE_IMAGE_TAR
+echo "Extracting base root image from $CACHED_TAR"
+sudo tar -C $TARGET_ROOT --numeric-owner -xzf $CACHED_TAR
 if [ -e "$TARGET_ROOT/lost+found" ]; then
   sudo rmdir $TARGET_ROOT/lost+found
 fi