From ed833e628ff85d767cf21e5c5935eba06d9be3db Mon Sep 17 00:00:00 2001 From: Jon-Paul Sullivan Date: Tue, 22 Jul 2014 19:52:33 +0100 Subject: [PATCH] Lock around extract-image downloads and extracts This wil be needed for parallel image builds. Change-Id: I6ba3a10b44fe37f8d8903ad06302a672c3e972c8 --- elements/redhat-common/bin/extract-image | 166 ++++++++++++----------- 1 file changed, 89 insertions(+), 77 deletions(-) diff --git a/elements/redhat-common/bin/extract-image b/elements/redhat-common/bin/extract-image index de74915e..e394cc3a 100755 --- a/elements/redhat-common/bin/extract-image +++ b/elements/redhat-common/bin/extract-image @@ -14,84 +14,96 @@ CACHED_IMAGE=$4 CACHED_TAR=$DIB_IMAGE_CACHE/$BASE_IMAGE_TAR DIB_LOCAL_IMAGE=${DIB_LOCAL_IMAGE:-""} +TAR_LOCK=$CACHED_TAR.lock -if [ -n "$DIB_OFFLINE" -a -f "$CACHED_TAR" ] ; then - echo "Not checking freshness of cached $CACHED_TAR." -else - - if [ -z "$DIB_LOCAL_IMAGE" ]; then - echo "Fetching Base Image" - - # There seems to be some bad Fedora mirrors returning http 404's for the cloud image. - # If the image fails to download due to a 404 we retry once. - set +e - $TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE - RV=$? - set -e - - if [ "$RV" == "44" ] ; then - $TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE - elif [ "$RV" != "0" ] ; then - exit 1 - fi - fi - - if [ ! -f $CACHED_TAR -o \ - $CACHED_IMAGE -nt $CACHED_TAR ] ; then - echo "Repacking base image as tarball." - WORKING=$(mktemp -d) - EACTION="rm -r $WORKING" - trap "$EACTION" EXIT - echo "Working in $WORKING" - RAW_FILE=$(basename $BASE_IMAGE_FILE) - RAW_FILE=${RAW_FILE#.qcow2}.raw - qemu-img convert -f qcow2 -O raw $CACHED_IMAGE $WORKING/$RAW_FILE - - # F19 images have the rootfs partition on p1 - # Centos7 images on p3 (p1 boot, p2 swap) - if [[ $DISTRO_NAME = "centos7" ]]; then - ROOT_PARTITON=p3 - else - ROOT_PARTITON=p1 - fi - - # kpartx fails if no /dev/loop* exists, "losetup -f" prints first unused - # loop device and creates it if it doesn't exist - sudo losetup -f - - # XXX: Parsing stdout is dangerous, would like a better way to discover - # the device used for the image. - ROOT_LOOPDEV=$(sudo kpartx -av $WORKING/$RAW_FILE | \ - awk "/loop[0-9]+$ROOT_PARTITON/ {print \$3}") - if ! timeout 5 sh -c "while ! [ -e /dev/mapper/$ROOT_LOOPDEV ]; do sleep 1; done"; then - echo "Error: Could not find /dev/mapper/$ROOT_LOOPDEV" - exit 1 - fi - EACTION="sudo kpartx -d $WORKING/$RAW_FILE ; $EACTION" - trap "$EACTION" EXIT - mkdir $WORKING/mnt - sudo mount /dev/mapper/$ROOT_LOOPDEV $WORKING/mnt - EACTION="sudo umount -f $WORKING/mnt ; $EACTION" - trap "$EACTION" EXIT - - # need to copy the contents of /boot into the image too, so - # mount it - if [[ $DISTRO_NAME = "centos7" ]]; then - BOOT_LOOPDEV=${ROOT_LOOPDEV/p3/p1} - sudo mount /dev/mapper/$BOOT_LOOPDEV $WORKING/mnt/boot - EACTION="sudo umount -f $WORKING/mnt/boot ; $EACTION" - trap "$EACTION" EXIT - fi - - # Chroot in so that we get the correct uid/gid - sudo chroot $WORKING/mnt bin/tar -cz . > $WORKING/tmp.tar - mv $WORKING/tmp.tar $CACHED_TAR +function extract_image() { + if [ -n "$DIB_OFFLINE" -a -f "$CACHED_TAR" ] ; then + echo "Not checking freshness of cached $CACHED_TAR." else - echo "Using cached tar from $CACHED_TAR" + if [ -z "$DIB_LOCAL_IMAGE" ]; then + echo "Fetching Base Image" + + # There seems to be some bad Fedora mirrors returning http 404's for the cloud image. + # If the image fails to download due to a 404 we retry once. + set +e + $TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE + RV=$? + set -e + + if [ "$RV" == "44" ] ; then + $TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE + elif [ "$RV" != "0" ] ; then + exit 1 + fi + fi + + if [ ! -f $CACHED_TAR -o \ + $CACHED_IMAGE -nt $CACHED_TAR ] ; then + echo "Repacking base image as tarball." + WORKING=$(mktemp -d) + EACTION="rm -r $WORKING" + trap "$EACTION" EXIT + echo "Working in $WORKING" + RAW_FILE=$(basename $BASE_IMAGE_FILE) + RAW_FILE=${RAW_FILE#.qcow2}.raw + qemu-img convert -f qcow2 -O raw $CACHED_IMAGE $WORKING/$RAW_FILE + + # F19 images have the rootfs partition on p1 + # Centos7 images on p3 (p1 boot, p2 swap) + if [[ $DISTRO_NAME = "centos7" ]]; then + ROOT_PARTITON=p3 + else + ROOT_PARTITON=p1 + fi + + # kpartx fails if no /dev/loop* exists, "losetup -f" prints first unused + # loop device and creates it if it doesn't exist + sudo losetup -f + + # XXX: Parsing stdout is dangerous, would like a better way to discover + # the device used for the image. + ROOT_LOOPDEV=$(sudo kpartx -av $WORKING/$RAW_FILE | \ + awk "/loop[0-9]+$ROOT_PARTITON/ {print \$3}") + if ! timeout 5 sh -c "while ! [ -e /dev/mapper/$ROOT_LOOPDEV ]; do sleep 1; done"; then + echo "Error: Could not find /dev/mapper/$ROOT_LOOPDEV" + exit 1 + fi + EACTION="sudo kpartx -d $WORKING/$RAW_FILE ; $EACTION" + trap "$EACTION" EXIT + mkdir $WORKING/mnt + sudo mount /dev/mapper/$ROOT_LOOPDEV $WORKING/mnt + EACTION="sudo umount -f $WORKING/mnt ; $EACTION" + trap "$EACTION" EXIT + + # need to copy the contents of /boot into the image too, so + # mount it + if [[ $DISTRO_NAME = "centos7" ]]; then + BOOT_LOOPDEV=${ROOT_LOOPDEV/p3/p1} + sudo mount /dev/mapper/$BOOT_LOOPDEV $WORKING/mnt/boot + EACTION="sudo umount -f $WORKING/mnt/boot ; $EACTION" + trap "$EACTION" EXIT + fi + + # Chroot in so that we get the correct uid/gid + sudo chroot $WORKING/mnt bin/tar -cz . > $WORKING/tmp.tar + mv $WORKING/tmp.tar $CACHED_TAR + else + echo "Using cached tar from $CACHED_TAR" + fi fi -fi -# Extract the base image (use --numeric-owner to avoid UID/GID mismatch between -# image tarball and host OS e.g. when building Fedora image on an openSUSE host) -echo "Extracting base root image from $CACHED_TAR" -sudo tar -C $TARGET_ROOT --numeric-owner -xzf $CACHED_TAR + # Extract the base image (use --numeric-owner to avoid UID/GID mismatch between + # image tarball and host OS e.g. when building Fedora image on an openSUSE host) + echo "Extracting base root image from $CACHED_TAR" + sudo tar -C $TARGET_ROOT --numeric-owner -xzf $CACHED_TAR +} + +( + echo "Getting $TAR_LOCK: $(date)" + # Wait up to 20 minutes for another process to download + if ! flock -w 1200 9 ; then + echo "Did not get $TAR_LOCK: $(date)" + exit 1 + fi + extract_image +) 9> $TAR_LOCK