From 79ca4d901cd5a41fca700b1ca1f7837dd3c62ee5 Mon Sep 17 00:00:00 2001
From: Robert Collins <rbtcollins@hp.com>
Date: Thu, 18 Jul 2013 15:02:31 +1200
Subject: [PATCH] Cache repository-sources data.

Cloning large repositories over the internet can take considerable
time.  Caching them locally makes repeated image builds significantly
faster, so lets do that.

When users override the element source they will often be using a
local repository, so in those cases I don't cache - but we could
easily change our minds on that in the future.

Change-Id: I9822362cf722b904d9806dbbb4bb07cfe2b33437
---
 elements/source-repositories/element-deps     |  1 +
 .../extra-data.d/99-getsources                | 35 +++++++++++++++++--
 2 files changed, 33 insertions(+), 3 deletions(-)
 create mode 100644 elements/source-repositories/element-deps

diff --git a/elements/source-repositories/element-deps b/elements/source-repositories/element-deps
new file mode 100644
index 00000000..5562b27b
--- /dev/null
+++ b/elements/source-repositories/element-deps
@@ -0,0 +1 @@
+cache-url
diff --git a/elements/source-repositories/extra-data.d/99-getsources b/elements/source-repositories/extra-data.d/99-getsources
index e352099a..1a531871 100755
--- a/elements/source-repositories/extra-data.d/99-getsources
+++ b/elements/source-repositories/extra-data.d/99-getsources
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -e
+set -eu
 
 # Gets Repositories listed in the a repository file and places them in
 # the repository directory.
@@ -20,6 +20,7 @@ function get_repos_for_element(){
             local REPOTYPE=${BASH_REMATCH[2]}
             local REPOPATH=${BASH_REMATCH[3]}
             local REPOLOCATION=${BASH_REMATCH[4]}
+            local REPO_ORIG_LOCATION=$REPOLOCATION
             local REPOREF=${BASH_REMATCH[5]:-master}
 
             local REPO_DIRECTORY=$TMP_MOUNT_PATH$REPOPATH
@@ -32,6 +33,14 @@ function get_repos_for_element(){
             # REPOLOCATION can be overridden with DIB_REPOLOCATION_{name}
             local REPOLOCATION_OVERRIDE=DIB_REPOLOCATION_${REPONAME/-/_}
             REPOLOCATION=${!REPOLOCATION_OVERRIDE:-$REPOLOCATION}
+            if [ "$REPOLOCATION" = "$REPO_ORIG_LOCATION" ] ; then
+                # We're pulling from the element specified repo, do caching for
+                # the user.
+                CACHE_NAME=$(echo $REPOLOCATION | sha1sum | awk '{ print $1 }' )
+                CACHE_PATH=~/.cache/image-create/repository-sources/$CACHE_NAME
+            else
+                CACHE_PATH=''
+            fi
 
             # REPOREF can be overridden with DIB_REPOREF_{name}
             local REPOREF_OVERRIDE=DIB_REPOREF_${REPONAME/-/_}
@@ -40,7 +49,18 @@ function get_repos_for_element(){
             case $REPOTYPE in
             git)
                 sudo mkdir -p $REPO_SUB_DIRECTORY
-                sudo git clone $REPOLOCATION $REPO_DIRECTORY
+                if [ -n "$CACHE_PATH" ] ; then
+                    if [ ! -e "$CACHE_PATH" ] ; then
+                        git clone $REPOLOCATION $CACHE_PATH.tmp
+                        mv ${CACHE_PATH}{.tmp,}
+                    elif [ -z "$DIB_OFFLINE" ] ; then
+                        git --git-dir=$CACHE_PATH/.git fetch
+                        git --git-dir=$CACHE_PATH/.git reset --hard origin/master
+                    fi
+                    sudo git clone $CACHE_PATH $REPO_DIRECTORY
+                else
+                    sudo git clone $REPOLOCATION $REPO_DIRECTORY
+                fi
                 pushd $REPO_DIRECTORY
                 sudo git reset --hard $REPOREF
                 popd
@@ -51,7 +71,14 @@ function get_repos_for_element(){
                 # the then move the contents into the directory we want it in, this does
                 # assume the tarball only contains a single top level directory
                 local tmpdir=$(mktemp --tmpdir=$TMP_MOUNT_PATH/tmp -d)
-                curl $REPOLOCATION | tar -C $tmpdir -xzf -
+                if [ -n "$CACHE_PATH" ] ; then
+                    if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then
+                        $TMP_MOUNT_PATH/usr/local/bin/cache-url $REPOLOCATION $CACHE_PATH
+                    fi
+                    tar -C $tmpdir -xzf $CACHE_PATH
+                else
+                    curl $REPOLOCATION | tar -C $tmpdir -xzf -
+                fi
                 sudo mkdir -p $REPO_DIRECTORY
                 sudo mv $tmpdir/*/* $REPO_DIRECTORY
                 rm -rf $tmpdir
@@ -69,6 +96,8 @@ function get_repos_for_element(){
     done < $REPO_SOURCES
 }
 
+mkdir -p ~/.cache/image-create/repository-sources/
+
 # Get source repositories for the target
 for _SOURCEREPO in $(find $TMP_HOOKS_PATH -maxdepth 1 -name "source-repository-*") ; do
     get_repos_for_element $_SOURCEREPO