9e5bc37265
Adds a file type to source-repositories to allow individual files to make use of source-repositories caching. REPO_DIRECTORY has been rename to REPO_DEST since it is now sometimes a file. This change also fixes the incorrect path to the cache-url in the tar type. Change-Id: Ib79e41969c982a02f0235318d9f254b39c3c6d93
120 lines
4.8 KiB
Bash
Executable File
120 lines
4.8 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set -eu
|
|
|
|
# Gets repositories or individual files listed in the a repository file
|
|
# and places them in the specified destination path.
|
|
# The format of the repository file is one or more lines matching
|
|
# <name> <type> <destination> <location> [<ref>]
|
|
function get_repos_for_element(){
|
|
local REPO_SOURCES=$1
|
|
local CACHE_URL=$TMP_HOOKS_PATH/bin/cache-url
|
|
|
|
local REGEX="^([^ ]+) (git|tar|file) (/[^ ]+) ([^ ]+) ?([^ ]*)$"
|
|
|
|
while read line ; do
|
|
|
|
# ignore blank lines and lines begining in '#'
|
|
[[ "$line" == \#* ]] || [[ -z "$line" ]] && continue
|
|
|
|
if [[ "$line" =~ $REGEX ]] ; then
|
|
local REPONAME=${BASH_REMATCH[1]}
|
|
local REPOTYPE=${BASH_REMATCH[2]}
|
|
local REPOPATH=${BASH_REMATCH[3]}
|
|
local REPOLOCATION=${BASH_REMATCH[4]}
|
|
local REPO_ORIG_LOCATION=$REPOLOCATION
|
|
local REPOREF=${BASH_REMATCH[5]:-master}
|
|
|
|
local REPO_DEST=$TMP_MOUNT_PATH$REPOPATH
|
|
local REPO_SUB_DIRECTORY=$(dirname $REPO_DEST)
|
|
|
|
# REPOTYPE can be overridden with DIB_REPOTYPE_{name}
|
|
local REPOTYPE_OVERRIDE=DIB_REPOTYPE_${REPONAME//-/_}
|
|
REPOTYPE=${!REPOTYPE_OVERRIDE:-$REPOTYPE}
|
|
|
|
# REPOLOCATION can be overridden with DIB_REPOLOCATION_{name}
|
|
local REPOLOCATION_OVERRIDE=DIB_REPOLOCATION_${REPONAME//-/_}
|
|
REPOLOCATION=${!REPOLOCATION_OVERRIDE:-$REPOLOCATION}
|
|
if [ "$REPOLOCATION" = "$REPO_ORIG_LOCATION" ] ; then
|
|
# We're pulling from the element specified repo, do caching for
|
|
# the user.
|
|
CACHE_NAME=$(echo $REPOLOCATION | sha1sum | awk '{ print $1 }' )
|
|
CACHE_PATH=~/.cache/image-create/repository-sources/$CACHE_NAME
|
|
else
|
|
CACHE_PATH=''
|
|
fi
|
|
|
|
# REPOREF can be overridden with DIB_REPOREF_{name}
|
|
local REPOREF_OVERRIDE=DIB_REPOREF_${REPONAME//-/_}
|
|
REPOREF=${!REPOREF_OVERRIDE:-$REPOREF}
|
|
|
|
case $REPOTYPE in
|
|
git)
|
|
sudo mkdir -p $REPO_SUB_DIRECTORY
|
|
if [ -n "$CACHE_PATH" ] ; then
|
|
if [ ! -e "$CACHE_PATH" ] ; then
|
|
git clone $REPOLOCATION $CACHE_PATH.tmp
|
|
mv ${CACHE_PATH}{.tmp,}
|
|
elif [ -z "$DIB_OFFLINE" ] ; then
|
|
git --git-dir=$CACHE_PATH/.git fetch
|
|
git --git-dir=$CACHE_PATH/.git --work-tree=$CACHE_PATH reset --hard origin/master
|
|
fi
|
|
sudo git clone $CACHE_PATH $REPO_DEST
|
|
else
|
|
sudo git clone $REPOLOCATION $REPO_DEST
|
|
fi
|
|
pushd $REPO_DEST
|
|
sudo git reset --hard $REPOREF
|
|
popd
|
|
;;
|
|
tar)
|
|
# The top level directory of the tarball mightn't have a fixed name i.e.
|
|
# it could contain version numbers etc... so we write it to a tmpdir
|
|
# the then move the contents into the directory we want it in, this does
|
|
# assume the tarball only contains a single top level directory
|
|
local tmpdir=$(mktemp --tmpdir=$TMP_MOUNT_PATH/tmp -d)
|
|
if [ -n "$CACHE_PATH" ] ; then
|
|
if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then
|
|
$CACHE_URL $REPOLOCATION $CACHE_PATH
|
|
fi
|
|
tar -C $tmpdir -xzf $CACHE_PATH
|
|
else
|
|
curl $REPOLOCATION | tar -C $tmpdir -xzf -
|
|
fi
|
|
sudo mkdir -p $REPO_DEST
|
|
sudo mv $tmpdir/*/* $REPO_DEST
|
|
rm -rf $tmpdir
|
|
;;
|
|
file)
|
|
sudo mkdir -p $REPO_SUB_DIRECTORY
|
|
if [ -n "$CACHE_PATH" ] ; then
|
|
if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then
|
|
$CACHE_URL $REPOLOCATION $CACHE_PATH
|
|
fi
|
|
sudo cp $CACHE_PATH $REPO_DEST
|
|
else
|
|
sudo curl $REPOLOCATION -o $REPO_DEST
|
|
fi
|
|
;;
|
|
*)
|
|
echo "Unsupported repository type"
|
|
return 1
|
|
;;
|
|
esac
|
|
# Capture the in-instance repository path for later review / other
|
|
# elements (like a pypi dependency cache).
|
|
echo "$REPOPATH" | sudo dd of=$TMP_MOUNT_PATH/etc/dib-source-repositories oflag=append conv=notrunc
|
|
else
|
|
echo "Couldn't parse '$line' as a source repository"
|
|
return 1
|
|
fi
|
|
done < $REPO_SOURCES
|
|
}
|
|
|
|
mkdir -p ~/.cache/image-create/repository-sources/
|
|
|
|
# Get source repositories for the target
|
|
for _SOURCEREPO in $(find $TMP_HOOKS_PATH -maxdepth 1 -name "source-repository-*" -not -name '*~'); do
|
|
get_repos_for_element $_SOURCEREPO
|
|
done
|