4c7a3bf79f
We now unconditionally cache the repositories being fetched with source-repositories. Additionally, by improving the way we handle repository refs, we are now able to build images with code taken directly from gerrit reviews. Change-Id: Ifd4c5691f6761eb5551663e6d0aa1c0e42afced3
124 lines
5.2 KiB
Bash
Executable File
124 lines
5.2 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set -eu
|
|
|
|
# Gets repositories or individual files listed in the a repository file
|
|
# and places them in the specified destination path.
|
|
# The format of the repository file is one or more lines matching
|
|
# <name> <type> <destination> <location> [<ref>]
|
|
function get_repos_for_element(){
|
|
local REPO_SOURCES=$1
|
|
local CACHE_URL=$TMP_HOOKS_PATH/bin/cache-url
|
|
|
|
local REGEX="^([^ ]+) (git|tar|file) (/[^ ]+) ([^ ]+) ?([^ ]*)$"
|
|
|
|
while read line ; do
|
|
|
|
# ignore blank lines and lines begining in '#'
|
|
[[ "$line" == \#* ]] || [[ -z "$line" ]] && continue
|
|
|
|
if [[ "$line" =~ $REGEX ]] ; then
|
|
local REPONAME=${BASH_REMATCH[1]}
|
|
local REPOTYPE=${BASH_REMATCH[2]}
|
|
local REPOPATH=${BASH_REMATCH[3]}
|
|
local REPOLOCATION=${BASH_REMATCH[4]}
|
|
local REPO_ORIG_LOCATION=$REPOLOCATION
|
|
local REPOREF=${BASH_REMATCH[5]:-master}
|
|
|
|
local REPO_DEST=$TMP_MOUNT_PATH$REPOPATH
|
|
local REPO_SUB_DIRECTORY=$(dirname $REPO_DEST)
|
|
|
|
# REPOTYPE can be overridden with DIB_REPOTYPE_{name}
|
|
local REPOTYPE_OVERRIDE=DIB_REPOTYPE_${REPONAME//-/_}
|
|
REPOTYPE=${!REPOTYPE_OVERRIDE:-$REPOTYPE}
|
|
|
|
# REPOLOCATION can be overridden with DIB_REPOLOCATION_{name}
|
|
local REPOLOCATION_OVERRIDE=DIB_REPOLOCATION_${REPONAME//-/_}
|
|
REPOLOCATION=${!REPOLOCATION_OVERRIDE:-$REPOLOCATION}
|
|
|
|
# REPOREF can be overridden with DIB_REPOREF_{name}
|
|
local REPOREF_OVERRIDE=DIB_REPOREF_${REPONAME//-/_}
|
|
REPOREF=${!REPOREF_OVERRIDE:-$REPOREF}
|
|
|
|
# Determine a unique cache path for this repo
|
|
CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' )
|
|
CACHE_PATH=~/.cache/image-create/repository-sources/$CACHE_NAME
|
|
|
|
case $REPOTYPE in
|
|
git)
|
|
sudo mkdir -p $REPO_SUB_DIRECTORY
|
|
|
|
if [ ! -e "$CACHE_PATH" ] ; then
|
|
echo "Caching $REPONAME from $REPOLOCATION in $CACHE_PATH"
|
|
git clone $REPOLOCATION $CACHE_PATH.tmp
|
|
mv ${CACHE_PATH}{.tmp,}
|
|
fi
|
|
|
|
HAS_REF=$(git --git-dir=$CACHE_PATH/.git show-ref $REPOREF || true)
|
|
if [ -z "$DIB_OFFLINE" -o -z "$HAS_REF" ] ; then
|
|
echo "Updating cache of $REPOLOCATION in $CACHE_PATH with ref $REPOREF"
|
|
git --git-dir=$CACHE_PATH/.git fetch --update-head-ok $REPOLOCATION ${REPOREF}:${REPOREF}
|
|
fi
|
|
|
|
echo "Cloning from $REPONAME cache and applying ref $REPOREF"
|
|
sudo git clone $CACHE_PATH $REPO_DEST
|
|
pushd $REPO_DEST
|
|
sudo git fetch $CACHE_PATH $REPOREF
|
|
sudo git reset --hard FETCH_HEAD
|
|
popd
|
|
;;
|
|
tar)
|
|
# The top level directory of the tarball mightn't have a fixed name i.e.
|
|
# it could contain version numbers etc... so we write it to a tmpdir
|
|
# the then move the contents into the directory we want it in, this does
|
|
# assume the tarball only contains a single top level directory
|
|
local tmpdir=$(mktemp --tmpdir=$TMP_MOUNT_PATH/tmp -d)
|
|
if [ -n "$CACHE_PATH" ] ; then
|
|
echo "Caching $REPONAME tarball from $REPOLOCATION in $CACHE_PATH"
|
|
if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then
|
|
$CACHE_URL $REPOLOCATION $CACHE_PATH
|
|
fi
|
|
tar -C $tmpdir -xzf $CACHE_PATH
|
|
else
|
|
echo "Fetching $REPONAME tarball from $REPOLOCATION"
|
|
curl $REPOLOCATION | tar -C $tmpdir -xzf -
|
|
fi
|
|
sudo mkdir -p $REPO_DEST
|
|
sudo mv $tmpdir/*/* $REPO_DEST
|
|
rm -rf $tmpdir
|
|
;;
|
|
file)
|
|
sudo mkdir -p $REPO_SUB_DIRECTORY
|
|
if [ -n "$CACHE_PATH" ] ; then
|
|
echo "Caching $REPONAME file from $REPOLOCATION in $CACHE_PATH"
|
|
if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then
|
|
$CACHE_URL $REPOLOCATION $CACHE_PATH
|
|
fi
|
|
sudo cp $CACHE_PATH $REPO_DEST
|
|
else
|
|
echo "Fetching $REPONAME file from $REPOLOCATION"
|
|
sudo curl $REPOLOCATION -o $REPO_DEST
|
|
fi
|
|
;;
|
|
*)
|
|
echo "Unsupported repository type"
|
|
return 1
|
|
;;
|
|
esac
|
|
# Capture the in-instance repository path for later review / other
|
|
# elements (like a pypi dependency cache).
|
|
echo "$REPOPATH" | sudo dd of=$TMP_MOUNT_PATH/etc/dib-source-repositories oflag=append conv=notrunc
|
|
else
|
|
echo "Couldn't parse '$line' as a source repository"
|
|
return 1
|
|
fi
|
|
done < $REPO_SOURCES
|
|
}
|
|
|
|
mkdir -p ~/.cache/image-create/repository-sources/
|
|
|
|
# Get source repositories for the target
|
|
for _SOURCEREPO in $(find $TMP_HOOKS_PATH -maxdepth 1 -name "source-repository-*" -not -name '*~'); do
|
|
get_repos_for_element $_SOURCEREPO
|
|
done
|