#!/bin/bash set -eu set -o pipefail # If the old cache exists, move it to the new name function make_new_cache(){ local OLD_CACHE_BASE=$1 local CACHE_BASE=$2 # If the old cache name exists, move it to the new cache name if [ -e "$OLD_CACHE_BASE" ] ; then if [ ! -e "$CACHE_BASE" ] ; then mv -n $OLD_CACHE_BASE $CACHE_BASE else echo "Not replacing new cache location with old cache" fi fi } # Gets repositories or individual files listed in the a repository file # and places them in the specified destination path. # The format of the repository file is one or more lines matching # [] function get_repos_for_element(){ local REPO_SOURCES=$1 local CACHE_URL=$TMP_HOOKS_PATH/bin/cache-url local REGEX="^([^ ]+) (git|tar|file|package) ?(/[^ ]+)? ?([^ ]+)? ?([^ ]*)$" while read line; do # expand variables line=$(eval echo $line) # ignore blank lines and lines beginning in '#' [[ "$line" == \#* ]] || [[ -z "$line" ]] && continue if [[ "$line" =~ $REGEX ]] ; then local REPONAME=${BASH_REMATCH[1]} local REPOTYPE=${BASH_REMATCH[2]} local REPOPATH=${BASH_REMATCH[3]} local REPOLOCATION=${BASH_REMATCH[4]} local REPO_ORIG_LOCATION=$REPOLOCATION local REPOREF=${BASH_REMATCH[5]:-master} local REPO_DEST=$TMP_MOUNT_PATH$REPOPATH local REPO_SUB_DIRECTORY=$(dirname $REPO_DEST) # REPOTYPE can be overridden with DIB_REPOTYPE_{name} local REPOTYPE_OVERRIDE=DIB_REPOTYPE_${REPONAME//[^A-Za-z0-9]/_} REPOTYPE=${!REPOTYPE_OVERRIDE:-$REPOTYPE} # REPOLOCATION can be overridden with DIB_REPOLOCATION_{name} local REPOLOCATION_OVERRIDE=DIB_REPOLOCATION_${REPONAME//[^A-Za-z0-9]/_} REPOLOCATION=${!REPOLOCATION_OVERRIDE:-$REPOLOCATION} # REPOREF can be overridden with DIB_REPOREF_{name} local REPOREF_OVERRIDE=DIB_REPOREF_${REPONAME//[^A-Za-z0-9]/_} REPOREF=${!REPOREF_OVERRIDE:-$REPOREF} # Determine a unique cache path for this repo CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' ) OLD_CACHE_PATH=${CACHE_BASE}/${CACHE_NAME} # Add the repo name to the sha1sum for readability CACHE_NAME=${REPONAME//[^A-Za-z0-9]/_}_${CACHE_NAME} CACHE_PATH=${CACHE_BASE}/$CACHE_NAME make_new_cache $OLD_CACHE_PATH $CACHE_PATH # Return if install type is not source local INSTALL_TYPE_VAR=DIB_INSTALLTYPE_${REPONAME//[^A-Za-z0-9]/_} local INSTALL_TYPE=${!INSTALL_TYPE_VAR:-source} if [ ! $INSTALL_TYPE = "source" ]; then echo "$REPONAME install type not set to source" continue fi case $REPOTYPE in git) if [ -z "${!REPOLOCATION_OVERRIDE:-""}" -a -n "${DIB_GITREPOBASE:-""}" ] ; then # Transform the current repo base to the new one local NEW_REPOLOCATION=$(echo $REPOLOCATION |\ sed "s,^[^:]\+://[^/]\+/\(~[^/]\+\)\?\(.*\)$,${DIB_GITREPOBASE}\2,g") echo "Transformed ${REPOLOCATION} to ${NEW_REPOLOCATION}" REPOLOCATION=$NEW_REPOLOCATION # Also update the cache location CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' ) CACHE_PATH=~/.cache/image-create/repository-sources/$CACHE_NAME fi sudo mkdir -p $REPO_SUB_DIRECTORY if [ ! -e "$CACHE_PATH" ] ; then echo "Caching $REPONAME from $REPOLOCATION in $CACHE_PATH" git clone $REPOLOCATION $CACHE_PATH.tmp mv ${CACHE_PATH}{.tmp,} fi HAS_REF=$(git --git-dir=$CACHE_PATH/.git name-rev $REPOREF 2>/dev/null || true) if [ -z "$DIB_OFFLINE" -o -z "$HAS_REF" ] ; then echo "Updating cache of $REPOLOCATION in $CACHE_PATH with ref $REPOREF" # Copy named refs (which might be outside the usual heads # pattern) - e.g. gerrit if ! git --git-dir=$CACHE_PATH/.git fetch --update-head-ok $REPOLOCATION \ +${REPOREF}:${REPOREF} ; then # Copy all heads from the remote repository - this permits # using a SHA1 object reference so long as the object # reference is reachable from one of the heads. git does # not permit arbitrary sha fetching from remote servers. # This is a separate fetch to the prior one as the prior # one will fail when REPOREF is a SHA1. git --git-dir=$CACHE_PATH/.git fetch --update-head-ok $REPOLOCATION \ +refs/heads/*:refs/heads/* fi # Ensure that we have a reference to the revision. git --git-dir=$CACHE_PATH/.git rev-parse -q --verify $REPOREF^{commit} fi echo "Cloning from $REPONAME cache and applying ref $REPOREF" sudo git clone $CACHE_PATH $REPO_DEST pushd $REPO_DEST sudo git fetch $CACHE_PATH $REPOREF sudo git reset --hard FETCH_HEAD # Get the sha in use git_sha=$(git rev-parse FETCH_HEAD) popd # Write the sha being used into the source-repositories manifest echo "$REPONAME git $REPOPATH $REPOLOCATION $git_sha" >> $GIT_MANIFEST ;; tar) # The top level directory of the tarball mightn't have a fixed name i.e. # it could contain version numbers etc... so we write it to a tmpdir # the then move the contents into the directory we want it in, this does # assume the tarball only contains a single top level directory local tmpdir=$(mktemp --tmpdir=$TMP_MOUNT_PATH/tmp -d) if [ -n "$CACHE_PATH" ] ; then echo "Caching $REPONAME tarball from $REPOLOCATION in $CACHE_PATH" if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then $CACHE_URL $REPOLOCATION $CACHE_PATH fi tar -C $tmpdir -xzf $CACHE_PATH else echo "Fetching $REPONAME tarball from $REPOLOCATION" curl $REPOLOCATION | tar -C $tmpdir -xzf - fi sudo mkdir -p $REPO_DEST sudo mv $tmpdir/*/* $REPO_DEST rm -rf $tmpdir ;; file) sudo mkdir -p $REPO_SUB_DIRECTORY if [ -n "$CACHE_PATH" ] ; then echo "Caching $REPONAME file from $REPOLOCATION in $CACHE_PATH" if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then $CACHE_URL $REPOLOCATION $CACHE_PATH fi sudo cp $CACHE_PATH $REPO_DEST else echo "Fetching $REPONAME file from $REPOLOCATION" sudo curl $REPOLOCATION -o $REPO_DEST fi ;; *) echo "Unsupported repository type: $REPOTYPE" return 1 ;; esac # Capture the in-instance repository path for later review / other # elements (like a pypi dependency cache). echo "$REPOPATH" | sudo dd of=$TMP_MOUNT_PATH/etc/dib-source-repositories oflag=append conv=notrunc else echo "Couldn't parse '$line' as a source repository" return 1 fi done < $REPO_SOURCES } CACHE_BASE=~/.cache/image-create/source-repositories OLD_CACHE_BASE=~/.cache/image-create/repository-sources make_new_cache $OLD_CACHE_BASE $CACHE_BASE mkdir -p $CACHE_BASE # Use the IMAGE_NAME from the calling script, and make it unique with the temporary path GIT_MANIFEST_NAME=dib-manifest-git-$(basename ${IMAGE_NAME}) GIT_MANIFEST_CACHE_NAME=${GIT_MANIFEST_NAME}_$(dirname ${TMP_MOUNT_PATH##*.}) GIT_MANIFEST=$CACHE_BASE/${GIT_MANIFEST_CACHE_NAME} rm -f $GIT_MANIFEST # Get source repositories for the target for _SOURCEREPO in $(find $TMP_HOOKS_PATH -maxdepth 1 -name "source-repository-*" -not -name '*~'); do repo_flock=$CACHE_BASE/repositories_flock ( echo "Getting $repo_flock: $(date)" if ! flock -w 600 9 ; then echo "Did not get $repo_flock: $(date)" exit 1 fi get_repos_for_element $_SOURCEREPO ) 9> $repo_flock done # Copy the manifest into the image if it exists (there may be no git repositories used) if [ -e "$GIT_MANIFEST" ] ; then sudo mv $GIT_MANIFEST $TMP_MOUNT_PATH/${DIB_MANIFEST_IMAGE_DIR}/$GIT_MANIFEST_NAME fi