sig_cloud/diskimage-builder
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge "Cleanup source-repositories output"

Browse Source
This commit is contained in:
Jenkins 2016-06-07 00:59:02 +00:00 committed by Gerrit Code Review
commit 536e8a2dd2
1 changed files with 28 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
elements/source-repositories/extra-data.d/98-source-repositories
View File

@ -37,19 +37,24 @@ function get_repos_for_element(){
local REGEX="^([^ ]+) (git|tar|file|package) ?(/[^ ]+)? ?([^ ]+)? ?([^ ]*)$" local REGEX="^([^ ]+) (git|tar|file|package) ?(/[^ ]+)? ?([^ ]+)? ?([^ ]*)$"
# this can be a rather long list (e.g. caching ~1000 openstack
# repos), so we make a rough count for progress display
local COUNT=1
local REPO_SOURCES_COUNT=$(grep -v '^$\|^\s*\#' $REPO_SOURCES | wc -l)
while read line; do while read line; do
# temporarily turn off globbing '*' (since it may be used as the REPOREF for tarballs)
set -f
# expand variables # temporarily turn off globbing '*' (since it may be used as
line=$(eval echo $line) # the REPOREF for tarballs) and expand variables
set -f; line=$(eval echo $line); set +f
# restore globbing
set +f
# ignore blank lines and lines beginning in '#' # ignore blank lines and lines beginning in '#'
[[ "$line" == \#* ]] || [[ -z "$line" ]] && continue [[ "$line" == \#* ]] || [[ -z "$line" ]] && continue
# rough progress counter
printf "(%04d / %04d)\n" ${COUNT} ${REPO_SOURCES_COUNT}
COUNT=$(( COUNT + 1))
if [[ "$line" =~ $REGEX ]] ; then if [[ "$line" =~ $REGEX ]] ; then
local REPONAME=${BASH_REMATCH[1]} local REPONAME=${BASH_REMATCH[1]}
local REPOTYPE=${BASH_REMATCH[2]} local REPOTYPE=${BASH_REMATCH[2]}
@ -115,7 +120,7 @@ function get_repos_for_element(){
if [ ! -e "$CACHE_PATH" ] ; then if [ ! -e "$CACHE_PATH" ] ; then
echo "Caching $REPONAME from $REPOLOCATION in $CACHE_PATH" echo "Caching $REPONAME from $REPOLOCATION in $CACHE_PATH"
git clone $REPOLOCATION $CACHE_PATH.tmp git clone -q $REPOLOCATION $CACHE_PATH.tmp
mv ${CACHE_PATH}{.tmp,} mv ${CACHE_PATH}{.tmp,}
fi fi
@ -126,7 +131,7 @@ function get_repos_for_element(){
echo "Updating cache of $REPOLOCATION in $CACHE_PATH with ref $REPOREF" echo "Updating cache of $REPOLOCATION in $CACHE_PATH with ref $REPOREF"
# Copy named refs (which might be outside the usual heads # Copy named refs (which might be outside the usual heads
# pattern) - e.g. gerrit # pattern) - e.g. gerrit
if [ "$REPOREF" == "*" ] || ! git --git-dir=$CACHE_PATH/.git fetch --prune --update-head-ok $REPOLOCATION \ if [ "$REPOREF" == "*" ] || ! git --git-dir=$CACHE_PATH/.git fetch -q --prune --update-head-ok $REPOLOCATION \
+${REPOREF}:${REPOREF} ; then +${REPOREF}:${REPOREF} ; then
# Copy all heads from the remote repository - this permits # Copy all heads from the remote repository - this permits
# using a SHA1 object reference so long as the object # using a SHA1 object reference so long as the object
@ -134,12 +139,15 @@ function get_repos_for_element(){
# not permit arbitrary sha fetching from remote servers. # not permit arbitrary sha fetching from remote servers.
# This is a separate fetch to the prior one as the prior # This is a separate fetch to the prior one as the prior
# one will fail when REPOREF is a SHA1. # one will fail when REPOREF is a SHA1.
git --git-dir=$CACHE_PATH/.git fetch --prune --update-head-ok $REPOLOCATION \ git --git-dir=$CACHE_PATH/.git fetch -q --prune --update-head-ok $REPOLOCATION \
+refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/* +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*
fi fi
# Ensure that we have a reference to the revision. # Ensure that we have a reference to the revision.
if [ "$REPOREF" != "*" ] ; then if [ "$REPOREF" != "*" ] ; then
git --git-dir=$CACHE_PATH/.git rev-parse -q --verify $REPOREF^{commit} if ! git --git-dir=$CACHE_PATH/.git rev-parse -q --verify $REPOREF^{commit} > /dev/null; then
echo "Failed to find reference to $REPOREF"
exit 1
fi
fi fi
fi fi
@ -151,7 +159,7 @@ function get_repos_for_element(){
echo "REPOLOCATIONS don't match ("$CACHE_PATH" != "$DESIRED")" >&2 echo "REPOLOCATIONS don't match ("$CACHE_PATH" != "$DESIRED")" >&2
exit 1 exit 1
elif [[ "$REPOREF" != "*" ]]; then elif [[ "$REPOREF" != "*" ]]; then
pushd $REPO_DEST pushd $REPO_DEST > /dev/null
# When we first clone we create a branch naming what we fetched # When we first clone we create a branch naming what we fetched
# that must match, or we are asking for two different references from the # that must match, or we are asking for two different references from the
# same repo, which is an error # same repo, which is an error
@ -159,22 +167,22 @@ function get_repos_for_element(){
echo "REPOREFS don't match - failed to get sha1 of fetch_$REPOREF" >&2 echo "REPOREFS don't match - failed to get sha1 of fetch_$REPOREF" >&2
exit 1 exit 1
fi fi
popd popd > /dev/null
fi fi
else else
sudo git clone $CACHE_PATH $REPO_DEST sudo git clone -q $CACHE_PATH $REPO_DEST
pushd $REPO_DEST pushd $REPO_DEST > /dev/null
if [[ "$REPOREF" == "*" ]]; then if [[ "$REPOREF" == "*" ]]; then
sudo git fetch --prune --update-head-ok $CACHE_PATH \ sudo git fetch -q --prune --update-head-ok $CACHE_PATH \
+refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/* +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*
git_sha=$(git rev-parse HEAD) git_sha=$(git rev-parse HEAD)
else else
sudo git fetch $CACHE_PATH $REPOREF:fetch_$REPOREF sudo git fetch -q $CACHE_PATH $REPOREF:fetch_$REPOREF
sudo git reset --hard FETCH_HEAD sudo git reset --hard FETCH_HEAD
# Get the sha in use # Get the sha in use
git_sha=$(git rev-parse FETCH_HEAD) git_sha=$(git rev-parse FETCH_HEAD)
fi fi
popd popd > /dev/null
# Write the sha being used into the source-repositories manifest # Write the sha being used into the source-repositories manifest
echo "$REPONAME git $REPOPATH $REPOLOCATION $git_sha" >> $GIT_MANIFEST echo "$REPONAME git $REPOPATH $REPOLOCATION $git_sha" >> $GIT_MANIFEST
@ -226,7 +234,7 @@ function get_repos_for_element(){
# Capture the in-instance repository path for later review / other # Capture the in-instance repository path for later review / other
# elements (like a pypi dependency cache). # elements (like a pypi dependency cache).
echo "$REPOPATH" | sudo dd of=$TMP_MOUNT_PATH/etc/dib-source-repositories oflag=append conv=notrunc echo "$REPOPATH" | sudo tee -a $TMP_MOUNT_PATH/etc/dib-source-repositories > /dev/null
else else
echo "Couldn't parse '$line' as a source repository" echo "Couldn't parse '$line' as a source repository"
@ -249,7 +257,7 @@ rm -f $GIT_MANIFEST
for _SOURCEREPO in $(find $TMP_HOOKS_PATH -maxdepth 1 -name "source-repository-*" -not -name '*~'); do for _SOURCEREPO in $(find $TMP_HOOKS_PATH -maxdepth 1 -name "source-repository-*" -not -name '*~'); do
repo_flock=$CACHE_BASE/repositories_flock repo_flock=$CACHE_BASE/repositories_flock
( (
echo "Getting $repo_flock: $(date)" echo "Getting $repo_flock: $(date) for $_SOURCEREPO"
if ! flock -w 600 9 ; then if ! flock -w 600 9 ; then
echo "Did not get $repo_flock: $(date)" echo "Did not get $repo_flock: $(date)"
exit 1 exit 1