sig_cloud/diskimage-builder
11
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Cleanup source-repositories output

Browse source 
This element takes up a *huge* part of the logs of openstack-infra
builds as we go and cache every git tree.

This silences most of the noise which will reduce the mess
in the logs considerably.

Note that we've had logging on this turned down since
I91c5e55814ba9840769357261d203f4850e2eba6 but it has been
ineffective in stopping the log-spew output, see the
dependency change.

Change-Id: I60f06f84d57087c82b3907575bff125015d35171
Depends-On: I1e39822f218dc0322e2490a770f3dc867a55802c
This commit is contained in:
Ian Wienand 2016-05-27 05:29:39 +00:00
parent f0b57d5efd
commit 74d0185dde
1 changed files with 28 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
elements/source-repositories/extra-data.d/98-source-repositories
View file

@ -36,19 +36,24 @@ function get_repos_for_element(){
local REGEX="^([^ ]+) (git|tar|file|package) ?(/[^ ]+)? ?([^ ]+)? ?([^ ]*)$" local REGEX="^([^ ]+) (git|tar|file|package) ?(/[^ ]+)? ?([^ ]+)? ?([^ ]*)$"
# this can be a rather long list (e.g. caching ~1000 openstack
# repos), so we make a rough count for progress display
local COUNT=1
local REPO_SOURCES_COUNT=$(grep -v '^$\|^\s*\#' $REPO_SOURCES | wc -l)
while read line; do while read line; do
# temporarily turn off globbing '*' (since it may be used as the REPOREF for tarballs)
set -f
# expand variables # temporarily turn off globbing '*' (since it may be used as
line=$(eval echo $line) # the REPOREF for tarballs) and expand variables
set -f; line=$(eval echo $line); set +f
# restore globbing
set +f
# ignore blank lines and lines beginning in '#' # ignore blank lines and lines beginning in '#'
[[ "$line" == \#* ]] || [[ -z "$line" ]] && continue [[ "$line" == \#* ]] || [[ -z "$line" ]] && continue
# rough progress counter
printf "(%04d / %04d)\n" ${COUNT} ${REPO_SOURCES_COUNT}
COUNT=$(( COUNT + 1))
if [[ "$line" =~ $REGEX ]] ; then if [[ "$line" =~ $REGEX ]] ; then
local REPONAME=${BASH_REMATCH[1]} local REPONAME=${BASH_REMATCH[1]}
local REPOTYPE=${BASH_REMATCH[2]} local REPOTYPE=${BASH_REMATCH[2]}
@ -114,7 +119,7 @@ function get_repos_for_element(){
if [ ! -e "$CACHE_PATH" ] ; then if [ ! -e "$CACHE_PATH" ] ; then
echo "Caching $REPONAME from $REPOLOCATION in $CACHE_PATH" echo "Caching $REPONAME from $REPOLOCATION in $CACHE_PATH"
git clone $REPOLOCATION $CACHE_PATH.tmp git clone -q $REPOLOCATION $CACHE_PATH.tmp
mv ${CACHE_PATH}{.tmp,} mv ${CACHE_PATH}{.tmp,}
fi fi
@ -125,7 +130,7 @@ function get_repos_for_element(){
echo "Updating cache of $REPOLOCATION in $CACHE_PATH with ref $REPOREF" echo "Updating cache of $REPOLOCATION in $CACHE_PATH with ref $REPOREF"
# Copy named refs (which might be outside the usual heads # Copy named refs (which might be outside the usual heads
# pattern) - e.g. gerrit # pattern) - e.g. gerrit
if [ "$REPOREF" == "*" ] || ! git --git-dir=$CACHE_PATH/.git fetch --prune --update-head-ok $REPOLOCATION \ if [ "$REPOREF" == "*" ] || ! git --git-dir=$CACHE_PATH/.git fetch -q --prune --update-head-ok $REPOLOCATION \
+${REPOREF}:${REPOREF} ; then +${REPOREF}:${REPOREF} ; then
# Copy all heads from the remote repository - this permits # Copy all heads from the remote repository - this permits
# using a SHA1 object reference so long as the object # using a SHA1 object reference so long as the object
@ -133,12 +138,15 @@ function get_repos_for_element(){
# not permit arbitrary sha fetching from remote servers. # not permit arbitrary sha fetching from remote servers.
# This is a separate fetch to the prior one as the prior # This is a separate fetch to the prior one as the prior
# one will fail when REPOREF is a SHA1. # one will fail when REPOREF is a SHA1.
git --git-dir=$CACHE_PATH/.git fetch --prune --update-head-ok $REPOLOCATION \ git --git-dir=$CACHE_PATH/.git fetch -q --prune --update-head-ok $REPOLOCATION \
+refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/* +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*
fi fi
# Ensure that we have a reference to the revision. # Ensure that we have a reference to the revision.
if [ "$REPOREF" != "*" ] ; then if [ "$REPOREF" != "*" ] ; then
git --git-dir=$CACHE_PATH/.git rev-parse -q --verify $REPOREF^{commit} if ! git --git-dir=$CACHE_PATH/.git rev-parse -q --verify $REPOREF^{commit} > /dev/null; then
echo "Failed to find reference to $REPOREF"
exit 1
fi
fi fi
fi fi
@ -150,7 +158,7 @@ function get_repos_for_element(){
echo "REPOLOCATIONS don't match ("$CACHE_PATH" != "$DESIRED")" >&2 echo "REPOLOCATIONS don't match ("$CACHE_PATH" != "$DESIRED")" >&2
exit 1 exit 1
elif [[ "$REPOREF" != "*" ]]; then elif [[ "$REPOREF" != "*" ]]; then
pushd $REPO_DEST pushd $REPO_DEST > /dev/null
# When we first clone we create a branch naming what we fetched # When we first clone we create a branch naming what we fetched
# that must match, or we are asking for two different references from the # that must match, or we are asking for two different references from the
# same repo, which is an error # same repo, which is an error
@ -158,22 +166,22 @@ function get_repos_for_element(){
echo "REPOREFS don't match - failed to get sha1 of fetch_$REPOREF" >&2 echo "REPOREFS don't match - failed to get sha1 of fetch_$REPOREF" >&2
exit 1 exit 1
fi fi
popd popd > /dev/null
fi fi
else else
sudo git clone $CACHE_PATH $REPO_DEST sudo git clone -q $CACHE_PATH $REPO_DEST
pushd $REPO_DEST pushd $REPO_DEST > /dev/null
if [[ "$REPOREF" == "*" ]]; then if [[ "$REPOREF" == "*" ]]; then
sudo git fetch --prune --update-head-ok $CACHE_PATH \ sudo git fetch -q --prune --update-head-ok $CACHE_PATH \
+refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/* +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*
git_sha=$(git rev-parse HEAD) git_sha=$(git rev-parse HEAD)
else else
sudo git fetch $CACHE_PATH $REPOREF:fetch_$REPOREF sudo git fetch -q $CACHE_PATH $REPOREF:fetch_$REPOREF
sudo git reset --hard FETCH_HEAD sudo git reset --hard FETCH_HEAD
# Get the sha in use # Get the sha in use
git_sha=$(git rev-parse FETCH_HEAD) git_sha=$(git rev-parse FETCH_HEAD)
fi fi
popd popd > /dev/null
# Write the sha being used into the source-repositories manifest # Write the sha being used into the source-repositories manifest
echo "$REPONAME git $REPOPATH $REPOLOCATION $git_sha" >> $GIT_MANIFEST echo "$REPONAME git $REPOPATH $REPOLOCATION $git_sha" >> $GIT_MANIFEST
@ -225,7 +233,7 @@ function get_repos_for_element(){
# Capture the in-instance repository path for later review / other # Capture the in-instance repository path for later review / other
# elements (like a pypi dependency cache). # elements (like a pypi dependency cache).
echo "$REPOPATH" | sudo dd of=$TMP_MOUNT_PATH/etc/dib-source-repositories oflag=append conv=notrunc echo "$REPOPATH" | sudo tee -a $TMP_MOUNT_PATH/etc/dib-source-repositories > /dev/null
else else
echo "Couldn't parse '$line' as a source repository" echo "Couldn't parse '$line' as a source repository"
@ -248,7 +256,7 @@ rm -f $GIT_MANIFEST
for _SOURCEREPO in $(find $TMP_HOOKS_PATH -maxdepth 1 -name "source-repository-*" -not -name '*~'); do for _SOURCEREPO in $(find $TMP_HOOKS_PATH -maxdepth 1 -name "source-repository-*" -not -name '*~'); do
repo_flock=$CACHE_BASE/repositories_flock repo_flock=$CACHE_BASE/repositories_flock
( (
echo "Getting $repo_flock: $(date)" echo "Getting $repo_flock: $(date) for $_SOURCEREPO"
if ! flock -w 600 9 ; then if ! flock -w 600 9 ; then
echo "Did not get $repo_flock: $(date)" echo "Did not get $repo_flock: $(date)"
exit 1 exit 1