Cleanup source-repositories output
This element takes up a *huge* part of the logs of openstack-infra builds as we go and cache every git tree. This silences most of the noise which will reduce the mess in the logs considerably. Note that we've had logging on this turned down since I91c5e55814ba9840769357261d203f4850e2eba6 but it has been ineffective in stopping the log-spew output, see the dependency change. Change-Id: I60f06f84d57087c82b3907575bff125015d35171 Depends-On: I1e39822f218dc0322e2490a770f3dc867a55802c
This commit is contained in:
Ian Wienand
parent
f0b57d5efd
commit
74d0185dde
1 changed files with 28 additions and 20 deletions
|
|
@ -36,19 +36,24 @@ function get_repos_for_element(){
|
|||
|
||||
local REGEX="^([^ ]+) (git|tar|file|package) ?(/[^ ]+)? ?([^ ]+)? ?([^ ]*)$"
|
||||
|
||||
# this can be a rather long list (e.g. caching ~1000 openstack
|
||||
# repos), so we make a rough count for progress display
|
||||
local COUNT=1
|
||||
local REPO_SOURCES_COUNT=$(grep -v '^$\|^\s*\#' $REPO_SOURCES | wc -l)
|
||||
|
||||
while read line; do
|
||||
# temporarily turn off globbing '*' (since it may be used as the REPOREF for tarballs)
|
||||
set -f
|
||||
|
||||
# expand variables
|
||||
line=$(eval echo $line)
|
||||
|
||||
# restore globbing
|
||||
set +f
|
||||
# temporarily turn off globbing '*' (since it may be used as
|
||||
# the REPOREF for tarballs) and expand variables
|
||||
set -f; line=$(eval echo $line); set +f
|
||||
|
||||
# ignore blank lines and lines beginning in '#'
|
||||
[[ "$line" == \#* ]] || [[ -z "$line" ]] && continue
|
||||
|
||||
# rough progress counter
|
||||
printf "(%04d / %04d)\n" ${COUNT} ${REPO_SOURCES_COUNT}
|
||||
COUNT=$(( COUNT + 1))
|
||||
|
||||
if [[ "$line" =~ $REGEX ]] ; then
|
||||
local REPONAME=${BASH_REMATCH[1]}
|
||||
local REPOTYPE=${BASH_REMATCH[2]}
|
||||
|
|
@ -114,7 +119,7 @@ function get_repos_for_element(){
|
|||
|
||||
if [ ! -e "$CACHE_PATH" ] ; then
|
||||
echo "Caching $REPONAME from $REPOLOCATION in $CACHE_PATH"
|
||||
git clone $REPOLOCATION $CACHE_PATH.tmp
|
||||
git clone -q $REPOLOCATION $CACHE_PATH.tmp
|
||||
mv ${CACHE_PATH}{.tmp,}
|
||||
fi
|
||||
|
||||
|
|
@ -125,7 +130,7 @@ function get_repos_for_element(){
|
|||
echo "Updating cache of $REPOLOCATION in $CACHE_PATH with ref $REPOREF"
|
||||
# Copy named refs (which might be outside the usual heads
|
||||
# pattern) - e.g. gerrit
|
||||
if [ "$REPOREF" == "*" ] || ! git --git-dir=$CACHE_PATH/.git fetch --prune --update-head-ok $REPOLOCATION \
|
||||
if [ "$REPOREF" == "*" ] || ! git --git-dir=$CACHE_PATH/.git fetch -q --prune --update-head-ok $REPOLOCATION \
|
||||
+${REPOREF}:${REPOREF} ; then
|
||||
# Copy all heads from the remote repository - this permits
|
||||
# using a SHA1 object reference so long as the object
|
||||
|
|
@ -133,12 +138,15 @@ function get_repos_for_element(){
|
|||
# not permit arbitrary sha fetching from remote servers.
|
||||
# This is a separate fetch to the prior one as the prior
|
||||
# one will fail when REPOREF is a SHA1.
|
||||
git --git-dir=$CACHE_PATH/.git fetch --prune --update-head-ok $REPOLOCATION \
|
||||
git --git-dir=$CACHE_PATH/.git fetch -q --prune --update-head-ok $REPOLOCATION \
|
||||
+refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*
|
||||
fi
|
||||
# Ensure that we have a reference to the revision.
|
||||
if [ "$REPOREF" != "*" ] ; then
|
||||
git --git-dir=$CACHE_PATH/.git rev-parse -q --verify $REPOREF^{commit}
|
||||
if ! git --git-dir=$CACHE_PATH/.git rev-parse -q --verify $REPOREF^{commit} > /dev/null; then
|
||||
echo "Failed to find reference to $REPOREF"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
|
|
@ -150,7 +158,7 @@ function get_repos_for_element(){
|
|||
echo "REPOLOCATIONS don't match ("$CACHE_PATH" != "$DESIRED")" >&2
|
||||
exit 1
|
||||
elif [[ "$REPOREF" != "*" ]]; then
|
||||
pushd $REPO_DEST
|
||||
pushd $REPO_DEST > /dev/null
|
||||
# When we first clone we create a branch naming what we fetched
|
||||
# that must match, or we are asking for two different references from the
|
||||
# same repo, which is an error
|
||||
|
|
@ -158,22 +166,22 @@ function get_repos_for_element(){
|
|||
echo "REPOREFS don't match - failed to get sha1 of fetch_$REPOREF" >&2
|
||||
exit 1
|
||||
fi
|
||||
popd
|
||||
popd > /dev/null
|
||||
fi
|
||||
else
|
||||
sudo git clone $CACHE_PATH $REPO_DEST
|
||||
pushd $REPO_DEST
|
||||
sudo git clone -q $CACHE_PATH $REPO_DEST
|
||||
pushd $REPO_DEST > /dev/null
|
||||
if [[ "$REPOREF" == "*" ]]; then
|
||||
sudo git fetch --prune --update-head-ok $CACHE_PATH \
|
||||
sudo git fetch -q --prune --update-head-ok $CACHE_PATH \
|
||||
+refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*
|
||||
git_sha=$(git rev-parse HEAD)
|
||||
else
|
||||
sudo git fetch $CACHE_PATH $REPOREF:fetch_$REPOREF
|
||||
sudo git fetch -q $CACHE_PATH $REPOREF:fetch_$REPOREF
|
||||
sudo git reset --hard FETCH_HEAD
|
||||
# Get the sha in use
|
||||
git_sha=$(git rev-parse FETCH_HEAD)
|
||||
fi
|
||||
popd
|
||||
popd > /dev/null
|
||||
|
||||
# Write the sha being used into the source-repositories manifest
|
||||
echo "$REPONAME git $REPOPATH $REPOLOCATION $git_sha" >> $GIT_MANIFEST
|
||||
|
|
@ -225,7 +233,7 @@ function get_repos_for_element(){
|
|||
|
||||
# Capture the in-instance repository path for later review / other
|
||||
# elements (like a pypi dependency cache).
|
||||
echo "$REPOPATH" | sudo dd of=$TMP_MOUNT_PATH/etc/dib-source-repositories oflag=append conv=notrunc
|
||||
echo "$REPOPATH" | sudo tee -a $TMP_MOUNT_PATH/etc/dib-source-repositories > /dev/null
|
||||
|
||||
else
|
||||
echo "Couldn't parse '$line' as a source repository"
|
||||
|
|
@ -248,7 +256,7 @@ rm -f $GIT_MANIFEST
|
|||
for _SOURCEREPO in $(find $TMP_HOOKS_PATH -maxdepth 1 -name "source-repository-*" -not -name '*~'); do
|
||||
repo_flock=$CACHE_BASE/repositories_flock
|
||||
(
|
||||
echo "Getting $repo_flock: $(date)"
|
||||
echo "Getting $repo_flock: $(date) for $_SOURCEREPO"
|
||||
if ! flock -w 600 9 ; then
|
||||
echo "Did not get $repo_flock: $(date)"
|
||||
exit 1
|
||||
|
|
|
|||
Loading…
Reference in a new issue