From 74d0185dde623a2948f25b28cd49ad4c044c163c Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Fri, 27 May 2016 05:29:39 +0000 Subject: [PATCH] Cleanup source-repositories output This element takes up a *huge* part of the logs of openstack-infra builds as we go and cache every git tree. This silences most of the noise which will reduce the mess in the logs considerably. Note that we've had logging on this turned down since I91c5e55814ba9840769357261d203f4850e2eba6 but it has been ineffective in stopping the log-spew output, see the dependency change. Change-Id: I60f06f84d57087c82b3907575bff125015d35171 Depends-On: I1e39822f218dc0322e2490a770f3dc867a55802c --- .../extra-data.d/98-source-repositories | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/elements/source-repositories/extra-data.d/98-source-repositories b/elements/source-repositories/extra-data.d/98-source-repositories index 4770bea6..6aef5689 100755 --- a/elements/source-repositories/extra-data.d/98-source-repositories +++ b/elements/source-repositories/extra-data.d/98-source-repositories @@ -36,19 +36,24 @@ function get_repos_for_element(){ local REGEX="^([^ ]+) (git|tar|file|package) ?(/[^ ]+)? ?([^ ]+)? ?([^ ]*)$" + # this can be a rather long list (e.g. caching ~1000 openstack + # repos), so we make a rough count for progress display + local COUNT=1 + local REPO_SOURCES_COUNT=$(grep -v '^$\|^\s*\#' $REPO_SOURCES | wc -l) + while read line; do - # temporarily turn off globbing '*' (since it may be used as the REPOREF for tarballs) - set -f - # expand variables - line=$(eval echo $line) - - # restore globbing - set +f + # temporarily turn off globbing '*' (since it may be used as + # the REPOREF for tarballs) and expand variables + set -f; line=$(eval echo $line); set +f # ignore blank lines and lines beginning in '#' [[ "$line" == \#* ]] || [[ -z "$line" ]] && continue + # rough progress counter + printf "(%04d / %04d)\n" ${COUNT} ${REPO_SOURCES_COUNT} + COUNT=$(( COUNT + 1)) + if [[ "$line" =~ $REGEX ]] ; then local REPONAME=${BASH_REMATCH[1]} local REPOTYPE=${BASH_REMATCH[2]} @@ -114,7 +119,7 @@ function get_repos_for_element(){ if [ ! -e "$CACHE_PATH" ] ; then echo "Caching $REPONAME from $REPOLOCATION in $CACHE_PATH" - git clone $REPOLOCATION $CACHE_PATH.tmp + git clone -q $REPOLOCATION $CACHE_PATH.tmp mv ${CACHE_PATH}{.tmp,} fi @@ -125,7 +130,7 @@ function get_repos_for_element(){ echo "Updating cache of $REPOLOCATION in $CACHE_PATH with ref $REPOREF" # Copy named refs (which might be outside the usual heads # pattern) - e.g. gerrit - if [ "$REPOREF" == "*" ] || ! git --git-dir=$CACHE_PATH/.git fetch --prune --update-head-ok $REPOLOCATION \ + if [ "$REPOREF" == "*" ] || ! git --git-dir=$CACHE_PATH/.git fetch -q --prune --update-head-ok $REPOLOCATION \ +${REPOREF}:${REPOREF} ; then # Copy all heads from the remote repository - this permits # using a SHA1 object reference so long as the object @@ -133,12 +138,15 @@ function get_repos_for_element(){ # not permit arbitrary sha fetching from remote servers. # This is a separate fetch to the prior one as the prior # one will fail when REPOREF is a SHA1. - git --git-dir=$CACHE_PATH/.git fetch --prune --update-head-ok $REPOLOCATION \ + git --git-dir=$CACHE_PATH/.git fetch -q --prune --update-head-ok $REPOLOCATION \ +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/* fi # Ensure that we have a reference to the revision. if [ "$REPOREF" != "*" ] ; then - git --git-dir=$CACHE_PATH/.git rev-parse -q --verify $REPOREF^{commit} + if ! git --git-dir=$CACHE_PATH/.git rev-parse -q --verify $REPOREF^{commit} > /dev/null; then + echo "Failed to find reference to $REPOREF" + exit 1 + fi fi fi @@ -150,7 +158,7 @@ function get_repos_for_element(){ echo "REPOLOCATIONS don't match ("$CACHE_PATH" != "$DESIRED")" >&2 exit 1 elif [[ "$REPOREF" != "*" ]]; then - pushd $REPO_DEST + pushd $REPO_DEST > /dev/null # When we first clone we create a branch naming what we fetched # that must match, or we are asking for two different references from the # same repo, which is an error @@ -158,22 +166,22 @@ function get_repos_for_element(){ echo "REPOREFS don't match - failed to get sha1 of fetch_$REPOREF" >&2 exit 1 fi - popd + popd > /dev/null fi else - sudo git clone $CACHE_PATH $REPO_DEST - pushd $REPO_DEST + sudo git clone -q $CACHE_PATH $REPO_DEST + pushd $REPO_DEST > /dev/null if [[ "$REPOREF" == "*" ]]; then - sudo git fetch --prune --update-head-ok $CACHE_PATH \ + sudo git fetch -q --prune --update-head-ok $CACHE_PATH \ +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/* git_sha=$(git rev-parse HEAD) else - sudo git fetch $CACHE_PATH $REPOREF:fetch_$REPOREF + sudo git fetch -q $CACHE_PATH $REPOREF:fetch_$REPOREF sudo git reset --hard FETCH_HEAD # Get the sha in use git_sha=$(git rev-parse FETCH_HEAD) fi - popd + popd > /dev/null # Write the sha being used into the source-repositories manifest echo "$REPONAME git $REPOPATH $REPOLOCATION $git_sha" >> $GIT_MANIFEST @@ -225,7 +233,7 @@ function get_repos_for_element(){ # Capture the in-instance repository path for later review / other # elements (like a pypi dependency cache). - echo "$REPOPATH" | sudo dd of=$TMP_MOUNT_PATH/etc/dib-source-repositories oflag=append conv=notrunc + echo "$REPOPATH" | sudo tee -a $TMP_MOUNT_PATH/etc/dib-source-repositories > /dev/null else echo "Couldn't parse '$line' as a source repository" @@ -248,7 +256,7 @@ rm -f $GIT_MANIFEST for _SOURCEREPO in $(find $TMP_HOOKS_PATH -maxdepth 1 -name "source-repository-*" -not -name '*~'); do repo_flock=$CACHE_BASE/repositories_flock ( - echo "Getting $repo_flock: $(date)" + echo "Getting $repo_flock: $(date) for $_SOURCEREPO" if ! flock -w 600 9 ; then echo "Did not get $repo_flock: $(date)" exit 1