f60dd38482
This adds a devstack-inspired output filter to standardise timestamping. Currently, python tools timestamp always (timestamp setup in logging_config.py) but all the surrounding bash does not. We have extra timestamps added in run_functests.sh for our own purposes to get the bash timestamps; but this ends up giving us double-timestamps for the python bits. Additionally, callers such as nodepool capture our output and put their own timestamps on it, and again have the double-timestamps. This uses a lightly modified outfilter.py from devstack to standardise this. All output is run through this filter, which will timestamp it. I have removed the places where we double-timestamp -- logging_config.py and the prefix in dib-run-parts. An env option is added to turn timestamps off completely (does not seem worth taking up a command-line option for). For callers like nodepool, they can set this and will just have their own timestamps as they collect the lines. Since all logging is going through outfilter, it's easy to add a --logfile option. I think this will be quite handy; personally I'm always redirecting dib runs to files for debugging. I've also added a "quiet" option. I think this could be useful in run_tests.sh if we were to start logging the output of each test to individual files. This would be much easier to deal with than the very large log files we get (especially if we wanted to turn on parallel running...) Change-Id: I202e1cb200bde17f6d7770cf1e2710bbf4cca64c
145 lines
4.0 KiB
Bash
Executable File
145 lines
4.0 KiB
Bash
Executable File
#!/bin/bash
|
|
# Inspired by Debian and RedHat run-parts but portable and specific to di-b.
|
|
#
|
|
# Copyright 2012 Hewlett-Packard Development Company, L.P.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
allowed_regex=${RUN_PARTS_REGEX:-"^[0-9A-Za-z_-]+$"}
|
|
show_list=
|
|
|
|
set -ue
|
|
set -o pipefail
|
|
|
|
name=$(basename $0)
|
|
|
|
usage() {
|
|
echo "Usage: $name [OPTION] scripts_directory"
|
|
echo "Option:"
|
|
echo " --list print names of all valid files"
|
|
echo
|
|
echo "Examples:"
|
|
echo " dib-run-parts --list /opt/stack/os-config-refresh/configure.d/"
|
|
echo " dib-run-parts /opt/stack/os-config-refresh/configure.d/"
|
|
exit 1
|
|
} >&2
|
|
|
|
|
|
output_prefix() {
|
|
printf "%s " "${name}" >&2
|
|
}
|
|
|
|
output () {
|
|
output_prefix
|
|
echo $* >&2
|
|
}
|
|
|
|
output_printf () {
|
|
local FORMAT="$1"
|
|
shift
|
|
output_prefix
|
|
printf "${FORMAT}" $@ >&2
|
|
}
|
|
|
|
# source the environment files from environment.d
|
|
# arg : target_dir
|
|
source_environment() {
|
|
|
|
local dir=$target_dir/../environment.d
|
|
local env_files
|
|
local xtrace
|
|
|
|
if [ -d ${dir} ] ; then
|
|
env_files=$(find ${dir} -maxdepth 1 -xtype f | \
|
|
grep -E "/[0-9A-Za-z_\.-]+$" | \
|
|
LANG=C sort -n)
|
|
for env_file in $env_files ; do
|
|
output "Sourcing environment file ${env_file}"
|
|
# Set tracing as we import these environment files; it's
|
|
# nice to see the definitions in the logs
|
|
xtrace=$(set +o | grep xtrace)
|
|
set -o xtrace
|
|
source $env_file
|
|
$xtrace
|
|
done
|
|
fi
|
|
}
|
|
|
|
if [ $# -lt 1 ] ; then
|
|
usage
|
|
fi
|
|
|
|
if [ "$1" == "--list" ] ; then
|
|
show_list="1"
|
|
shift
|
|
fi
|
|
|
|
target_dir="${1:-}"
|
|
|
|
if ! [ -d "$target_dir" ] ; then
|
|
output "Scripts directory [$target_dir] must exist and be a directory"
|
|
usage
|
|
fi
|
|
|
|
# We specifically only want to sort *by the numbers*.
|
|
# Lexical sorting is not guaranteed, and identical numbers may be
|
|
# parallelized later
|
|
# Note: -maxdepth 1 ensures only files in the target directory (but not
|
|
# subdirectories) are run, which is the way run-parts behaves.
|
|
targets=$(find $target_dir -maxdepth 1 -xtype f -executable -printf '%f\n' | grep -E "$allowed_regex" | LANG=C sort -n || echo "")
|
|
|
|
if [ "$show_list" == "1" ] ; then
|
|
for target in $targets ; do
|
|
echo "${target_dir}/${target}"
|
|
done
|
|
exit 0
|
|
fi
|
|
|
|
PROFILE_DIR=$(mktemp -d --tmpdir profiledir.XXXXXX)
|
|
|
|
# note, run this in a sub-shell so we don't pollute our
|
|
# own environment with source_environment
|
|
(
|
|
source_environment
|
|
|
|
for target in $targets ; do
|
|
output "Running $target_dir/$target"
|
|
target_tag=${target//\//_}
|
|
date +%s.%N > $PROFILE_DIR/start_$target_tag
|
|
$target_dir/$target
|
|
target_tag=${target//\//_}
|
|
date +%s.%N > $PROFILE_DIR/stop_$target_tag
|
|
output "$target completed"
|
|
done
|
|
)
|
|
|
|
output "----------------------- PROFILING -----------------------"
|
|
output ""
|
|
output "Target: $(basename $target_dir)"
|
|
output ""
|
|
output_printf "%-40s %9s\n" Script Seconds
|
|
output_printf "%-40s %9s\n" --------------------------------------- ----------
|
|
output ""
|
|
pushd $PROFILE_DIR > /dev/null
|
|
for target in $(find . -name 'start_*' -printf '%f\n' | env LC_ALL=C sort -n) ; do
|
|
stop_file=stop_${target##start_}
|
|
start_seconds=$(cat $target)
|
|
stop_seconds=$(cat $stop_file)
|
|
duration=$(echo - | awk "{ print $stop_seconds - $start_seconds }")
|
|
LC_NUMERIC=C LC_ALL=C output_printf "%-40s %10.3f\n" ${target##start_} $duration
|
|
done
|
|
popd > /dev/null
|
|
rm -rf $PROFILE_DIR
|
|
output ""
|
|
output "--------------------- END PROFILING ---------------------"
|