Merge "Add timestamp output filter"

This commit is contained in:
Jenkins 2017-10-09 15:56:31 +00:00 committed by Gerrit Code Review
commit 7a76270227
7 changed files with 138 additions and 8 deletions

View File

@ -36,7 +36,7 @@ usage() {
output_prefix() { output_prefix() {
printf "%s %s " "${name}" "$(date)" >&2 printf "%s " "${name}" >&2
} }
output () { output () {

View File

@ -59,6 +59,7 @@ function show_options () {
echo " -x -- turn on tracing (use -x -x for very detailed tracing)." echo " -x -- turn on tracing (use -x -x for very detailed tracing)."
echo " -u -- uncompressed; do not compress the image - larger but faster" echo " -u -- uncompressed; do not compress the image - larger but faster"
echo " -c -- clear environment before starting work" echo " -c -- clear environment before starting work"
echo " --logfile -- save run output to given logfile"
echo " --checksum -- generate MD5 and SHA256 checksum files for the created image" echo " --checksum -- generate MD5 and SHA256 checksum files for the created image"
echo " --image-size size -- image size in GB for the created image" echo " --image-size size -- image size in GB for the created image"
echo " --image-cache directory -- location for cached images(default ~/.cache/image-create)" echo " --image-cache directory -- location for cached images(default ~/.cache/image-create)"
@ -88,7 +89,12 @@ function show_options () {
echo " -h|--help -- display this help and exit" echo " -h|--help -- display this help and exit"
echo " --version -- display version and exit" echo " --version -- display version and exit"
echo echo
echo "ELEMENTS_PATH will allow you to specify multiple locations for the elements." echo "Environment Variables:"
echo " (this is not a complete list)"
echo
echo " * ELEMENTS_PATH: specify external locations for the elements. As for \$PATH"
echo " * DIB_NO_TIMESTAMP: no timestamp prefix on output. Useful if capturing output"
echo " * DIB_QUIET: do not output log output to stdout. See --logfile"
echo echo
echo "NOTE: At least one distribution root element must be specified." echo "NOTE: At least one distribution root element must be specified."
echo echo
@ -120,7 +126,8 @@ DIB_DEFAULT_INSTALLTYPE=${DIB_DEFAULT_INSTALLTYPE:-"source"}
MKFS_OPTS="" MKFS_OPTS=""
ACI_MANIFEST=${ACI_MANIFEST:-} ACI_MANIFEST=${ACI_MANIFEST:-}
DOCKER_TARGET="" DOCKER_TARGET=""
TEMP=`getopt -o a:ho:t:xucnp: -l checksum,no-tmpfs,offline,help,version,min-tmpfs:,image-size:,image-cache:,max-online-resize:,mkfs-options:,qemu-img-options:,ramdisk-element:,root-label:,install-type:,docker-target: -n $SCRIPTNAME -- "$@"` LOGFILE=""
TEMP=`getopt -o a:ho:t:xucnp: -l checksum,no-tmpfs,offline,help,version,min-tmpfs:,image-size:,image-cache:,max-online-resize:,mkfs-options:,qemu-img-options:,ramdisk-element:,root-label:,install-type:,docker-target:,logfile: -n $SCRIPTNAME -- "$@"`
if [ $? -ne 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi if [ $? -ne 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi
# Note the quotes around `$TEMP': they are essential! # Note the quotes around `$TEMP': they are essential!
@ -151,6 +158,7 @@ while true ; do
--ramdisk-element) RAMDISK_ELEMENT=$2; shift 2;; --ramdisk-element) RAMDISK_ELEMENT=$2; shift 2;;
--install-type) DIB_DEFAULT_INSTALLTYPE=$2; shift 2;; --install-type) DIB_DEFAULT_INSTALLTYPE=$2; shift 2;;
--docker-target) export DOCKER_TARGET=$2; shift 2 ;; --docker-target) export DOCKER_TARGET=$2; shift 2 ;;
--logfile) export LOGFILE=$2; shift 2 ;;
--) shift ; break ;; --) shift ; break ;;
*) echo "Internal error!" ; exit 1 ;; *) echo "Internal error!" ; exit 1 ;;
esac esac
@ -167,6 +175,23 @@ if [ "$CLEAR_ENV" = "1" -a "$HOME" != "" ]; then
exec -c $0 "$@" exec -c $0 "$@"
fi fi
# We send stdout & stderr through "outfilter" which does timestamping,
# basic filtering and log file output.
_TS_FLAG=""
if [[ "${DIB_NO_TIMESTAMP:-0}" -eq 1 ]]; then
_TS_FLAG="--no-timestamp"
fi
_QUIET_FLAG="-v"
if [[ "${DIB_QUIET:-0}" -eq 1 ]]; then
_QUIET_FLAG=""
fi
_LOGFILE_FLAG=""
if [[ -n "${LOGFILE}" ]]; then
echo "Output logs going to: ${LOGFILE}"
_LOGFILE_FLAG="-o ${LOGFILE}"
fi
exec 1> >( ${DIB_PYTHON_EXEC:-python} $_LIB/outfilter.py ${_TS_FLAG} ${_QUIET_FLAG} ${_LOGFILE_FLAG} ) 2>&1
# Display the current file/function/line in the debug output # Display the current file/function/line in the debug output
function _ps4 { function _ps4 {

View File

@ -0,0 +1,89 @@
#!/usr/bin/env python
#
# Copyright 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is an output filter to filter and timestamp the logs from Grenade and
# DevStack. Largely our awk filters got beyond the complexity level which were
# sustainable, so this provides us much more control in a single place.
#
# The overhead of running python should be less than execing `date` a million
# times during a run.
import argparse
import datetime
import re
import sys
IGNORE_LINES = re.compile('(set \+o|xtrace)')
def get_options():
parser = argparse.ArgumentParser(
description='Filter output by DevStack and friends')
parser.add_argument('-o', '--outfile',
help='Output file for content',
default=None)
parser.add_argument('-v', '--verbose', action='store_true',
help='Write to stdout',
default=False)
parser.add_argument('-b', '--no-timestamp', action='store_true',
help='Do not prefix stdout with timestamp (bare)',
default=False)
return parser.parse_args()
def skip_line(line):
"""Should we skip this line."""
return IGNORE_LINES.search(line) is not None
def main():
opts = get_options()
outfile = None
if opts.outfile:
outfile = open(opts.outfile, 'ab', 0)
# Otherwise fileinput reprocess args as files
sys.argv = []
for line in iter(sys.stdin.readline, ''):
# put skip lines here
if skip_line(line):
continue
now = datetime.datetime.utcnow()
ts_line = ("%s | %s" % (
now.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3],
line))
if opts.verbose:
sys.stdout.write(line if opts.no_timestamp else ts_line)
sys.stdout.flush()
if outfile:
# We've opened outfile as a binary file to get the
# non-buffered behaviour. on python3, sys.stdin was
# opened with the system encoding and made the line into
# utf-8, so write the logfile out in utf-8 bytes.
if sys.version_info < (3,):
outfile.write(ts_line)
else:
outfile.write(ts_line.encode('utf-8'))
outfile.flush()
if __name__ == '__main__':
try:
sys.exit(main())
except KeyboardInterrupt:
sys.exit(1)

View File

@ -65,11 +65,13 @@ def setup():
'version': 1, 'version': 1,
'disable_existing_loggers': False, 'disable_existing_loggers': False,
# note that disk-image-create runs stdout through
# outfilter.py, which adds the timestamp. this doesn't have a
# timestamp to avoid double logging
'formatters': { 'formatters': {
'standard': { 'standard': {
'()': 'diskimage_builder.logging_config.DibFormatter', '()': 'diskimage_builder.logging_config.DibFormatter',
'datefmt': '%Y-%m-%d %H:%M:%S', 'fmt': '%(levelname)s '
'fmt': '%(asctime)s.%(msecs)03d %(levelname)s '
'%(name)s [-] %(message)s', '%(name)s [-] %(message)s',
'debug_suffix': ' %(funcName)s %(pathname)s:%(lineno)d' 'debug_suffix': ' %(funcName)s %(pathname)s:%(lineno)d'
} }

View File

@ -0,0 +1,14 @@
---
features:
- A ``--logfile`` option is added to save output to a given file.
- By default, all ``stdout`` will log with timestamps (this used to
be inconsistent; python tools logged with a timestamp, and bash
parts did not). If you set ``DIB_NO_TIMESTAMP`` to ``1`` the
timestamp prefix will be suppressed (this would be appropriate if
you are running diskimage-builder and capturing its output, and
adding your own timestamp). Note that output to ``--logfile``
will always be timestamped.
- You can set ``DIB_QUIET`` to ``1`` to suppress all output. This
is likely only useful when used with the ``--logfile`` command.

View File

@ -29,6 +29,8 @@ autodoc_exclude_modules =
diskimage_builder.block_device.tests.* diskimage_builder.block_device.tests.*
# elements are a weird mix of scripts & python # elements are a weird mix of scripts & python
diskimage_builder.elements.* diskimage_builder.elements.*
# this is non-pythony stuff
diskimage_builder.lib.*
[files] [files]
packages = packages =

View File

@ -63,9 +63,7 @@ function log_with_prefix {
while read a; do while read a; do
log="[$pr] $a" log="[$pr] $a"
if [[ ${LOG_DATESTAMP} -ne 0 ]]; then # note: dib logs have timestamp by default now
log="$(date +"%Y%m%d-%H%M%S.%N") ${log}"
fi
echo "${log}" echo "${log}"
done done
} }