0c94eef7be
This reverts commit6ee2995214ande85c2a6f03. I missed that if you pip install and then run dib-lint, it's not going to pick up the .yamllint file shipped here. Thus it gives spurious errors. The reason for this was simply better duplicate key detection in yaml files, which caused us problems with the kernel installs. However, at this point it seems just the old "does it load" test from pyyaml will be enough. Change-Id: I87a9fc9bb119cfeffad48fc0fa0df31f0181825d
315 lines
9.6 KiB
Bash
Executable File
315 lines
9.6 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Copyright 2014 Red Hat, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# This script checks all files in the "elements" directory for some
|
|
# common mistakes and exits with a non-zero status if it finds any.
|
|
|
|
set -eu
|
|
set -o pipefail
|
|
|
|
ELEMENTS_DIR=${ELEMENTS_DIR:-diskimage_builder/elements}
|
|
LIB_DIR=${LIB_DIR:-diskimage_builder/lib}
|
|
|
|
parse_exclusions() {
|
|
# Per-file exclusions
|
|
# Example: # dib-lint: disable=sete setpipefail
|
|
local filename=$1
|
|
local disable_pattern="# dib-lint: disable="
|
|
local exclusions=$(grep "^$disable_pattern.*$" $filename | sed "s/$disable_pattern//g")
|
|
|
|
# Global exclusions read from tox.ini
|
|
# Example section in tox.ini:
|
|
# [dib-lint]
|
|
# ignore = sete setu
|
|
section="dib-lint"
|
|
option="ignore"
|
|
global_exclusions=$(python - <<EOF
|
|
try:
|
|
import configparser
|
|
except ImportError:
|
|
import ConfigParser as configparser
|
|
conf=configparser.ConfigParser()
|
|
conf.read('tox.ini')
|
|
print(conf.get('$section', '$option')) if conf.has_option('$section', '$option') else ''
|
|
EOF
|
|
)
|
|
echo $exclusions $global_exclusions
|
|
}
|
|
|
|
excluded() {
|
|
local test_name=$1
|
|
for e in $exclusions; do
|
|
if [ "$e" = "$test_name" ]; then
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
error() {
|
|
echo -e "ERROR: $1"
|
|
rc=1
|
|
}
|
|
|
|
echo "Running dib-lint in $(pwd)"
|
|
|
|
rc=0
|
|
TMPDIR=$(mktemp -d /tmp/tmp.XXXXXXXXXX)
|
|
trap "rm -rf $TMPDIR" EXIT
|
|
|
|
# Elements have a README.rst file
|
|
for i in $(find $ELEMENTS_DIR -mindepth 1 -maxdepth 1 -type d); do
|
|
[ -f "${i}/README.rst" ] || error "$i lacks a README.rst"
|
|
done
|
|
|
|
# note .py files are run through flake8 directly in tox.ini
|
|
for i in $(find $ELEMENTS_DIR -type f \
|
|
-not -name \*.rst \
|
|
-not -name \*.yaml \
|
|
-not -name \*.py); do
|
|
|
|
# Skip files in .gitignore
|
|
if git check-ignore -q "$i" ; then
|
|
echo Skipping $i
|
|
continue
|
|
fi
|
|
echo "Checking $i"
|
|
|
|
exclusions=("$(parse_exclusions $i)")
|
|
|
|
# source-repository does a read < $file and can miss the last line
|
|
# (or only line, if there's only one) when not newline terminated.
|
|
if [[ $(basename "${i}") =~ "source-repository-" ]]; then
|
|
nl=$(tail -c 1 ${i})
|
|
if [[ "${nl}" != "" ]]; then
|
|
error "$i does not end with a newline"
|
|
fi
|
|
fi
|
|
|
|
# Check that files starting with a shebang are +x
|
|
firstline=$(head -n 1 "$i")
|
|
if [ "${firstline:0:2}" = "#!" ]; then
|
|
if [ ! -x "$i" ] && ! excluded executable; then
|
|
error "$i is not executable"
|
|
fi
|
|
|
|
# run flake8 over python files that don't have .py. Note our
|
|
# "dib-python" interpreter can confuse the magic matching
|
|
# being done in "file" and make it think the file is not
|
|
# python; special-case it.
|
|
if [[ "$(file -b -k --mime-type $i)" =~ "text/x-python" ]] || \
|
|
[[ $firstline =~ "dib-python" ]]; then
|
|
flake8 $i || error "$i failed flake8"
|
|
else
|
|
# Ensure 4 spaces indent are used
|
|
if ! excluded indent ; then
|
|
indent_regex='^\( \{4\}\)* \{1,3\}[^ ]'
|
|
if grep -q "$indent_regex" ${i}; then
|
|
error "$i should use 4 spaces indent"
|
|
# outline the failing lines with line number
|
|
grep -n "$indent_regex" ${i}
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Check alphabetical ordering of element-deps
|
|
if [ $(basename $i) = "element-deps" ]; then
|
|
UNSORTED=${TMPDIR}/element-deps.unsorted
|
|
SORTED=${TMPDIR}/element-deps.sorted
|
|
grep -v -e '^#' -e '^$' $i > ${UNSORTED}
|
|
sort ${UNSORTED} > ${SORTED}
|
|
if [ -n "$(diff -c ${UNSORTED} ${SORTED})" ]; then
|
|
error "$i is not sorted alphabetically"
|
|
diff -y ${UNSORTED} ${SORTED}
|
|
fi
|
|
fi
|
|
|
|
# for consistency, let's just use #!/bin/bash everywhere (not
|
|
# /usr/bin/env, etc)
|
|
regex='^#!.*bash'
|
|
if [[ "$firstline" =~ $regex &&
|
|
"$firstline" != "#!/bin/bash" ]]; then
|
|
error "$i : only use #!/bin/bash for scripts"
|
|
fi
|
|
|
|
# Check that all scripts are set -eu -o pipefail and look for
|
|
# DIB_DEBUG_TRACE
|
|
# NOTE(bnemec): This doesn't verify that the set call occurs high
|
|
# enough in the file to be useful, but hopefully nobody will be
|
|
# sticking set calls at the end of their file to trick us. And if
|
|
# they are, that's easy enough to catch in reviews.
|
|
# Also, this is only going to check bash scripts - we've decided to
|
|
# explicitly require bash for any scripts that don't have a specific
|
|
# need to run under other shells, and any exceptions to that rule
|
|
# may not want these checks either.
|
|
if [[ "$firstline" =~ '#!/bin/bash' ]]; then
|
|
if ! excluded sete; then
|
|
if [ -z "$(grep "^set -[^ ]*e" $i)" ]; then
|
|
error "$i is not set -e"
|
|
fi
|
|
fi
|
|
if ! excluded setu; then
|
|
if [ -z "$(grep "^set -[^ ]*u" $i)" ]; then
|
|
error "$i is not set -u"
|
|
fi
|
|
fi
|
|
if ! excluded setpipefail; then
|
|
if [ -z "$(grep "^set -o pipefail" $i)" ]; then
|
|
error "$i is not set -o pipefail"
|
|
fi
|
|
fi
|
|
if ! excluded dibdebugtrace; then
|
|
if [ -z "$(grep "DIB_DEBUG_TRACE" $i)" ]; then
|
|
error "$i does not follow DIB_DEBUG_TRACE"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# check that environment files don't "set -x" and they have no executable
|
|
# bits set
|
|
if [[ "$i" =~ (environment.d) ]]; then
|
|
if grep -q "set -x" $i; then
|
|
error "Environment file $i should not set tracing"
|
|
fi
|
|
if [[ -x $i ]]; then
|
|
error "Environment file $i should not be marked as executable"
|
|
fi
|
|
fi
|
|
|
|
# check for
|
|
# export FOO=$(bar)
|
|
# calls. These are dangerous, because the export hides the return
|
|
# code of the $(bar) call. Split this into 2 lines and -e will
|
|
# fail on the assignment
|
|
if grep -q 'export .*\$(' $i; then
|
|
error "Split export and assignments in $i"
|
|
fi
|
|
|
|
# check that sudo calls in phases run outside the chroot look
|
|
# "safe"; meaning that they seem to operate within the chroot
|
|
# somehow. This is not fool-proof, but catches egregious errors,
|
|
# and makes you think about it if you're doing something outside
|
|
# the box.
|
|
if ! excluded safe_sudo; then
|
|
if [[ $(dirname $i) =~ (root.d|extra-data.d|block-device.d|finalise.d|cleanup.d) ]]; then
|
|
while read LINE
|
|
do
|
|
if [[ $LINE =~ "sudo " ]]; then
|
|
# messy regex ahead! Don't match:
|
|
# - explicitly ignored
|
|
# - basic comments
|
|
# - install-packages ... sudo ...
|
|
# - any of the paths passed into the out-of-chroot elements
|
|
if [[ $LINE =~ (dib-lint: safe_sudo|^#|install-packages|TARGET_ROOT|IMAGE_BLOCK_DEVICE|TMP_MOUNT_PATH|TMP_IMAGE_PATH) ]]; then
|
|
continue
|
|
fi
|
|
error "$i : potentially unsafe sudo\n -- $LINE"
|
|
fi
|
|
done < $i
|
|
fi
|
|
fi
|
|
|
|
# check that which calls are not used. It is not built in and is missing
|
|
# from some constrained environments
|
|
if ! excluded which; then
|
|
while read LINE
|
|
do
|
|
if [[ $LINE =~ "which " ]]; then
|
|
# Don't match:
|
|
# - explicitly ignored
|
|
# - commented
|
|
if [[ $LINE =~ (dib-lint: which|^#) ]]; then
|
|
continue
|
|
fi
|
|
error "$i : potential use of which\n -- $LINE"
|
|
fi
|
|
done < $i
|
|
fi
|
|
|
|
done
|
|
|
|
echo "Checking indents..."
|
|
|
|
for i in $(find $ELEMENTS_DIR -type f -and -name '*.rst' -or -type f -executable) \
|
|
$(find $LIB_DIR -type f); do
|
|
# Skip files in .gitignore
|
|
if git check-ignore -q "$i" ; then
|
|
echo Skipping $i
|
|
continue
|
|
fi
|
|
# Check for tab indentation
|
|
if ! excluded tabindent; then
|
|
if grep -q $'^ *\t' ${i}; then
|
|
error "$i contains tab characters"
|
|
fi
|
|
fi
|
|
|
|
if ! excluded newline; then
|
|
if [ "$(tail -c 1 $i)" != "" ]; then
|
|
error "No newline at end of file: $i"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if ! excluded mddocs; then
|
|
md_docs=$(find $ELEMENTS_DIR -name '*.md')
|
|
if [ -n "$md_docs" ]; then
|
|
error ".md docs found: $md_docs"
|
|
fi
|
|
fi
|
|
|
|
echo "Checking YAML parsing..."
|
|
for i in $(find $ELEMENTS_DIR -type f -name '*.yaml'); do
|
|
echo "Parsing $i"
|
|
py_check="
|
|
import yaml
|
|
import sys
|
|
try:
|
|
objs = yaml.safe_load(open('$i'))
|
|
except yaml.parser.ParserError:
|
|
sys.exit(1)
|
|
"
|
|
if ! python -c "$py_check"; then
|
|
error "$i is not a valid YAML file"
|
|
fi
|
|
done
|
|
echo "Checking pkg-map files..."
|
|
for i in $(find $ELEMENTS_DIR -type f \
|
|
-name 'pkg-map' -a \! -executable); do
|
|
echo "Parsing $i"
|
|
py_check="
|
|
import json
|
|
import sys
|
|
try:
|
|
objs = json.load(open('$i'))
|
|
except ValueError:
|
|
sys.exit(1)
|
|
"
|
|
if ! python -c "$py_check"; then
|
|
error "$i is not a valid JSON file"
|
|
fi
|
|
done
|
|
|
|
if [[ $rc == 0 ]]; then
|
|
echo "PASS"
|
|
else
|
|
echo "*** FAIL: Some tests failed!"
|
|
fi
|
|
|
|
exit $rc
|