diskimage-builder/bin/dib-lint
Ian Wienand eed1be9992 dib-lint: match text/x-script.python
It seems libmagic changed it's Python output to
"text/x-script.python", which I see on Fedora.  Handle this too so we
detect Python files correctly.

[1] eb373e431c

Change-Id: I35992c70523a8f2bc5efff2e5167ed1ac1514d34
2021-05-10 14:29:51 +10:00

315 lines
9.6 KiB
Bash
Executable File

#!/bin/bash
# Copyright 2014 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script checks all files in the "elements" directory for some
# common mistakes and exits with a non-zero status if it finds any.
set -eu
set -o pipefail
ELEMENTS_DIR=${ELEMENTS_DIR:-diskimage_builder/elements}
LIB_DIR=${LIB_DIR:-diskimage_builder/lib}
parse_exclusions() {
# Per-file exclusions
# Example: # dib-lint: disable=sete setpipefail
local filename=$1
local disable_pattern="# dib-lint: disable="
local exclusions=$(grep "^$disable_pattern.*$" $filename | sed "s/$disable_pattern//g")
# Global exclusions read from tox.ini
# Example section in tox.ini:
# [dib-lint]
# ignore = sete setu
section="dib-lint"
option="ignore"
global_exclusions=$(python - <<EOF
try:
import configparser
except ImportError:
import ConfigParser as configparser
conf=configparser.ConfigParser()
conf.read('tox.ini')
print(conf.get('$section', '$option')) if conf.has_option('$section', '$option') else ''
EOF
)
echo $exclusions $global_exclusions
}
excluded() {
local test_name=$1
for e in $exclusions; do
if [ "$e" = "$test_name" ]; then
return 0
fi
done
return 1
}
error() {
echo -e "ERROR: $1"
rc=1
}
echo "Running dib-lint in $(pwd)"
rc=0
TMPDIR=$(mktemp -d /tmp/tmp.XXXXXXXXXX)
trap "rm -rf $TMPDIR" EXIT
# Elements have a README.rst file
for i in $(find $ELEMENTS_DIR -mindepth 1 -maxdepth 1 -type d); do
[ -f "${i}/README.rst" ] || error "$i lacks a README.rst"
done
# note .py files are run through flake8 directly in tox.ini
for i in $(find $ELEMENTS_DIR -type f \
-not -name \*.rst \
-not -name \*.yaml \
-not -name \*.py); do
# Skip files in .gitignore
if git check-ignore -q "$i" ; then
echo Skipping $i
continue
fi
echo "Checking $i"
exclusions=("$(parse_exclusions $i)")
# source-repository does a read < $file and can miss the last line
# (or only line, if there's only one) when not newline terminated.
if [[ $(basename "${i}") =~ "source-repository-" ]]; then
nl=$(tail -c 1 ${i})
if [[ "${nl}" != "" ]]; then
error "$i does not end with a newline"
fi
fi
# Check that files starting with a shebang are +x
firstline=$(head -n 1 "$i")
if [ "${firstline:0:2}" = "#!" ]; then
if [ ! -x "$i" ] && ! excluded executable; then
error "$i is not executable"
fi
# run flake8 over python files that don't have .py. Note our
# "dib-python" interpreter can confuse the magic matching
# being done in "file" and make it think the file is not
# python; special-case it.
if [[ "$(file -b -k --mime-type $i)" =~ (text/x-python|text/x-script.python) ]] || \
[[ $firstline =~ "dib-python" ]]; then
flake8 $i || error "$i failed flake8"
else
# Ensure 4 spaces indent are used
if ! excluded indent ; then
indent_regex='^\( \{4\}\)* \{1,3\}[^ ]'
if grep -q "$indent_regex" ${i}; then
error "$i should use 4 spaces indent"
# outline the failing lines with line number
grep -n "$indent_regex" ${i}
fi
fi
fi
fi
# Check alphabetical ordering of element-deps
if [ $(basename $i) = "element-deps" ]; then
UNSORTED=${TMPDIR}/element-deps.unsorted
SORTED=${TMPDIR}/element-deps.sorted
grep -v -e '^#' -e '^$' $i > ${UNSORTED}
sort ${UNSORTED} > ${SORTED}
if [ -n "$(diff -c ${UNSORTED} ${SORTED})" ]; then
error "$i is not sorted alphabetically"
diff -y ${UNSORTED} ${SORTED}
fi
fi
# for consistency, let's just use #!/bin/bash everywhere (not
# /usr/bin/env, etc)
regex='^#!.*bash'
if [[ "$firstline" =~ $regex &&
"$firstline" != "#!/bin/bash" ]]; then
error "$i : only use #!/bin/bash for scripts"
fi
# Check that all scripts are set -eu -o pipefail and look for
# DIB_DEBUG_TRACE
# NOTE(bnemec): This doesn't verify that the set call occurs high
# enough in the file to be useful, but hopefully nobody will be
# sticking set calls at the end of their file to trick us. And if
# they are, that's easy enough to catch in reviews.
# Also, this is only going to check bash scripts - we've decided to
# explicitly require bash for any scripts that don't have a specific
# need to run under other shells, and any exceptions to that rule
# may not want these checks either.
if [[ "$firstline" =~ '#!/bin/bash' ]]; then
if ! excluded sete; then
if [ -z "$(grep "^set -[^ ]*e" $i)" ]; then
error "$i is not set -e"
fi
fi
if ! excluded setu; then
if [ -z "$(grep "^set -[^ ]*u" $i)" ]; then
error "$i is not set -u"
fi
fi
if ! excluded setpipefail; then
if [ -z "$(grep "^set -o pipefail" $i)" ]; then
error "$i is not set -o pipefail"
fi
fi
if ! excluded dibdebugtrace; then
if [ -z "$(grep "DIB_DEBUG_TRACE" $i)" ]; then
error "$i does not follow DIB_DEBUG_TRACE"
fi
fi
fi
# check that environment files don't "set -x" and they have no executable
# bits set
if [[ "$i" =~ (environment.d) ]]; then
if grep -q "set -x" $i; then
error "Environment file $i should not set tracing"
fi
if [[ -x $i ]]; then
error "Environment file $i should not be marked as executable"
fi
fi
# check for
# export FOO=$(bar)
# calls. These are dangerous, because the export hides the return
# code of the $(bar) call. Split this into 2 lines and -e will
# fail on the assignment
if grep -q 'export .*\$(' $i; then
error "Split export and assignments in $i"
fi
# check that sudo calls in phases run outside the chroot look
# "safe"; meaning that they seem to operate within the chroot
# somehow. This is not fool-proof, but catches egregious errors,
# and makes you think about it if you're doing something outside
# the box.
if ! excluded safe_sudo; then
if [[ $(dirname $i) =~ (root.d|extra-data.d|block-device.d|finalise.d|cleanup.d) ]]; then
while read LINE
do
if [[ $LINE =~ "sudo " ]]; then
# messy regex ahead! Don't match:
# - explicitly ignored
# - basic comments
# - install-packages ... sudo ...
# - any of the paths passed into the out-of-chroot elements
if [[ $LINE =~ (dib-lint: safe_sudo|^#|install-packages|TARGET_ROOT|IMAGE_BLOCK_DEVICE|TMP_MOUNT_PATH|TMP_IMAGE_PATH) ]]; then
continue
fi
error "$i : potentially unsafe sudo\n -- $LINE"
fi
done < $i
fi
fi
# check that which calls are not used. It is not built in and is missing
# from some constrained environments
if ! excluded which; then
while read LINE
do
if [[ $LINE =~ "which " ]]; then
# Don't match:
# - explicitly ignored
# - commented
if [[ $LINE =~ (dib-lint: which|^#) ]]; then
continue
fi
error "$i : potential use of which\n -- $LINE"
fi
done < $i
fi
done
echo "Checking indents..."
for i in $(find $ELEMENTS_DIR -type f -and -name '*.rst' -or -type f -executable) \
$(find $LIB_DIR -type f); do
# Skip files in .gitignore
if git check-ignore -q "$i" ; then
echo Skipping $i
continue
fi
# Check for tab indentation
if ! excluded tabindent; then
if grep -q $'^ *\t' ${i}; then
error "$i contains tab characters"
fi
fi
if ! excluded newline; then
if [ "$(tail -c 1 $i)" != "" ]; then
error "No newline at end of file: $i"
fi
fi
done
if ! excluded mddocs; then
md_docs=$(find $ELEMENTS_DIR -name '*.md')
if [ -n "$md_docs" ]; then
error ".md docs found: $md_docs"
fi
fi
echo "Checking YAML parsing..."
for i in $(find $ELEMENTS_DIR -type f -name '*.yaml'); do
echo "Parsing $i"
py_check="
import yaml
import sys
try:
objs = yaml.safe_load(open('$i'))
except yaml.parser.ParserError:
sys.exit(1)
"
if ! python -c "$py_check"; then
error "$i is not a valid YAML file"
fi
done
echo "Checking pkg-map files..."
for i in $(find $ELEMENTS_DIR -type f \
-name 'pkg-map' -a \! -executable); do
echo "Parsing $i"
py_check="
import json
import sys
try:
objs = json.load(open('$i'))
except ValueError:
sys.exit(1)
"
if ! python -c "$py_check"; then
error "$i is not a valid JSON file"
fi
done
if [[ $rc == 0 ]]; then
echo "PASS"
else
echo "*** FAIL: Some tests failed!"
fi
exit $rc