eed1be9992
It seems libmagic changed it's Python output to
"text/x-script.python", which I see on Fedora. Handle this too so we
detect Python files correctly.
[1] eb373e431c
Change-Id: I35992c70523a8f2bc5efff2e5167ed1ac1514d34
315 lines
9.6 KiB
Bash
Executable File
315 lines
9.6 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Copyright 2014 Red Hat, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# This script checks all files in the "elements" directory for some
|
|
# common mistakes and exits with a non-zero status if it finds any.
|
|
|
|
set -eu
|
|
set -o pipefail
|
|
|
|
ELEMENTS_DIR=${ELEMENTS_DIR:-diskimage_builder/elements}
|
|
LIB_DIR=${LIB_DIR:-diskimage_builder/lib}
|
|
|
|
parse_exclusions() {
|
|
# Per-file exclusions
|
|
# Example: # dib-lint: disable=sete setpipefail
|
|
local filename=$1
|
|
local disable_pattern="# dib-lint: disable="
|
|
local exclusions=$(grep "^$disable_pattern.*$" $filename | sed "s/$disable_pattern//g")
|
|
|
|
# Global exclusions read from tox.ini
|
|
# Example section in tox.ini:
|
|
# [dib-lint]
|
|
# ignore = sete setu
|
|
section="dib-lint"
|
|
option="ignore"
|
|
global_exclusions=$(python - <<EOF
|
|
try:
|
|
import configparser
|
|
except ImportError:
|
|
import ConfigParser as configparser
|
|
conf=configparser.ConfigParser()
|
|
conf.read('tox.ini')
|
|
print(conf.get('$section', '$option')) if conf.has_option('$section', '$option') else ''
|
|
EOF
|
|
)
|
|
echo $exclusions $global_exclusions
|
|
}
|
|
|
|
excluded() {
|
|
local test_name=$1
|
|
for e in $exclusions; do
|
|
if [ "$e" = "$test_name" ]; then
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
error() {
|
|
echo -e "ERROR: $1"
|
|
rc=1
|
|
}
|
|
|
|
echo "Running dib-lint in $(pwd)"
|
|
|
|
rc=0
|
|
TMPDIR=$(mktemp -d /tmp/tmp.XXXXXXXXXX)
|
|
trap "rm -rf $TMPDIR" EXIT
|
|
|
|
# Elements have a README.rst file
|
|
for i in $(find $ELEMENTS_DIR -mindepth 1 -maxdepth 1 -type d); do
|
|
[ -f "${i}/README.rst" ] || error "$i lacks a README.rst"
|
|
done
|
|
|
|
# note .py files are run through flake8 directly in tox.ini
|
|
for i in $(find $ELEMENTS_DIR -type f \
|
|
-not -name \*.rst \
|
|
-not -name \*.yaml \
|
|
-not -name \*.py); do
|
|
|
|
# Skip files in .gitignore
|
|
if git check-ignore -q "$i" ; then
|
|
echo Skipping $i
|
|
continue
|
|
fi
|
|
echo "Checking $i"
|
|
|
|
exclusions=("$(parse_exclusions $i)")
|
|
|
|
# source-repository does a read < $file and can miss the last line
|
|
# (or only line, if there's only one) when not newline terminated.
|
|
if [[ $(basename "${i}") =~ "source-repository-" ]]; then
|
|
nl=$(tail -c 1 ${i})
|
|
if [[ "${nl}" != "" ]]; then
|
|
error "$i does not end with a newline"
|
|
fi
|
|
fi
|
|
|
|
# Check that files starting with a shebang are +x
|
|
firstline=$(head -n 1 "$i")
|
|
if [ "${firstline:0:2}" = "#!" ]; then
|
|
if [ ! -x "$i" ] && ! excluded executable; then
|
|
error "$i is not executable"
|
|
fi
|
|
|
|
# run flake8 over python files that don't have .py. Note our
|
|
# "dib-python" interpreter can confuse the magic matching
|
|
# being done in "file" and make it think the file is not
|
|
# python; special-case it.
|
|
if [[ "$(file -b -k --mime-type $i)" =~ (text/x-python|text/x-script.python) ]] || \
|
|
[[ $firstline =~ "dib-python" ]]; then
|
|
flake8 $i || error "$i failed flake8"
|
|
else
|
|
# Ensure 4 spaces indent are used
|
|
if ! excluded indent ; then
|
|
indent_regex='^\( \{4\}\)* \{1,3\}[^ ]'
|
|
if grep -q "$indent_regex" ${i}; then
|
|
error "$i should use 4 spaces indent"
|
|
# outline the failing lines with line number
|
|
grep -n "$indent_regex" ${i}
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Check alphabetical ordering of element-deps
|
|
if [ $(basename $i) = "element-deps" ]; then
|
|
UNSORTED=${TMPDIR}/element-deps.unsorted
|
|
SORTED=${TMPDIR}/element-deps.sorted
|
|
grep -v -e '^#' -e '^$' $i > ${UNSORTED}
|
|
sort ${UNSORTED} > ${SORTED}
|
|
if [ -n "$(diff -c ${UNSORTED} ${SORTED})" ]; then
|
|
error "$i is not sorted alphabetically"
|
|
diff -y ${UNSORTED} ${SORTED}
|
|
fi
|
|
fi
|
|
|
|
# for consistency, let's just use #!/bin/bash everywhere (not
|
|
# /usr/bin/env, etc)
|
|
regex='^#!.*bash'
|
|
if [[ "$firstline" =~ $regex &&
|
|
"$firstline" != "#!/bin/bash" ]]; then
|
|
error "$i : only use #!/bin/bash for scripts"
|
|
fi
|
|
|
|
# Check that all scripts are set -eu -o pipefail and look for
|
|
# DIB_DEBUG_TRACE
|
|
# NOTE(bnemec): This doesn't verify that the set call occurs high
|
|
# enough in the file to be useful, but hopefully nobody will be
|
|
# sticking set calls at the end of their file to trick us. And if
|
|
# they are, that's easy enough to catch in reviews.
|
|
# Also, this is only going to check bash scripts - we've decided to
|
|
# explicitly require bash for any scripts that don't have a specific
|
|
# need to run under other shells, and any exceptions to that rule
|
|
# may not want these checks either.
|
|
if [[ "$firstline" =~ '#!/bin/bash' ]]; then
|
|
if ! excluded sete; then
|
|
if [ -z "$(grep "^set -[^ ]*e" $i)" ]; then
|
|
error "$i is not set -e"
|
|
fi
|
|
fi
|
|
if ! excluded setu; then
|
|
if [ -z "$(grep "^set -[^ ]*u" $i)" ]; then
|
|
error "$i is not set -u"
|
|
fi
|
|
fi
|
|
if ! excluded setpipefail; then
|
|
if [ -z "$(grep "^set -o pipefail" $i)" ]; then
|
|
error "$i is not set -o pipefail"
|
|
fi
|
|
fi
|
|
if ! excluded dibdebugtrace; then
|
|
if [ -z "$(grep "DIB_DEBUG_TRACE" $i)" ]; then
|
|
error "$i does not follow DIB_DEBUG_TRACE"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# check that environment files don't "set -x" and they have no executable
|
|
# bits set
|
|
if [[ "$i" =~ (environment.d) ]]; then
|
|
if grep -q "set -x" $i; then
|
|
error "Environment file $i should not set tracing"
|
|
fi
|
|
if [[ -x $i ]]; then
|
|
error "Environment file $i should not be marked as executable"
|
|
fi
|
|
fi
|
|
|
|
# check for
|
|
# export FOO=$(bar)
|
|
# calls. These are dangerous, because the export hides the return
|
|
# code of the $(bar) call. Split this into 2 lines and -e will
|
|
# fail on the assignment
|
|
if grep -q 'export .*\$(' $i; then
|
|
error "Split export and assignments in $i"
|
|
fi
|
|
|
|
# check that sudo calls in phases run outside the chroot look
|
|
# "safe"; meaning that they seem to operate within the chroot
|
|
# somehow. This is not fool-proof, but catches egregious errors,
|
|
# and makes you think about it if you're doing something outside
|
|
# the box.
|
|
if ! excluded safe_sudo; then
|
|
if [[ $(dirname $i) =~ (root.d|extra-data.d|block-device.d|finalise.d|cleanup.d) ]]; then
|
|
while read LINE
|
|
do
|
|
if [[ $LINE =~ "sudo " ]]; then
|
|
# messy regex ahead! Don't match:
|
|
# - explicitly ignored
|
|
# - basic comments
|
|
# - install-packages ... sudo ...
|
|
# - any of the paths passed into the out-of-chroot elements
|
|
if [[ $LINE =~ (dib-lint: safe_sudo|^#|install-packages|TARGET_ROOT|IMAGE_BLOCK_DEVICE|TMP_MOUNT_PATH|TMP_IMAGE_PATH) ]]; then
|
|
continue
|
|
fi
|
|
error "$i : potentially unsafe sudo\n -- $LINE"
|
|
fi
|
|
done < $i
|
|
fi
|
|
fi
|
|
|
|
# check that which calls are not used. It is not built in and is missing
|
|
# from some constrained environments
|
|
if ! excluded which; then
|
|
while read LINE
|
|
do
|
|
if [[ $LINE =~ "which " ]]; then
|
|
# Don't match:
|
|
# - explicitly ignored
|
|
# - commented
|
|
if [[ $LINE =~ (dib-lint: which|^#) ]]; then
|
|
continue
|
|
fi
|
|
error "$i : potential use of which\n -- $LINE"
|
|
fi
|
|
done < $i
|
|
fi
|
|
|
|
done
|
|
|
|
echo "Checking indents..."
|
|
|
|
for i in $(find $ELEMENTS_DIR -type f -and -name '*.rst' -or -type f -executable) \
|
|
$(find $LIB_DIR -type f); do
|
|
# Skip files in .gitignore
|
|
if git check-ignore -q "$i" ; then
|
|
echo Skipping $i
|
|
continue
|
|
fi
|
|
# Check for tab indentation
|
|
if ! excluded tabindent; then
|
|
if grep -q $'^ *\t' ${i}; then
|
|
error "$i contains tab characters"
|
|
fi
|
|
fi
|
|
|
|
if ! excluded newline; then
|
|
if [ "$(tail -c 1 $i)" != "" ]; then
|
|
error "No newline at end of file: $i"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if ! excluded mddocs; then
|
|
md_docs=$(find $ELEMENTS_DIR -name '*.md')
|
|
if [ -n "$md_docs" ]; then
|
|
error ".md docs found: $md_docs"
|
|
fi
|
|
fi
|
|
|
|
echo "Checking YAML parsing..."
|
|
for i in $(find $ELEMENTS_DIR -type f -name '*.yaml'); do
|
|
echo "Parsing $i"
|
|
py_check="
|
|
import yaml
|
|
import sys
|
|
try:
|
|
objs = yaml.safe_load(open('$i'))
|
|
except yaml.parser.ParserError:
|
|
sys.exit(1)
|
|
"
|
|
if ! python -c "$py_check"; then
|
|
error "$i is not a valid YAML file"
|
|
fi
|
|
done
|
|
echo "Checking pkg-map files..."
|
|
for i in $(find $ELEMENTS_DIR -type f \
|
|
-name 'pkg-map' -a \! -executable); do
|
|
echo "Parsing $i"
|
|
py_check="
|
|
import json
|
|
import sys
|
|
try:
|
|
objs = json.load(open('$i'))
|
|
except ValueError:
|
|
sys.exit(1)
|
|
"
|
|
if ! python -c "$py_check"; then
|
|
error "$i is not a valid JSON file"
|
|
fi
|
|
done
|
|
|
|
if [[ $rc == 0 ]]; then
|
|
echo "PASS"
|
|
else
|
|
echo "*** FAIL: Some tests failed!"
|
|
fi
|
|
|
|
exit $rc
|