diskimage-builder/bin/dib-lint

315 lines
9.6 KiB
Plaintext
Raw Normal View History

#!/bin/bash
# Copyright 2014 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script checks all files in the "elements" directory for some
# common mistakes and exits with a non-zero status if it finds any.
set -eu
set -o pipefail
ELEMENTS_DIR=${ELEMENTS_DIR:-diskimage_builder/elements}
LIB_DIR=${LIB_DIR:-diskimage_builder/lib}
Move elements & lib relative to diskimage_builder package Currently we have all our elements and library files in a top-level directory and install them into <root>/share/diskimage-builder/[elements|lib] (where root is either / or the root of a virtualenv). The problem with this is that editable/development installs (pip -e) do *not* install data_files. Thus we have no canonical location to look for elements -- leading to the various odd things we do such as a whole bunch of guessing at the top of disk-image-create and having a special test-loader in tests/test_elements.py so we can run python unit tests on those elements that have it. data_files is really the wrong thing to use for what are essentially assets of the program. data_files install works well for things like config-files, init.d files or dropping documentation files. By moving the elements under the diskimage_builder package, we always know where they are relative to where we import from. In fact, pkg_resources has an api for this which we wrap in the new diskimage_builder/paths.py helper [1]. We use this helper to find the correct path in the couple of places we need to find the base-elements dir, and for the paths to import the library shell functions. Elements such as svc-map and pkg-map include python unit-tests, which we do not need tests/test_elements.py to special-case load any more. They just get found automatically by the normal subunit loader. I have a follow-on change (I69ca3d26fede0506a6353c077c69f735c8d84d28) to move disk-image-create to a regular python entry-point. Unfortunately, this has to move to work with setuptools. You'd think a symlink under diskimage_builder/[elements|lib] would work, but it doesn't. [1] this API handles stuff like getting files out of .zip archive modules, which we don't do. Essentially for us it's returning __file__. Change-Id: I5e3e3c97f385b1a4ff2031a161a55b231895df5b
2016-09-09 03:11:52 +00:00
parse_exclusions() {
# Per-file exclusions
# Example: # dib-lint: disable=sete setpipefail
local filename=$1
local disable_pattern="# dib-lint: disable="
local exclusions=$(grep "^$disable_pattern.*$" $filename | sed "s/$disable_pattern//g")
# Global exclusions read from tox.ini
# Example section in tox.ini:
# [dib-lint]
# ignore = sete setu
section="dib-lint"
option="ignore"
global_exclusions=$(python - <<EOF
try:
import configparser
except ImportError:
import ConfigParser as configparser
conf=configparser.ConfigParser()
conf.read('tox.ini')
print(conf.get('$section', '$option')) if conf.has_option('$section', '$option') else ''
EOF
)
echo $exclusions $global_exclusions
}
excluded() {
local test_name=$1
for e in $exclusions; do
if [ "$e" = "$test_name" ]; then
return 0
fi
done
return 1
}
error() {
echo -e "ERROR: $1"
rc=1
}
echo "Running dib-lint in $(pwd)"
rc=0
TMPDIR=$(mktemp -d /tmp/tmp.XXXXXXXXXX)
trap "rm -rf $TMPDIR" EXIT
# Elements have a README.rst file
for i in $(find $ELEMENTS_DIR -mindepth 1 -maxdepth 1 -type d); do
[ -f "${i}/README.rst" ] || error "$i lacks a README.rst"
done
# note .py files are run through flake8 directly in tox.ini
for i in $(find $ELEMENTS_DIR -type f \
-not -name \*.rst \
-not -name \*.yaml \
-not -name \*.py); do
# Skip files in .gitignore
if git check-ignore -q "$i" ; then
echo Skipping $i
continue
fi
echo "Checking $i"
exclusions=("$(parse_exclusions $i)")
# source-repository does a read < $file and can miss the last line
# (or only line, if there's only one) when not newline terminated.
if [[ $(basename "${i}") =~ "source-repository-" ]]; then
nl=$(tail -c 1 ${i})
if [[ "${nl}" != "" ]]; then
error "$i does not end with a newline"
fi
fi
# Check that files starting with a shebang are +x
firstline=$(head -n 1 "$i")
if [ "${firstline:0:2}" = "#!" ]; then
if [ ! -x "$i" ] && ! excluded executable; then
error "$i is not executable"
fi
# run flake8 over python files that don't have .py. Note our
# "dib-python" interpreter can confuse the magic matching
# being done in "file" and make it think the file is not
# python; special-case it.
if [[ "$(file -b -k --mime-type $i)" =~ (text/x-python|text/x-script.python) ]] || \
[[ $firstline =~ "dib-python" ]]; then
flake8 $i || error "$i failed flake8"
else
# Ensure 4 spaces indent are used
if ! excluded indent ; then
indent_regex='^\( \{4\}\)* \{1,3\}[^ ]'
if grep -q "$indent_regex" ${i}; then
error "$i should use 4 spaces indent"
# outline the failing lines with line number
grep -n "$indent_regex" ${i}
fi
fi
fi
fi
# Check alphabetical ordering of element-deps
if [ $(basename $i) = "element-deps" ]; then
UNSORTED=${TMPDIR}/element-deps.unsorted
SORTED=${TMPDIR}/element-deps.sorted
grep -v -e '^#' -e '^$' $i > ${UNSORTED}
sort ${UNSORTED} > ${SORTED}
if [ -n "$(diff -c ${UNSORTED} ${SORTED})" ]; then
error "$i is not sorted alphabetically"
diff -y ${UNSORTED} ${SORTED}
fi
fi
# for consistency, let's just use #!/bin/bash everywhere (not
# /usr/bin/env, etc)
regex='^#!.*bash'
if [[ "$firstline" =~ $regex &&
"$firstline" != "#!/bin/bash" ]]; then
error "$i : only use #!/bin/bash for scripts"
fi
# Check that all scripts are set -eu -o pipefail and look for
# DIB_DEBUG_TRACE
# NOTE(bnemec): This doesn't verify that the set call occurs high
# enough in the file to be useful, but hopefully nobody will be
# sticking set calls at the end of their file to trick us. And if
# they are, that's easy enough to catch in reviews.
# Also, this is only going to check bash scripts - we've decided to
# explicitly require bash for any scripts that don't have a specific
# need to run under other shells, and any exceptions to that rule
# may not want these checks either.
if [[ "$firstline" =~ '#!/bin/bash' ]]; then
if ! excluded sete; then
if [ -z "$(grep "^set -[^ ]*e" $i)" ]; then
error "$i is not set -e"
fi
fi
if ! excluded setu; then
if [ -z "$(grep "^set -[^ ]*u" $i)" ]; then
error "$i is not set -u"
fi
fi
if ! excluded setpipefail; then
if [ -z "$(grep "^set -o pipefail" $i)" ]; then
error "$i is not set -o pipefail"
fi
fi
if ! excluded dibdebugtrace; then
if [ -z "$(grep "DIB_DEBUG_TRACE" $i)" ]; then
error "$i does not follow DIB_DEBUG_TRACE"
fi
fi
fi
# check that environment files don't "set -x" and they have no executable
# bits set
if [[ "$i" =~ (environment.d) ]]; then
if grep -q "set -x" $i; then
error "Environment file $i should not set tracing"
fi
if [[ -x $i ]]; then
error "Environment file $i should not be marked as executable"
fi
fi
# check for
# export FOO=$(bar)
# calls. These are dangerous, because the export hides the return
# code of the $(bar) call. Split this into 2 lines and -e will
# fail on the assignment
if grep -q 'export .*\$(' $i; then
error "Split export and assignments in $i"
fi
# check that sudo calls in phases run outside the chroot look
# "safe"; meaning that they seem to operate within the chroot
# somehow. This is not fool-proof, but catches egregious errors,
# and makes you think about it if you're doing something outside
# the box.
if ! excluded safe_sudo; then
if [[ $(dirname $i) =~ (root.d|extra-data.d|block-device.d|finalise.d|cleanup.d) ]]; then
while read LINE
do
if [[ $LINE =~ "sudo " ]]; then
# messy regex ahead! Don't match:
# - explicitly ignored
# - basic comments
# - install-packages ... sudo ...
# - any of the paths passed into the out-of-chroot elements
if [[ $LINE =~ (dib-lint: safe_sudo|^#|install-packages|TARGET_ROOT|IMAGE_BLOCK_DEVICE|TMP_MOUNT_PATH|TMP_IMAGE_PATH) ]]; then
continue
fi
error "$i : potentially unsafe sudo\n -- $LINE"
fi
done < $i
fi
fi
# check that which calls are not used. It is not built in and is missing
# from some constrained environments
if ! excluded which; then
while read LINE
do
if [[ $LINE =~ "which " ]]; then
# Don't match:
# - explicitly ignored
# - commented
if [[ $LINE =~ (dib-lint: which|^#) ]]; then
continue
fi
error "$i : potential use of which\n -- $LINE"
fi
done < $i
fi
done
echo "Checking indents..."
for i in $(find $ELEMENTS_DIR -type f -and -name '*.rst' -or -type f -executable) \
$(find $LIB_DIR -type f); do
# Skip files in .gitignore
if git check-ignore -q "$i" ; then
echo Skipping $i
continue
fi
# Check for tab indentation
if ! excluded tabindent; then
if grep -q $'^ *\t' ${i}; then
error "$i contains tab characters"
fi
fi
if ! excluded newline; then
if [ "$(tail -c 1 $i)" != "" ]; then
error "No newline at end of file: $i"
fi
fi
done
if ! excluded mddocs; then
Move elements & lib relative to diskimage_builder package Currently we have all our elements and library files in a top-level directory and install them into <root>/share/diskimage-builder/[elements|lib] (where root is either / or the root of a virtualenv). The problem with this is that editable/development installs (pip -e) do *not* install data_files. Thus we have no canonical location to look for elements -- leading to the various odd things we do such as a whole bunch of guessing at the top of disk-image-create and having a special test-loader in tests/test_elements.py so we can run python unit tests on those elements that have it. data_files is really the wrong thing to use for what are essentially assets of the program. data_files install works well for things like config-files, init.d files or dropping documentation files. By moving the elements under the diskimage_builder package, we always know where they are relative to where we import from. In fact, pkg_resources has an api for this which we wrap in the new diskimage_builder/paths.py helper [1]. We use this helper to find the correct path in the couple of places we need to find the base-elements dir, and for the paths to import the library shell functions. Elements such as svc-map and pkg-map include python unit-tests, which we do not need tests/test_elements.py to special-case load any more. They just get found automatically by the normal subunit loader. I have a follow-on change (I69ca3d26fede0506a6353c077c69f735c8d84d28) to move disk-image-create to a regular python entry-point. Unfortunately, this has to move to work with setuptools. You'd think a symlink under diskimage_builder/[elements|lib] would work, but it doesn't. [1] this API handles stuff like getting files out of .zip archive modules, which we don't do. Essentially for us it's returning __file__. Change-Id: I5e3e3c97f385b1a4ff2031a161a55b231895df5b
2016-09-09 03:11:52 +00:00
md_docs=$(find $ELEMENTS_DIR -name '*.md')
if [ -n "$md_docs" ]; then
error ".md docs found: $md_docs"
fi
fi
echo "Checking YAML parsing..."
Move elements & lib relative to diskimage_builder package Currently we have all our elements and library files in a top-level directory and install them into <root>/share/diskimage-builder/[elements|lib] (where root is either / or the root of a virtualenv). The problem with this is that editable/development installs (pip -e) do *not* install data_files. Thus we have no canonical location to look for elements -- leading to the various odd things we do such as a whole bunch of guessing at the top of disk-image-create and having a special test-loader in tests/test_elements.py so we can run python unit tests on those elements that have it. data_files is really the wrong thing to use for what are essentially assets of the program. data_files install works well for things like config-files, init.d files or dropping documentation files. By moving the elements under the diskimage_builder package, we always know where they are relative to where we import from. In fact, pkg_resources has an api for this which we wrap in the new diskimage_builder/paths.py helper [1]. We use this helper to find the correct path in the couple of places we need to find the base-elements dir, and for the paths to import the library shell functions. Elements such as svc-map and pkg-map include python unit-tests, which we do not need tests/test_elements.py to special-case load any more. They just get found automatically by the normal subunit loader. I have a follow-on change (I69ca3d26fede0506a6353c077c69f735c8d84d28) to move disk-image-create to a regular python entry-point. Unfortunately, this has to move to work with setuptools. You'd think a symlink under diskimage_builder/[elements|lib] would work, but it doesn't. [1] this API handles stuff like getting files out of .zip archive modules, which we don't do. Essentially for us it's returning __file__. Change-Id: I5e3e3c97f385b1a4ff2031a161a55b231895df5b
2016-09-09 03:11:52 +00:00
for i in $(find $ELEMENTS_DIR -type f -name '*.yaml'); do
echo "Parsing $i"
py_check="
import yaml
import sys
try:
objs = yaml.safe_load(open('$i'))
except yaml.parser.ParserError:
sys.exit(1)
"
if ! python -c "$py_check"; then
error "$i is not a valid YAML file"
fi
done
echo "Checking pkg-map files..."
Move elements & lib relative to diskimage_builder package Currently we have all our elements and library files in a top-level directory and install them into <root>/share/diskimage-builder/[elements|lib] (where root is either / or the root of a virtualenv). The problem with this is that editable/development installs (pip -e) do *not* install data_files. Thus we have no canonical location to look for elements -- leading to the various odd things we do such as a whole bunch of guessing at the top of disk-image-create and having a special test-loader in tests/test_elements.py so we can run python unit tests on those elements that have it. data_files is really the wrong thing to use for what are essentially assets of the program. data_files install works well for things like config-files, init.d files or dropping documentation files. By moving the elements under the diskimage_builder package, we always know where they are relative to where we import from. In fact, pkg_resources has an api for this which we wrap in the new diskimage_builder/paths.py helper [1]. We use this helper to find the correct path in the couple of places we need to find the base-elements dir, and for the paths to import the library shell functions. Elements such as svc-map and pkg-map include python unit-tests, which we do not need tests/test_elements.py to special-case load any more. They just get found automatically by the normal subunit loader. I have a follow-on change (I69ca3d26fede0506a6353c077c69f735c8d84d28) to move disk-image-create to a regular python entry-point. Unfortunately, this has to move to work with setuptools. You'd think a symlink under diskimage_builder/[elements|lib] would work, but it doesn't. [1] this API handles stuff like getting files out of .zip archive modules, which we don't do. Essentially for us it's returning __file__. Change-Id: I5e3e3c97f385b1a4ff2031a161a55b231895df5b
2016-09-09 03:11:52 +00:00
for i in $(find $ELEMENTS_DIR -type f \
-name 'pkg-map' -a \! -executable); do
echo "Parsing $i"
py_check="
import json
import sys
try:
objs = json.load(open('$i'))
except ValueError:
sys.exit(1)
"
if ! python -c "$py_check"; then
error "$i is not a valid JSON file"
fi
done
if [[ $rc == 0 ]]; then
echo "PASS"
else
echo "*** FAIL: Some tests failed!"
fi
exit $rc