diskimage-builder/bin/dib-lint

#!/bin/bash

# Copyright 2014 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script checks all files in the "elements" directory for some
# common mistakes and exits with a non-zero status if it finds any.

set -eu
set -o pipefail

ELEMENTS_DIR=${ELEMENTS_DIR:-diskimage_builder/elements}
LIB_DIR=${LIB_DIR:-diskimage_builder/lib}

parse_exclusions() {
    # Per-file exclusions
    # Example: # dib-lint: disable=sete setpipefail
    local filename=$1
    local disable_pattern="# dib-lint: disable="
    local exclusions=$(grep "^$disable_pattern.*$" $filename | sed "s/$disable_pattern//g")

    # Global exclusions read from tox.ini
    # Example section in tox.ini:
    #   [dib-lint]
    #   ignore = sete setu
    section="dib-lint"
    option="ignore"
    global_exclusions=$(python - <<EOF
try:
  import configparser
except ImportError:
  import ConfigParser as configparser
conf=configparser.ConfigParser()
conf.read('tox.ini')
print(conf.get('$section', '$option')) if conf.has_option('$section', '$option') else ''
EOF
    )
    echo $exclusions $global_exclusions
}

excluded() {
    local test_name=$1
    for e in $exclusions; do
        if [ "$e" = "$test_name" ]; then
            return 0
        fi
    done
    return 1
}

error() {
    echo -e "ERROR: $1"
    rc=1
}

echo "Running dib-lint in $(pwd)"

rc=0
TMPDIR=$(mktemp -d /tmp/tmp.XXXXXXXXXX)
trap "rm -rf $TMPDIR" EXIT

# Elements have a README.rst file
for i in $(find $ELEMENTS_DIR -mindepth 1 -maxdepth 1 -type d); do
    [ -f "${i}/README.rst" ] || error "$i lacks a README.rst"
done

# note .py files are run through flake8 directly in tox.ini
for i in $(find $ELEMENTS_DIR -type f \
                -not -name \*.rst     \
                -not -name \*.yaml    \
                -not -name \*.py); do

    # Skip files in .gitignore
    if git check-ignore -q "$i" ; then
        echo Skipping $i
        continue
    fi
    echo "Checking $i"

    exclusions=("$(parse_exclusions $i)")

    # source-repository does a read < $file and can miss the last line
    # (or only line, if there's only one) when not newline terminated.
    if [[ $(basename "${i}") =~ "source-repository-" ]]; then
        nl=$(tail -c 1 ${i})
        if [[ "${nl}" != "" ]]; then
            error "$i does not end with a newline"
        fi
    fi

    # Check that files starting with a shebang are +x
    firstline=$(head -n 1 "$i")
    if [ "${firstline:0:2}" = "#!" ]; then
        if [ ! -x "$i" ] && ! excluded executable; then
            error "$i is not executable"
        fi

        # run flake8 over python files that don't have .py.  Note our
        # "dib-python" interpreter can confuse the magic matching
        # being done in "file" and make it think the file is not
        # python; special-case it.
        if [[ "$(file -b -k --mime-type $i)" =~ "text/x-python" ]] || \
               [[ $firstline =~ "dib-python" ]]; then
            flake8 $i || error "$i failed flake8"
        else
            # Ensure 4 spaces indent are used
            if ! excluded indent ; then
                indent_regex='^\( \{4\}\)* \{1,3\}[^ ]'
                if grep -q "$indent_regex" ${i}; then
                    error "$i should use 4 spaces indent"
                    # outline the failing lines with line number
                    grep -n "$indent_regex" ${i}
                fi
            fi
        fi
    fi

    # Check alphabetical ordering of element-deps
    if [ $(basename $i) = "element-deps" ]; then
        UNSORTED=${TMPDIR}/element-deps.unsorted
        SORTED=${TMPDIR}/element-deps.sorted
        grep -v -e '^#' -e '^$' $i > ${UNSORTED}
        sort ${UNSORTED} > ${SORTED}
        if [ -n "$(diff -c ${UNSORTED} ${SORTED})" ]; then
            error "$i is not sorted alphabetically"
            diff -y ${UNSORTED} ${SORTED}
        fi
    fi

    # for consistency, let's just use #!/bin/bash everywhere (not
    # /usr/bin/env, etc)
    regex='^#!.*bash'
    if [[ "$firstline" =~ $regex &&
                "$firstline" != "#!/bin/bash" ]]; then
        error "$i : only use #!/bin/bash for scripts"
    fi

    # Check that all scripts are set -eu -o pipefail and look for
    # DIB_DEBUG_TRACE
    # NOTE(bnemec): This doesn't verify that the set call occurs high
    # enough in the file to be useful, but hopefully nobody will be
    # sticking set calls at the end of their file to trick us.  And if
    # they are, that's easy enough to catch in reviews.
    # Also, this is only going to check bash scripts - we've decided to
    # explicitly require bash for any scripts that don't have a specific
    # need to run under other shells, and any exceptions to that rule
    # may not want these checks either.
    if [[ "$firstline" =~ '#!/bin/bash' ]]; then
        if ! excluded sete; then
            if [ -z "$(grep "^set -[^ ]*e" $i)" ]; then
                error "$i is not set -e"
            fi
        fi
        if ! excluded setu; then
            if [ -z "$(grep "^set -[^ ]*u" $i)" ]; then
                error "$i is not set -u"
            fi
        fi
        if ! excluded setpipefail; then
            if [ -z "$(grep "^set -o pipefail" $i)" ]; then
                error "$i is not set -o pipefail"
            fi
        fi
        if ! excluded dibdebugtrace; then
            if [ -z "$(grep "DIB_DEBUG_TRACE" $i)" ]; then
                error "$i does not follow DIB_DEBUG_TRACE"
            fi
        fi
    fi

    # check that environment files don't "set -x" and they have no executable
    # bits set
    if [[ "$i" =~ (environment.d) ]]; then
        if grep -q "set -x" $i; then
            error "Environment file $i should not set tracing"
        fi
        if [[ -x $i ]]; then
            error "Environment file $i should not be marked as executable"
        fi
    fi

    # check for
    #  export FOO=$(bar)
    # calls.  These are dangerous, because the export hides the return
    # code of the $(bar) call.  Split this into 2 lines and -e will
    # fail on the assignment
    if grep -q 'export .*\$(' $i; then
        error "Split export and assignments in $i"
    fi

    # check that sudo calls in phases run outside the chroot look
    # "safe"; meaning that they seem to operate within the chroot
    # somehow.  This is not fool-proof, but catches egregious errors,
    # and makes you think about it if you're doing something outside
    # the box.
    if ! excluded safe_sudo; then
        if [[ $(dirname $i) =~ (root.d|extra-data.d|block-device.d|finalise.d|cleanup.d) ]]; then
            while read LINE
            do
                if [[ $LINE =~ "sudo " ]]; then
                    # messy regex ahead!  Don't match:
                    #  - explicitly ignored
                    #  - basic comments
                    #  - install-packages ... sudo ...
                    #  - any of the paths passed into the out-of-chroot elements
                    if [[ $LINE =~ (dib-lint: safe_sudo|^#|install-packages|TARGET_ROOT|IMAGE_BLOCK_DEVICE|TMP_MOUNT_PATH|TMP_IMAGE_PATH) ]]; then
                        continue
                    fi
                    error "$i : potentially unsafe sudo\n -- $LINE"
                fi
            done < $i
        fi
    fi

    # check that which calls are not used. It is not built in and is missing
    # from some constrained environments
    if ! excluded which; then
        while read LINE
        do
            if [[ $LINE =~ "which " ]]; then
                #  Don't match:
                #  - explicitly ignored
                #  - commented
                if [[ $LINE =~ (dib-lint: which|^#) ]]; then
                    continue
                fi
                error "$i : potential use of which\n -- $LINE"
            fi
        done < $i
    fi

done

echo "Checking indents..."

for i in $(find $ELEMENTS_DIR -type f -and -name '*.rst' -or -type f -executable) \
             $(find $LIB_DIR -type f); do
    # Skip files in .gitignore
    if git check-ignore -q "$i" ; then
        echo Skipping $i
        continue
    fi
    # Check for tab indentation
    if ! excluded tabindent; then
        if grep -q $'^ *\t' ${i}; then
            error "$i contains tab characters"
        fi
    fi

    if ! excluded newline; then
        if [ "$(tail -c 1 $i)" != "" ]; then
            error "No newline at end of file: $i"
        fi
    fi
done

if ! excluded mddocs; then
    md_docs=$(find $ELEMENTS_DIR -name '*.md')
    if [ -n "$md_docs" ]; then
        error ".md docs found: $md_docs"
    fi
fi

echo "Checking YAML parsing..."
for i in $(find $ELEMENTS_DIR -type f -name '*.yaml'); do
    echo "Parsing  $i"
    py_check="
import yaml
import sys
try:
    objs = yaml.safe_load(open('$i'))
except yaml.parser.ParserError:
    sys.exit(1)
"
    if ! python -c "$py_check"; then
        error "$i is not a valid YAML file"
    fi
done
echo "Checking pkg-map files..."
for i in $(find $ELEMENTS_DIR -type f \
                -name 'pkg-map' -a \! -executable); do
    echo "Parsing  $i"
    py_check="
import json
import sys
try:
    objs = json.load(open('$i'))
except ValueError:
    sys.exit(1)
"
    if ! python -c "$py_check"; then
        error "$i is not a valid JSON file"
    fi
done

if [[ $rc == 0 ]]; then
    echo "PASS"
else
    echo "*** FAIL: Some tests failed!"
fi

exit $rc