#!/bin/bash # Copyright 2014 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. # This script checks all files in the "elements" directory for some # common mistakes and exits with a non-zero status if it finds any. set -eu set -o pipefail ELEMENTS_DIR=diskimage_builder/elements parse_exclusions() { # Per-file exclusions # Example: # dib-lint: disable=sete setpipefail local filename=$1 local disable_pattern="# dib-lint: disable=" local exclusions=$(grep "^$disable_pattern.*$" $filename | sed "s/$disable_pattern//g") # Global exclusions read from tox.ini # Example section in tox.ini: # [dib-lint] # ignore = sete setu section="dib-lint" option="ignore" global_exclusions=$(python -c \ "import ConfigParser; \ conf=ConfigParser.ConfigParser(); \ conf.read('tox.ini'); \ print conf.get('$section', '$option') if conf.has_option('$section', '$option') else ''" ) echo $exclusions $global_exclusions } excluded() { local test_name=$1 for e in $exclusions; do if [ "$e" = "$test_name" ]; then return 0 fi done return 1 } error() { echo -e "ERROR: $1" rc=1 } echo "Running dib-lint in $(pwd)" rc=0 TMPDIR=$(mktemp -d /tmp/tmp.XXXXXXXXXX) trap "rm -rf $TMPDIR" EXIT for i in $(find $ELEMENTS_DIR -type f \ -not -name \*~ \ -not -name \#\*\# \ -not -name \*.orig \ -not -name \*.rst \ -not -name \*.yaml \ -not -name \*.py \ -not -name \*.pyc); do echo "Checking $i" exclusions=("$(parse_exclusions $i)") # Check that files starting with a shebang are +x firstline=$(head -n 1 "$i") if [ "${firstline:0:2}" = "#!" ]; then if [ ! -x "$i" ] && ! excluded executable; then error "$i is not executable" fi # run flake8 over python files. note our "dib-python" # interpreter can confuse the magic matching being done in # "file" and make it think the file is not python; # special-case it. if [[ "$(file -b -k --mime-type $i)" =~ "text/x-python" ]] || \ [[ $firstline =~ "dib-python" ]]; then flake8 $i || error "$i failed flake8" else # Ensure 4 spaces indent are used if ! excluded indent ; then indent_regex='^\( \{4\}\)* \{1,3\}[^ ]' if grep -q "$indent_regex" ${i}; then error "$i should use 4 spaces indent" # outline the failing lines with line number grep -n "$indent_regex" ${i} fi fi fi fi # Check alphabetical ordering of element-deps if [ $(basename $i) = "element-deps" ]; then UNSORTED=${TMPDIR}/element-deps.unsorted SORTED=${TMPDIR}/element-deps.sorted grep -v -e '^#' -e '^$' $i > ${UNSORTED} sort ${UNSORTED} > ${SORTED} if [ -n "$(diff -c ${UNSORTED} ${SORTED})" ]; then error "$i is not sorted alphabetically" diff -y ${UNSORTED} ${SORTED} fi fi # for consistency, let's just use #!/bin/bash everywhere (not # /usr/bin/env, etc) regex='^#!.*bash' if [[ "$firstline" =~ $regex && "$firstline" != "#!/bin/bash" ]]; then error "$i : only use #!/bin/bash for scripts" fi # Check that all scripts are set -eu -o pipefail and look for # DIB_DEBUG_TRACE # NOTE(bnemec): This doesn't verify that the set call occurs high # enough in the file to be useful, but hopefully nobody will be # sticking set calls at the end of their file to trick us. And if # they are, that's easy enough to catch in reviews. # Also, this is only going to check bash scripts - we've decided to # explicitly require bash for any scripts that don't have a specific # need to run under other shells, and any exceptions to that rule # may not want these checks either. if [[ "$firstline" =~ '#!/bin/bash' ]]; then if ! excluded sete; then if [ -z "$(grep "^set -[^ ]*e" $i)" ]; then error "$i is not set -e" fi fi if ! excluded setu; then if [ -z "$(grep "^set -[^ ]*u" $i)" ]; then error "$i is not set -u" fi fi if ! excluded setpipefail; then if [ -z "$(grep "^set -o pipefail" $i)" ]; then error "$i is not set -o pipefail" fi fi if ! excluded dibdebugtrace; then if [ -z "$(grep "DIB_DEBUG_TRACE" $i)" ]; then error "$i does not follow DIB_DEBUG_TRACE" fi fi fi # check that environment files don't "set -x" and they have no executable # bits set if [[ "$i" =~ (environment.d) ]]; then if grep -q "set -x" $i; then error "Environment file $i should not set tracing" fi if [[ -x $i ]]; then error "Environment file $i should not be marked as executable" fi fi # check for # export FOO=$(bar) # calls. These are dangerous, because the export hides the return # code of the $(bar) call. Split this into 2 lines and -e will # fail on the assignment if grep -q 'export .*\$(' $i; then error "Split export and assignments in $i" fi # check that sudo calls in phases run outside the chroot look # "safe"; meaning that they seem to operate within the chroot # somehow. This is not fool-proof, but catches egregious errors, # and makes you think about it if you're doing something outside # the box. if ! excluded safe_sudo; then if [[ $(dirname $i) =~ (root.d|extra-data.d|block-device.d|finalise.d|cleanup.d) ]]; then while read LINE do if [[ $LINE =~ "sudo " ]]; then # messy regex ahead! Don't match: # - explicitly ignored # - basic comments # - install-packages ... sudo ... # - any of the paths passed into the out-of-chroot elements if [[ $LINE =~ (dib-lint: safe_sudo|^#|install-packages|TARGET_ROOT|IMAGE_BLOCK_DEVICE|TMP_MOUNT_PATH|TMP_IMAGE_PATH) ]]; then continue fi error "$i : potentially unsafe sudo\n -- $LINE" fi done < $i fi fi done echo "Checking indents..." for i in $(find $ELEMENTS_DIR -type f -and -name '*.rst' -or -type f -executable); do # Check for tab indentation if ! excluded tabindent; then if grep -q $'^ *\t' ${i}; then error "$i contains tab characters" fi fi if ! excluded newline; then if [ "$(tail -c 1 $i)" != "" ]; then error "No newline at end of file: $i" fi fi done if ! excluded mddocs; then md_docs=$(find $ELEMENTS_DIR -name '*.md') if [ -n "$md_docs" ]; then error ".md docs found: $md_docs" fi fi echo "Checking YAML parsing..." for i in $(find $ELEMENTS_DIR -type f -name '*.yaml'); do echo "Parsing $i" py_check=" import yaml import sys try: objs = yaml.load(open('$i')) except yaml.parser.ParserError: sys.exit(1) " if ! python -c "$py_check"; then error "$i is not a valid YAML file" fi done echo "Checking pkg-map files..." for i in $(find $ELEMENTS_DIR -type f \ -name 'pkg-map' -a \! -executable); do echo "Parsing $i" py_check=" import json import sys try: objs = json.load(open('$i')) except ValueError: sys.exit(1) " if ! python -c "$py_check"; then error "$i is not a valid JSON file" fi done if [[ $rc == 0 ]]; then echo "PASS" else echo "*** FAIL: Some tests failed!" fi exit $rc