ff8ae43265
Avoid dangerous file parsing and object serialization libraries. yaml.load is the obvious function to use but it is dangerous[1] Because yaml.load return Python object may be dangerous if you receive a YAML document from an untrusted source such as the Internet. The function yaml.safe_load limits this ability to simple Python objects like integers or lists. In addition, Bandit flags yaml.load() as security risk so replace all occurrences with yaml.safe_load(). Thus I replace yaml.load() with yaml.safe_load() [1]https://security.openstack.org/guidelines/dg_avoid-dangerous-input-parsing-libraries.html Change-Id: I84640973fd9f45a69d2b21f6d594cd5bf10660a6 Closes-Bug: #1634265
268 lines
8.2 KiB
Bash
Executable File
268 lines
8.2 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Copyright 2014 Red Hat, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# This script checks all files in the "elements" directory for some
|
|
# common mistakes and exits with a non-zero status if it finds any.
|
|
|
|
set -eu
|
|
set -o pipefail
|
|
|
|
parse_exclusions() {
|
|
# Per-file exclusions
|
|
# Example: # dib-lint: disable=sete setpipefail
|
|
local filename=$1
|
|
local disable_pattern="# dib-lint: disable="
|
|
local exclusions=$(grep "^$disable_pattern.*$" $filename | sed "s/$disable_pattern//g")
|
|
|
|
# Global exclusions read from tox.ini
|
|
# Example section in tox.ini:
|
|
# [dib-lint]
|
|
# ignore = sete setu
|
|
section="dib-lint"
|
|
option="ignore"
|
|
global_exclusions=$(python -c \
|
|
"import ConfigParser; \
|
|
conf=ConfigParser.ConfigParser(); \
|
|
conf.read('tox.ini'); \
|
|
print conf.get('$section', '$option') if conf.has_option('$section', '$option') else ''"
|
|
)
|
|
echo $exclusions $global_exclusions
|
|
}
|
|
|
|
excluded() {
|
|
local test_name=$1
|
|
for e in $exclusions; do
|
|
if [ "$e" = "$test_name" ]; then
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
error() {
|
|
echo -e "ERROR: $1"
|
|
rc=1
|
|
}
|
|
|
|
echo "Running dib-lint in $(pwd)"
|
|
|
|
rc=0
|
|
TMPDIR=$(mktemp -d /tmp/tmp.XXXXXXXXXX)
|
|
trap "rm -rf $TMPDIR" EXIT
|
|
for i in $(find elements -type f \
|
|
-not -name \*~ \
|
|
-not -name \#\*\# \
|
|
-not -name \*.orig \
|
|
-not -name \*.rst \
|
|
-not -name \*.yaml \
|
|
-not -name \*.py \
|
|
-not -name \*.pyc); do
|
|
|
|
echo "Checking $i"
|
|
|
|
exclusions=("$(parse_exclusions $i)")
|
|
|
|
# Check that files starting with a shebang are +x
|
|
firstline=$(head -n 1 "$i")
|
|
if [ "${firstline:0:2}" = "#!" ]; then
|
|
if [ ! -x "$i" ] && ! excluded executable; then
|
|
error "$i is not executable"
|
|
fi
|
|
|
|
# run flake8 over python files. note our "dib-python"
|
|
# interpreter can confuse the magic matching being done in
|
|
# "file" and make it think the file is not python;
|
|
# special-case it.
|
|
if [[ "$(file -b -k --mime-type $i)" =~ "text/x-python" ]] || \
|
|
[[ $firstline =~ "dib-python" ]]; then
|
|
flake8 $i || error "$i failed flake8"
|
|
else
|
|
# Ensure 4 spaces indent are used
|
|
if ! excluded indent ; then
|
|
indent_regex='^\( \{4\}\)* \{1,3\}[^ ]'
|
|
if grep -q "$indent_regex" ${i}; then
|
|
error "$i should use 4 spaces indent"
|
|
# outline the failing lines with line number
|
|
grep -n "$indent_regex" ${i}
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Check alphabetical ordering of element-deps
|
|
if [ $(basename $i) = "element-deps" ]; then
|
|
UNSORTED=${TMPDIR}/element-deps.unsorted
|
|
SORTED=${TMPDIR}/element-deps.sorted
|
|
grep -v -e '^#' -e '^$' $i > ${UNSORTED}
|
|
sort ${UNSORTED} > ${SORTED}
|
|
if [ -n "$(diff -c ${UNSORTED} ${SORTED})" ]; then
|
|
error "$i is not sorted alphabetically"
|
|
diff -y ${UNSORTED} ${SORTED}
|
|
fi
|
|
fi
|
|
|
|
# for consistency, let's just use #!/bin/bash everywhere (not
|
|
# /usr/bin/env, etc)
|
|
regex='^#!.*bash'
|
|
if [[ "$firstline" =~ $regex &&
|
|
"$firstline" != "#!/bin/bash" ]]; then
|
|
error "$i : only use #!/bin/bash for scripts"
|
|
fi
|
|
|
|
# Check that all scripts are set -eu -o pipefail and look for
|
|
# DIB_DEBUG_TRACE
|
|
# NOTE(bnemec): This doesn't verify that the set call occurs high
|
|
# enough in the file to be useful, but hopefully nobody will be
|
|
# sticking set calls at the end of their file to trick us. And if
|
|
# they are, that's easy enough to catch in reviews.
|
|
# Also, this is only going to check bash scripts - we've decided to
|
|
# explicitly require bash for any scripts that don't have a specific
|
|
# need to run under other shells, and any exceptions to that rule
|
|
# may not want these checks either.
|
|
if [[ "$firstline" =~ '#!/bin/bash' ]]; then
|
|
if ! excluded sete; then
|
|
if [ -z "$(grep "^set -[^ ]*e" $i)" ]; then
|
|
error "$i is not set -e"
|
|
fi
|
|
fi
|
|
if ! excluded setu; then
|
|
if [ -z "$(grep "^set -[^ ]*u" $i)" ]; then
|
|
error "$i is not set -u"
|
|
fi
|
|
fi
|
|
if ! excluded setpipefail; then
|
|
if [ -z "$(grep "^set -o pipefail" $i)" ]; then
|
|
error "$i is not set -o pipefail"
|
|
fi
|
|
fi
|
|
if ! excluded dibdebugtrace; then
|
|
if [ -z "$(grep "DIB_DEBUG_TRACE" $i)" ]; then
|
|
error "$i does not follow DIB_DEBUG_TRACE"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# check that environment files don't "set -x" and they have no executable
|
|
# bits set
|
|
if [[ "$i" =~ (environment.d) ]]; then
|
|
if grep -q "set -x" $i; then
|
|
error "Environment file $i should not set tracing"
|
|
fi
|
|
if [[ -x $i ]]; then
|
|
error "Environment file $i should not be marked as executable"
|
|
fi
|
|
fi
|
|
|
|
# check for
|
|
# export FOO=$(bar)
|
|
# calls. These are dangerous, because the export hides the return
|
|
# code of the $(bar) call. Split this into 2 lines and -e will
|
|
# fail on the assignment
|
|
if grep -q 'export .*\$(' $i; then
|
|
error "Split export and assignments in $i"
|
|
fi
|
|
|
|
# check that sudo calls in phases run outside the chroot look
|
|
# "safe"; meaning that they seem to operate within the chroot
|
|
# somehow. This is not fool-proof, but catches egregious errors,
|
|
# and makes you think about it if you're doing something outside
|
|
# the box.
|
|
if ! excluded safe_sudo; then
|
|
if [[ $(dirname $i) =~ (root.d|extra-data.d|block-device.d|finalise.d|cleanup.d) ]]; then
|
|
while read LINE
|
|
do
|
|
if [[ $LINE =~ "sudo " ]]; then
|
|
# messy regex ahead! Don't match:
|
|
# - explicitly ignored
|
|
# - basic comments
|
|
# - install-packages ... sudo ...
|
|
# - any of the paths passed into the out-of-chroot elements
|
|
if [[ $LINE =~ (dib-lint: safe_sudo|^#|install-packages|TARGET_ROOT|IMAGE_BLOCK_DEVICE|TMP_MOUNT_PATH|TMP_IMAGE_PATH) ]]; then
|
|
continue
|
|
fi
|
|
error "$i : potentially unsafe sudo\n -- $LINE"
|
|
fi
|
|
done < $i
|
|
fi
|
|
fi
|
|
|
|
done
|
|
|
|
echo "Checking indents..."
|
|
|
|
for i in $(find bin elements -type f -and -name '*.rst' -or -type f -executable); do
|
|
# Check for tab indentation
|
|
if ! excluded tabindent; then
|
|
if grep -q $'^ *\t' ${i}; then
|
|
error "$i contains tab characters"
|
|
fi
|
|
fi
|
|
|
|
if ! excluded newline; then
|
|
if [ "$(tail -c 1 $i)" != "" ]; then
|
|
error "No newline at end of file: $i"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if ! excluded mddocs; then
|
|
md_docs=$(find elements -name '*.md')
|
|
if [ -n "$md_docs" ]; then
|
|
error ".md docs found: $md_docs"
|
|
fi
|
|
fi
|
|
|
|
echo "Checking YAML parsing..."
|
|
for i in $(find elements -type f -name '*.yaml'); do
|
|
echo "Parsing $i"
|
|
py_check="
|
|
import yaml
|
|
import sys
|
|
try:
|
|
objs = yaml.safe_load(open('$i'))
|
|
except yaml.parser.ParserError:
|
|
sys.exit(1)
|
|
"
|
|
if ! python -c "$py_check"; then
|
|
error "$i is not a valid YAML file"
|
|
fi
|
|
done
|
|
echo "Checking pkg-map files..."
|
|
for i in $(find elements -type f \
|
|
-name 'pkg-map' -a \! -executable); do
|
|
echo "Parsing $i"
|
|
py_check="
|
|
import json
|
|
import sys
|
|
try:
|
|
objs = json.load(open('$i'))
|
|
except ValueError:
|
|
sys.exit(1)
|
|
"
|
|
if ! python -c "$py_check"; then
|
|
error "$i is not a valid JSON file"
|
|
fi
|
|
done
|
|
|
|
if [[ $rc == 0 ]]; then
|
|
echo "PASS"
|
|
else
|
|
echo "*** FAIL: Some tests failed!"
|
|
fi
|
|
|
|
exit $rc
|