diff --git a/doc/source/developer/developing_elements.rst b/doc/source/developer/developing_elements.rst index bbbad47e..84a2fc83 100644 --- a/doc/source/developer/developing_elements.rst +++ b/doc/source/developer/developing_elements.rst @@ -372,6 +372,14 @@ In order to create a test case, follow these steps: * To exit early and indicate a success, touch a file ``/tmp/dib-test-should-fail`` in the image chroot, then exit 1. +Tests are run with ``tools/run_functests.sh``. Running +``run_functests.sh -l`` will show available tests (the example above +would be called ``apt-sources/test-case-1``, for example). Specify +your test (or a series of tests as separate arguments) on the command +line to run it. If it should not be run as part of the default CI +run, you can submit a change with it added to ``DEFAULT_SKIP_TESTS`` +in that file. + python """""" diff --git a/tests/README.rst b/tests/README.rst new file mode 100644 index 00000000..f2b9be69 --- /dev/null +++ b/tests/README.rst @@ -0,0 +1,12 @@ +DIB Testing +=========== + +There are several interfaces for testing various parts of DIB + + * run_dib_library_tests.sh : run unit-tests for internal + diskimage-builder library functions + + * run_functests.sh : run functional testing provided by elements + + * run_output_format_test.sh : run a test of image conversion + (requires docker) diff --git a/tests/image_output_formats.bash b/tests/image_output_formats.bash deleted file mode 100755 index 92ef97f9..00000000 --- a/tests/image_output_formats.bash +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -eux -set -o pipefail - -source $(dirname $0)/test_functions.bash - -test_formats="tar raw qcow2 docker aci" -for binary in qemu-img docker ; do - if [ -z "$(which $binary)" ]; then - echo "Warning: No $binary binary found, cowardly refusing to run tests." - exit 0 - fi -done - -for format in '' $test_formats; do - build_test_image $format - echo "Test passed for output formats '$format'." -done - -combined_format=$(echo $test_formats | tr ' ' ',') -build_test_image $combined_format -echo "Test passed for output format '$combined_format'." diff --git a/tests/dib_functions_test.sh b/tests/run_dib_library_tests.sh similarity index 100% rename from tests/dib_functions_test.sh rename to tests/run_dib_library_tests.sh diff --git a/tests/run_functests.sh b/tests/run_functests.sh index cb98cb1c..4bb53b50 100755 --- a/tests/run_functests.sh +++ b/tests/run_functests.sh @@ -1,13 +1,179 @@ #!/bin/bash -set -eux +set -eu set -o pipefail -element=${1:-} +BASE_DIR=$(cd $(dirname "$0")/.. && pwd) +export DIB_ELEMENTS=$BASE_DIR/elements +export DIB_CMD=$BASE_DIR/bin/disk-image-create -if [ -z $element ]; then - $(dirname $0)/image_output_formats.bash +# +# Default skip tests +# +# For time reasons, we do not run these tests by default; i.e. these +# tests are not run by "tox -e func" in the gate. +# +DEFAULT_SKIP_TESTS=( + # we run version pinned test in gate (this just runs latest) + fedora/build-succeeds +) + +# run_disk_element_test +# Run a disk-image-build .tar build of ELEMENT including any elements +# specified by TEST_ELEMENT +function run_disk_element_test() { + local test_element=$1 + local element=$2 + local dest_dir=$(mktemp -d) + + trap "rm -rf $dest_dir /tmp/dib-test-should-fail" EXIT + + if break="after-error" break_outside_target=1 \ + break_cmd="cp \$TMP_MOUNT_PATH/tmp/dib-test-should-fail /tmp/ 2>&1 > /dev/null || true" \ + ELEMENTS_PATH=$DIB_ELEMENTS:$DIB_ELEMENTS/$element/test-elements \ + $DIB_CMD -t tar -o $dest_dir/image -n $element $test_element; then + if ! [ -f "$dest_dir/image.tar" ]; then + echo "Error: Build failed for element: $element, test-element: $test_element." + echo "No image $dest_dir/image.tar found!" + exit 1 + else + if tar -tf $dest_dir/image.tar | grep -q /tmp/dib-test-should-fail; then + echo "Error: Element: $element, test-element $test_element should have failed, but passed." + exit 1 + else + echo "PASS: Element $element, test-element: $test_element" + fi + fi + else + if [ -f "/tmp/dib-test-should-fail" ]; then + echo "PASS: Element $element, test-element: $test_element" + else + echo "Error: Build failed for element: $element, test-element: $test_element." + exit 1 + fi + fi + + trap EXIT + rm -rf $dest_dir /tmp/dib-test-should-fail +} + +# run_ramdisk_element_test +# Run a disk-image-builder default build of ELEMENT including any +# elements specified by TEST_ELEMENT +function run_ramdisk_element_test() { + local test_element=$1 + local element=$2 + local dest_dir=$(mktemp -d) + + if ELEMENTS_PATH=$DIB_ELEMENTS/$element/test-elements \ + $DIB_CMD -o $dest_dir/image $element $test_element; then + # TODO(dtantsur): test also kernel presence once we sort out its naming + # problem (vmlinuz vs kernel) + if ! [ -f "$dest_dir/image.initramfs" ]; then + echo "Error: Build failed for element: $element, test-element: $test_element." + echo "No image $dest_dir/image.initramfs found!" + exit 1 + else + echo "PASS: Element $element, test-element: $test_element" + fi + else + echo "Error: Build failed for element: $element, test-element: $test_element." + exit 1 + fi +} + +# +# run_functests.sh +# run the functional tests for dib elements +# + +# find elements that have functional test elements. TESTS will be an +# array with each value being "element/test-element" +TESTS=() +for e in $DIB_ELEMENTS/*/test-elements/*; do + test_element=$(echo $e | awk 'BEGIN {FS="/"}{print $NF}') + element=$(echo $e | awk 'BEGIN {FS="/"}{print $(NF-2)}') + TESTS+=("$element/$test_element") +done + +while getopts ":hl" opt; do + case $opt in + h) + echo "run_functests.sh [-h] [-l] ..." + echo " -h : show this help" + echo " -l : list available tests" + echo " : functional test to run" + echo " Special test 'all' will run all tests" + exit 0 + ;; + l) + echo "The available functional tests are:" + echo + for t in ${TESTS[@]}; do + echo " $t" + done + echo + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +# cull the list of tests to run into TESTS_TO_RUN +TESTS_TO_RUN=() +title="" +if [[ -z "$@" ]]; then + # remove the skipped tests + title="Running default tests:" + for test in "${TESTS[@]}"; do + if [[ " ${DEFAULT_SKIP_TESTS[@]} " =~ " ${test} " ]]; then + continue + else + TESTS_TO_RUN+=("${test}") + fi + done +elif [[ $1 == "all" ]]; then + title="Running all tests:" + TESTS_TO_RUN=("${TESTS[@]}") +else + title="Running specified tests:" + for test in $@; do + if [[ ! " ${TESTS[@]} " =~ " ${test} " ]]; then + echo "${test} : not a known test (see -l)" + exit 1 + fi + TESTS_TO_RUN+=("${test}") + done fi -$(dirname $0)/test_elements.bash $element + +# print a little status info +echo "------" +echo ${title} +for test in "${TESTS_TO_RUN[@]}"; do + echo " ${test}" +done +echo "------" + +for test in "${TESTS_TO_RUN[@]}"; do + # from above; each array value is element/test_element. split it + # back up + element=${test%/*} + test_element=${test#*/} + + # tests default to disk-based, but "element-type" can optionally + # override that + element_type=disk + element_type_override=$DIB_ELEMENTS/${element}/test-elements/${test_element}/element-type + if [ -f ${element_type_override} ]; then + element_type=$(cat ${element_type_override}) + fi + + echo "Running $test ($element_type)" + run_${element_type}_element_test $test_element $element +done echo "Tests passed!" diff --git a/tests/run_output_format_test.sh b/tests/run_output_format_test.sh new file mode 100755 index 00000000..f42d0ac8 --- /dev/null +++ b/tests/run_output_format_test.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +set -eu +set -o pipefail + +# +# run_output_format_test.sh +# +# Use docker to test generation of various output formats. +# + +BASE_DIR=$(cd $(dirname "$0")/.. && pwd) +export DIB_ELEMENTS=$BASE_DIR/elements +export TEST_ELEMENTS=$BASE_DIR/tests/elements +export DIB_CMD=$BASE_DIR/bin/disk-image-create + +function build_test_image() { + format=${1:-} + + if [ -n "$format" ]; then + type_arg="-t $format" + else + type_arg= + format="qcow2" + fi + dest_dir=$(mktemp -d) + base_dest=$(basename $dest_dir) + + trap "rm -rf $dest_dir; docker rmi $base_dest/image" EXIT + + ELEMENTS_PATH=$DIB_ELEMENTS:$TEST_ELEMENTS \ + $DIB_CMD -x $type_arg --docker-target=$base_dest/image \ + -o $dest_dir/image -n fake-os + + format=$(echo $format | tr ',' ' ') + for format in $format; do + if [ $format != 'docker' ]; then + img_path="$dest_dir/image.$format" + if ! [ -f "$img_path" ]; then + echo "Error: No image with name $img_path found!" + exit 1 + else + echo "Found image $img_path." + fi + else + if ! docker images | grep $base_dest/image ; then + echo "Error: No docker image with name $base_dest/image found!" + exit 1 + else + echo "Found docker image $base_dest/image" + fi + fi + done + + trap EXIT + rm -rf $dest_dir + if docker images | grep $base_dest/image ; then + docker rmi $base_dest/image + fi +} + +test_formats="tar raw qcow2 docker aci" +for binary in qemu-img docker ; do + if [ -z "$(which $binary)" ]; then + echo "Warning: No $binary binary found, cowardly refusing to run tests." + exit 0 + fi +done + +for format in '' $test_formats; do + build_test_image $format + echo "Test passed for output formats '$format'." +done + +combined_format=$(echo $test_formats | tr ' ' ',') +build_test_image $combined_format +echo "Test passed for output format '$combined_format'." diff --git a/tests/test_elements.bash b/tests/test_elements.bash deleted file mode 100755 index 9acaa2bb..00000000 --- a/tests/test_elements.bash +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -set -eux -set -o pipefail - -basedir=$(dirname $0) -requested_element=${1:-} -source $basedir/test_functions.bash - -function run_on_element { - test_element=$1 - # our element name is two dirs up - local element_name=$(basename $(dirname $(dirname $test_element))) - local element_type=disk - if [ -f "$test_element/element-type" ]; then - element_type=$(cat "$test_element/element-type") - fi - run_${element_type}_element_test "$(basename $test_element)" "$element_name" -} - -if [ -z $requested_element ]; then - for test_element in $basedir/../elements/*/test-elements/*; do - if [ -d "$test_element" ]; then - run_on_element "$test_element" - fi - done -else - for test_element in $basedir/../elements/$requested_element/test-elements/*; do - if [ -d "$test_element" ]; then - run_on_element "$test_element" - fi - done -fi diff --git a/tests/test_functions.bash b/tests/test_functions.bash deleted file mode 100644 index 37344ed8..00000000 --- a/tests/test_functions.bash +++ /dev/null @@ -1,108 +0,0 @@ -export TEST_ELEMENTS=$(dirname $0)/elements -export DIB_ELEMENTS=$(dirname $0)/../elements -export DIB_CMD=$(dirname $0)/../bin/disk-image-create - -function build_test_image() { - format=${1:-} - - if [ -n "$format" ]; then - type_arg="-t $format" - else - type_arg= - format="qcow2" - fi - dest_dir=$(mktemp -d) - base_dest=$(basename $dest_dir) - - trap "rm -rf $dest_dir; docker rmi $base_dest/image" EXIT - - ELEMENTS_PATH=$DIB_ELEMENTS:$TEST_ELEMENTS \ - $DIB_CMD -x $type_arg --docker-target=$base_dest/image \ - -o $dest_dir/image -n fake-os - - format=$(echo $format | tr ',' ' ') - for format in $format; do - if [ $format != 'docker' ]; then - img_path="$dest_dir/image.$format" - if ! [ -f "$img_path" ]; then - echo "Error: No image with name $img_path found!" - exit 1 - else - echo "Found image $img_path." - fi - else - if ! docker images | grep $base_dest/image ; then - echo "Error: No docker image with name $base_dest/image found!" - exit 1 - else - echo "Found docker image $base_dest/image" - fi - fi - done - - trap EXIT - rm -rf $dest_dir - if docker images | grep $base_dest/image ; then - docker rmi $base_dest/image - fi -} - -function run_disk_element_test() { - test_element=$1 - element=$2 - - dest_dir=$(mktemp -d) - - trap "rm -rf $dest_dir /tmp/dib-test-should-fail" EXIT - - if break="after-error" break_outside_target=1 \ - break_cmd="cp \$TMP_MOUNT_PATH/tmp/dib-test-should-fail /tmp/ 2>&1 > /dev/null || true" \ - ELEMENTS_PATH=$DIB_ELEMENTS:$DIB_ELEMENTS/$element/test-elements \ - $DIB_CMD -t tar -o $dest_dir/image -n $element $test_element; then - if ! [ -f "$dest_dir/image.tar" ]; then - echo "Error: Build failed for element: $element, test-element: $test_element." - echo "No image $dest_dir/image.tar found!" - exit 1 - else - if tar -tf $dest_dir/image.tar | grep -q /tmp/dib-test-should-fail; then - echo "Error: Element: $element, test-element $test_element should have failed, but passed." - exit 1 - else - echo "PASS: Element $element, test-element: $test_element" - fi - fi - else - if [ -f "/tmp/dib-test-should-fail" ]; then - echo "PASS: Element $element, test-element: $test_element" - else - echo "Error: Build failed for element: $element, test-element: $test_element." - exit 1 - fi - fi - - trap EXIT - rm -rf $dest_dir /tmp/dib-test-should-fail -} - -function run_ramdisk_element_test() { - test_element=$1 - element=$2 - - dest_dir=$(mktemp -d) - - if ELEMENTS_PATH=$DIB_ELEMENTS/$element/test-elements \ - $DIB_CMD -o $dest_dir/image $element $test_element; then - # TODO(dtantsur): test also kernel presence once we sort out its naming - # problem (vmlinuz vs kernel) - if ! [ -f "$dest_dir/image.initramfs" ]; then - echo "Error: Build failed for element: $element, test-element: $test_element." - echo "No image $dest_dir/image.initramfs found!" - exit 1 - else - echo "PASS: Element $element, test-element: $test_element" - fi - else - echo "Error: Build failed for element: $element, test-element: $test_element." - exit 1 - fi -}