Merge "Rework functional test runner"

doc/source/developer/developing_elements.rst

@@ -372,6 +372,14 @@ In order to create a test case, follow these steps:
 * To exit early and indicate a success, touch a file
   ``/tmp/dib-test-should-fail`` in the image chroot, then exit 1.
 
+Tests are run with ``tools/run_functests.sh``.  Running
+``run_functests.sh -l`` will show available tests (the example above
+would be called ``apt-sources/test-case-1``, for example).  Specify
+your test (or a series of tests as separate arguments) on the command
+line to run it.  If it should not be run as part of the default CI
+run, you can submit a change with it added to ``DEFAULT_SKIP_TESTS``
+in that file.
+
 python
 """"""
 

tests/README.rst

@@ -0,0 +1,12 @@
+DIB Testing
+===========
+
+There are several interfaces for testing various parts of DIB
+
+ * run_dib_library_tests.sh : run unit-tests for internal
+     diskimage-builder library functions
+
+ * run_functests.sh : run functional testing provided by elements
+
+ * run_output_format_test.sh : run a test of image conversion
+     (requires docker)

tests/image_output_formats.bash

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-set -eux
-set -o pipefail
-
-source $(dirname $0)/test_functions.bash
-
-test_formats="tar raw qcow2 docker aci"
-for binary in qemu-img docker ; do
-    if [ -z "$(which $binary)" ]; then
-        echo "Warning: No $binary binary found, cowardly refusing to run tests."
-        exit 0
-    fi
-done
-
-for format in '' $test_formats; do
-    build_test_image $format
-    echo "Test passed for output formats '$format'."
-done
-
-combined_format=$(echo $test_formats | tr ' ' ',')
-build_test_image $combined_format
-echo "Test passed for output format '$combined_format'."

tests/run_functests.sh

@@ -1,13 +1,179 @@
 #!/bin/bash
 
-set -eux
+set -eu
 set -o pipefail
 
-element=${1:-}
+BASE_DIR=$(cd $(dirname "$0")/.. && pwd)
+export DIB_ELEMENTS=$BASE_DIR/elements
+export DIB_CMD=$BASE_DIR/bin/disk-image-create
 
-if [ -z $element ]; then
+#
-    $(dirname $0)/image_output_formats.bash
+# Default skip tests
+#
+#  For time reasons, we do not run these tests by default; i.e. these
+#  tests are not run by "tox -e func" in the gate.
+#
+DEFAULT_SKIP_TESTS=(
+    # we run version pinned test in gate (this just runs latest)
+    fedora/build-succeeds
+)
+
+# run_disk_element_test <test_element> <element>
+#  Run a disk-image-build .tar build of ELEMENT including any elements
+#  specified by TEST_ELEMENT
+function run_disk_element_test() {
+    local test_element=$1
+    local element=$2
+    local dest_dir=$(mktemp -d)
+
+    trap "rm -rf $dest_dir /tmp/dib-test-should-fail" EXIT
+
+    if break="after-error" break_outside_target=1 \
+        break_cmd="cp \$TMP_MOUNT_PATH/tmp/dib-test-should-fail /tmp/ 2>&1 > /dev/null || true" \
+        ELEMENTS_PATH=$DIB_ELEMENTS:$DIB_ELEMENTS/$element/test-elements \
+        $DIB_CMD -t tar -o $dest_dir/image -n $element $test_element; then
+        if ! [ -f "$dest_dir/image.tar" ]; then
+            echo "Error: Build failed for element: $element, test-element: $test_element."
+            echo "No image $dest_dir/image.tar found!"
+            exit 1
+        else
+            if tar -tf $dest_dir/image.tar | grep -q /tmp/dib-test-should-fail; then
+                echo "Error: Element: $element, test-element $test_element should have failed, but passed."
+                exit 1
+            else
+                echo "PASS: Element $element, test-element: $test_element"
             fi
-$(dirname $0)/test_elements.bash $element
+        fi
+    else
+        if [ -f "/tmp/dib-test-should-fail" ]; then
+            echo "PASS: Element $element, test-element: $test_element"
+        else
+            echo "Error: Build failed for element: $element, test-element: $test_element."
+            exit 1
+        fi
+    fi
+
+    trap EXIT
+    rm -rf $dest_dir /tmp/dib-test-should-fail
+}
+
+# run_ramdisk_element_test <test_element> <element>
+#  Run a disk-image-builder default build of ELEMENT including any
+#  elements specified by TEST_ELEMENT
+function run_ramdisk_element_test() {
+    local test_element=$1
+    local element=$2
+    local dest_dir=$(mktemp -d)
+
+    if ELEMENTS_PATH=$DIB_ELEMENTS/$element/test-elements \
+        $DIB_CMD -o $dest_dir/image $element $test_element; then
+        # TODO(dtantsur): test also kernel presence once we sort out its naming
+        # problem (vmlinuz vs kernel)
+        if ! [ -f "$dest_dir/image.initramfs" ]; then
+            echo "Error: Build failed for element: $element, test-element: $test_element."
+            echo "No image $dest_dir/image.initramfs found!"
+            exit 1
+        else
+            echo "PASS: Element $element, test-element: $test_element"
+        fi
+    else
+        echo "Error: Build failed for element: $element, test-element: $test_element."
+        exit 1
+    fi
+}
+
+#
+# run_functests.sh
+#  run the functional tests for dib elements
+#
+
+# find elements that have functional test elements.  TESTS will be an
+# array with each value being "element/test-element"
+TESTS=()
+for e in $DIB_ELEMENTS/*/test-elements/*; do
+    test_element=$(echo $e | awk 'BEGIN {FS="/"}{print $NF}')
+    element=$(echo $e | awk 'BEGIN {FS="/"}{print $(NF-2)}')
+    TESTS+=("$element/$test_element")
+done
+
+while getopts ":hl" opt; do
+    case $opt in
+        h)
+            echo "run_functests.sh [-h] [-l] <test> <test> ..."
+            echo "  -h : show this help"
+            echo "  -l : list available tests"
+            echo "  <test> : functional test to run"
+            echo "           Special test 'all' will run all tests"
+            exit 0
+            ;;
+        l)
+            echo "The available functional tests are:"
+            echo
+            for t in ${TESTS[@]}; do
+                echo "  $t"
+            done
+            echo
+            exit 0
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG"
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+# cull the list of tests to run into TESTS_TO_RUN
+TESTS_TO_RUN=()
+title=""
+if [[ -z "$@" ]]; then
+    # remove the skipped tests
+    title="Running default tests:"
+    for test in "${TESTS[@]}"; do
+        if [[ " ${DEFAULT_SKIP_TESTS[@]} " =~ " ${test} " ]]; then
+            continue
+        else
+            TESTS_TO_RUN+=("${test}")
+        fi
+    done
+elif [[ $1 == "all" ]]; then
+    title="Running all tests:"
+    TESTS_TO_RUN=("${TESTS[@]}")
+else
+    title="Running specified tests:"
+    for test in $@; do
+        if [[ ! " ${TESTS[@]} " =~ " ${test} " ]]; then
+            echo "${test} : not a known test (see -l)"
+            exit 1
+        fi
+        TESTS_TO_RUN+=("${test}")
+    done
+fi
+
+# print a little status info
+echo "------"
+echo ${title}
+for test in "${TESTS_TO_RUN[@]}"; do
+    echo "  ${test}"
+done
+echo "------"
+
+for test in "${TESTS_TO_RUN[@]}"; do
+    # from above; each array value is element/test_element.  split it
+    # back up
+    element=${test%/*}
+    test_element=${test#*/}
+
+    # tests default to disk-based, but "element-type" can optionally
+    # override that
+    element_type=disk
+    element_type_override=$DIB_ELEMENTS/${element}/test-elements/${test_element}/element-type
+    if [ -f ${element_type_override} ]; then
+        element_type=$(cat ${element_type_override})
+    fi
+
+    echo "Running $test ($element_type)"
+    run_${element_type}_element_test $test_element $element
+done
 
 echo "Tests passed!"

tests/run_output_format_test.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+#
+# run_output_format_test.sh
+#
+#  Use docker to test generation of various output formats.
+#
+
+BASE_DIR=$(cd $(dirname "$0")/.. && pwd)
+export DIB_ELEMENTS=$BASE_DIR/elements
+export TEST_ELEMENTS=$BASE_DIR/tests/elements
+export DIB_CMD=$BASE_DIR/bin/disk-image-create
+
+function build_test_image() {
+    format=${1:-}
+
+    if [ -n "$format" ]; then
+        type_arg="-t $format"
+    else
+        type_arg=
+        format="qcow2"
+    fi
+    dest_dir=$(mktemp -d)
+    base_dest=$(basename $dest_dir)
+
+    trap "rm -rf $dest_dir; docker rmi $base_dest/image" EXIT
+
+    ELEMENTS_PATH=$DIB_ELEMENTS:$TEST_ELEMENTS \
+        $DIB_CMD -x $type_arg --docker-target=$base_dest/image \
+        -o $dest_dir/image -n fake-os
+
+    format=$(echo $format | tr ',' ' ')
+    for format in $format; do
+        if [ $format != 'docker' ]; then
+            img_path="$dest_dir/image.$format"
+            if ! [ -f "$img_path" ]; then
+                echo "Error: No image with name $img_path found!"
+                exit 1
+            else
+                echo "Found image $img_path."
+            fi
+        else
+            if ! docker images | grep $base_dest/image ; then
+                echo "Error: No docker image with name $base_dest/image found!"
+                exit 1
+            else
+                echo "Found docker image $base_dest/image"
+            fi
+        fi
+    done
+
+    trap EXIT
+    rm -rf $dest_dir
+    if docker images | grep $base_dest/image ; then
+        docker rmi $base_dest/image
+    fi
+}
+
+test_formats="tar raw qcow2 docker aci"
+for binary in qemu-img docker ; do
+    if [ -z "$(which $binary)" ]; then
+        echo "Warning: No $binary binary found, cowardly refusing to run tests."
+        exit 0
+    fi
+done
+
+for format in '' $test_formats; do
+    build_test_image $format
+    echo "Test passed for output formats '$format'."
+done
+
+combined_format=$(echo $test_formats | tr ' ' ',')
+build_test_image $combined_format
+echo "Test passed for output format '$combined_format'."

tests/test_elements.bash

@@ -1,33 +0,0 @@
-#!/bin/bash
-
-set -eux
-set -o pipefail
-
-basedir=$(dirname $0)
-requested_element=${1:-}
-source $basedir/test_functions.bash
-
-function run_on_element {
-    test_element=$1
-    # our element name is two dirs up
-    local element_name=$(basename $(dirname $(dirname $test_element)))
-    local element_type=disk
-    if [ -f "$test_element/element-type" ]; then
-        element_type=$(cat "$test_element/element-type")
-    fi
-    run_${element_type}_element_test "$(basename $test_element)" "$element_name"
-}
-
-if [ -z $requested_element ]; then
-    for test_element in $basedir/../elements/*/test-elements/*; do
-        if [ -d "$test_element" ]; then
-            run_on_element "$test_element"
-        fi
-    done
-else
-    for test_element in $basedir/../elements/$requested_element/test-elements/*; do
-        if [ -d "$test_element" ]; then
-            run_on_element "$test_element"
-        fi
-    done
-fi

tests/test_functions.bash

@@ -1,108 +0,0 @@
-export TEST_ELEMENTS=$(dirname $0)/elements
-export DIB_ELEMENTS=$(dirname $0)/../elements
-export DIB_CMD=$(dirname $0)/../bin/disk-image-create
-
-function build_test_image() {
-    format=${1:-}
-
-    if [ -n "$format" ]; then
-        type_arg="-t $format"
-    else
-        type_arg=
-        format="qcow2"
-    fi
-    dest_dir=$(mktemp -d)
-    base_dest=$(basename $dest_dir)
-
-    trap "rm -rf $dest_dir; docker rmi $base_dest/image" EXIT
-
-    ELEMENTS_PATH=$DIB_ELEMENTS:$TEST_ELEMENTS \
-        $DIB_CMD -x $type_arg --docker-target=$base_dest/image \
-        -o $dest_dir/image -n fake-os
-
-    format=$(echo $format | tr ',' ' ')
-    for format in $format; do
-        if [ $format != 'docker' ]; then
-            img_path="$dest_dir/image.$format"
-            if ! [ -f "$img_path" ]; then
-                echo "Error: No image with name $img_path found!"
-                exit 1
-            else
-                echo "Found image $img_path."
-            fi
-        else
-            if ! docker images | grep $base_dest/image ; then
-                echo "Error: No docker image with name $base_dest/image found!"
-                exit 1
-            else
-                echo "Found docker image $base_dest/image"
-            fi
-        fi
-    done
-
-    trap EXIT
-    rm -rf $dest_dir
-    if docker images | grep $base_dest/image ; then
-        docker rmi $base_dest/image
-    fi
-}
-
-function run_disk_element_test() {
-    test_element=$1
-    element=$2
-
-    dest_dir=$(mktemp -d)
-
-    trap "rm -rf $dest_dir /tmp/dib-test-should-fail" EXIT
-
-    if break="after-error" break_outside_target=1 \
-        break_cmd="cp \$TMP_MOUNT_PATH/tmp/dib-test-should-fail /tmp/ 2>&1 > /dev/null || true" \
-        ELEMENTS_PATH=$DIB_ELEMENTS:$DIB_ELEMENTS/$element/test-elements \
-        $DIB_CMD -t tar -o $dest_dir/image -n $element $test_element; then
-        if ! [ -f "$dest_dir/image.tar" ]; then
-            echo "Error: Build failed for element: $element, test-element: $test_element."
-            echo "No image $dest_dir/image.tar found!"
-            exit 1
-        else
-            if tar -tf $dest_dir/image.tar | grep -q /tmp/dib-test-should-fail; then
-                echo "Error: Element: $element, test-element $test_element should have failed, but passed."
-                exit 1
-            else
-                echo "PASS: Element $element, test-element: $test_element"
-            fi
-        fi
-    else
-        if [ -f "/tmp/dib-test-should-fail" ]; then
-            echo "PASS: Element $element, test-element: $test_element"
-        else
-            echo "Error: Build failed for element: $element, test-element: $test_element."
-            exit 1
-        fi
-    fi
-
-    trap EXIT
-    rm -rf $dest_dir /tmp/dib-test-should-fail
-}
-
-function run_ramdisk_element_test() {
-    test_element=$1
-    element=$2
-
-    dest_dir=$(mktemp -d)
-
-    if ELEMENTS_PATH=$DIB_ELEMENTS/$element/test-elements \
-        $DIB_CMD -o $dest_dir/image $element $test_element; then
-        # TODO(dtantsur): test also kernel presence once we sort out its naming
-        # problem (vmlinuz vs kernel)
-        if ! [ -f "$dest_dir/image.initramfs" ]; then
-            echo "Error: Build failed for element: $element, test-element: $test_element."
-            echo "No image $dest_dir/image.initramfs found!"
-            exit 1
-        else
-            echo "PASS: Element $element, test-element: $test_element"
-        fi
-    else
-        echo "Error: Build failed for element: $element, test-element: $test_element."
-        exit 1
-    fi
-}