Initial element tests

Adding a test function which allows us to use elements to perform
element-specific tests. In order for this to work sanely, also adding
some configuration to our break system so we can assert on negative
tests.

Also adding a test for apt-sources to verify this code actually works.

Change-Id: I378a74255010eca192f5766b653f8a42404be5ea

doc/source/conf.py

@@ -45,7 +45,8 @@ version = dib_version.version_info.version_string()
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build', 'doc/build', '.tox', '.venv']
+exclude_patterns = ['_build', 'doc/build', '.tox', '.venv',
+                    'elements/*/test-elements']
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.

doc/source/developer/developing_elements.rst

@@ -266,7 +266,24 @@ interfaces or disks are not detected correctly).
 Testing Elements
 ----------------
 
-Elements can be tested using python. To create a test:
+An element can have functional tests encapsulated inside the element itself. In
+order to create a test case, follow these steps:
+
+* Create a directory called 'test-elements' inside your element.
+
+* Inside the test-elements directory, create a directory with the name of your
+  test case. The test case directory should have the same structure as an
+  element.
+  i.e. elements/apt-sources/test-elements/test-case-1
+
+* Assert state during each of the element build phases you would like to test.
+  You can exit 1 to indicate a failure.
+
+* To exit early and indicate a success, touch a file /tmp/dib-test-should-fail
+  in the image chroot, then exit 1.
+
+Additionally, elements can be tested using python unittests. To create a
+a python test:
 
 * Create a directory called 'tests' in the element directory.
 

elements/apt-sources/test-elements/test-sources/element-deps

@@ -0,0 +1,2 @@
+base
+ubuntu

elements/apt-sources/test-elements/test-sources/environment.d/00-set-apt-sources

@@ -0,0 +1 @@
+export DIB_APT_SOURCES=$(mktemp)

elements/apt-sources/test-elements/test-sources/extra-data.d/00-write-apt-sources

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+echo "testdata" > $DIB_APT_SOURCES

elements/apt-sources/test-elements/test-sources/pre-install.d/00-test-apt-sources

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eux
+set -o pipefail
+
+echo "Verifying apt sources.list content"
+[ -f /etc/apt/sources.list ]
+[ "$(cat /etc/apt/sources.list)" = "testdata" ]
+
+touch /tmp/dib-test-should-fail && exit 1

lib/img-functions

@@ -79,7 +79,8 @@ function run_d_in_target () {
       sudo mount --bind ${TMP_HOOKS_PATH} $TMP_MOUNT_PATH/tmp/in_target.d
       sudo mount -o remount,ro,bind ${TMP_HOOKS_PATH} $TMP_MOUNT_PATH/tmp/in_target.d
       check_break before-$1 run_in_target bash
-      trap "check_break after-error run_in_target bash" ERR
+      [ -z "$break_outside_target" ] && in_target_arg="run_in_target" || in_target_arg=
+      trap "check_break after-error $in_target_arg ${break_cmd:-bash}" ERR
       run_in_target dib-run-parts /tmp/in_target.d/$1.d
       trap - ERR
       check_break after-$1 run_in_target bash

tests/run_functests.sh

@@ -4,5 +4,6 @@ set -eux
 set -o pipefail
 
 $(dirname $0)/image_output_formats.bash
+$(dirname $0)/test_elements.bash
 
 echo "Tests passed!"

tests/test_elements.bash

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -eux
+set -o pipefail
+
+basedir=$(dirname $0)
+source $basedir/test_functions.bash
+
+for test_element in $basedir/../elements/*/test-elements/*; do
+    if [ -d "$test_element" ]; then
+        # our element name is two dirs up
+        element_name=$(basename $(dirname $(dirname $test_element)))
+        run_element_test "$(basename $test_element)" "$element_name"
+    fi
+done

tests/test_functions.bash

@@ -16,7 +16,7 @@ function build_test_image() {
     trap "rm -rf $dest_dir" EXIT
 
     ELEMENTS_PATH=$DIB_ELEMENTS:$TEST_ELEMENTS \
-        $DIB_CMD $type_arg -o $dest_dir/image -n fake-os
+        $DIB_CMD -x $type_arg -o $dest_dir/image -n fake-os
 
     format=$(echo $format | tr ',' ' ')
     for format in $format; do
@@ -32,3 +32,40 @@ function build_test_image() {
     trap EXIT
     rm -rf $dest_dir
 }
+
+function run_element_test() {
+    test_element=$1
+    element=$2
+
+    dest_dir=$(mktemp -d)
+
+    trap "rm -rf $dest_dir /tmp/dib-test-should-fail" EXIT
+
+    if break="after-error" break_outside_target=1 \
+        break_cmd="cp \$TMP_MOUNT_PATH/tmp/dib-test-should-fail /tmp/ || true" \
+        ELEMENTS_PATH=$DIB_ELEMENTS:$DIB_ELEMENTS/$element/test-elements \
+        $DIB_CMD -t tar -o $dest_dir/image -n $element $test_element; then
+        if ! [ -f "$dest_dir/image.tar" ]; then
+            echo "Error: Build failed for element: $element, test-element: $test_element."
+            echo "No image $dest_dir/image.tar found!"
+            exit 1
+        else
+            if tar -l $dest_dir/image.tar | grep -q /tmp/dib-test-should-fail; then
+                echo "Error: Element: $element, test-element $test_element should have failed, but passed."
+                exit 1
+            else
+                echo "PASS: Element $element, test-element: $test_element"
+            fi
+        fi
+    else
+        if [ -f "/tmp/dib-test-should-fail" ]; then
+            echo "PASS: Element $element, test-element: $test_element"
+        else
+            echo "Error: Build failed for element: $element, test-element: $test_element."
+            exit 1
+        fi
+    fi
+
+    trap EXIT
+    rm -rf $dest_dir /tmp/dib-test-should-fail
+}