Merge pull request #21 from davidgiven/dtrg-tests

Run all tests, even if some fail, and produce a test summary at the end of the build.

Makefile

@@ -63,7 +63,7 @@ PLATDEP = $(INSDIR)/lib/ack
 
 .NOTPARALLEL:
 
-MAKECMDGOALS ?= +ack
+MAKECMDGOALS ?= +ack +tests
 BUILD_FILES = $(shell find * -name '*.lua')
 
 ifneq ($(shell which ninja),)

build.lua

@@ -46,8 +46,18 @@ installable {
 		"examples+pkg",
 		plat_packages
 	},
-	deps = {
-		test_packages
-	}
 }
 
+normalrule {
+	name = "tests",
+	ins = {
+		"first/testsummary.sh",
+		test_packages
+	},
+	outleaves = {
+		"stamp"
+	},
+	commands = {
+		"%{ins}"
+	}
+}

first/testsummary.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+echo ""
+
+succeeding="$(find "$@" -size 0)"
+notsucceeding="$(find "$@" ! -size 0)"
+skipped="$(grep -l @@SKIPPED $notsucceeding)"
+timedout="$(grep -l @@TIMEDOUT $notsucceeding)"
+failed="$(grep -l @@FAIL $notsucceeding)"
+
+for a in $failed $timedout; do
+    echo "**** $a"
+    cat $a
+    echo ""
+done
+
+echo "$(echo $succeeding | wc -w) tests passed"
+echo "$(echo $notsucceeding | wc -w) tests failed to pass"
+echo "$(echo $skipped | wc -w) were skipped (see build log for details)"
+echo "$(echo $timedout | wc -w) timed out"
+echo "$(echo $failed | wc -w) failed"
+echo ""
+
+if [ "$failed" != "" -o "$timedout" != "" ]; then
+	echo "Test status: SAD FACE (tests are failing)"
+	exit 1
+fi
+if [ "$succeeding" = "" ]; then
+	echo "Test status: PUZZLED FACE (all tests were skipped)"
+	exit 0
+fi
+if [ "$skipped" != "" ]; then
+	echo "Test status: MILDLY PLEASED FACE (some tests were skipped, but the rest pass)"
+	exit 0
+fi
+echo "Test status: HAPPY FACE (all tests are passing)"
+exit 0
+

tests/plat/_dummy.c

@@ -6,4 +6,3 @@ void _m_a_i_n(void)
     ASSERT(0 == 0);
     finished();
 }
-

tests/plat/build.lua

@@ -42,24 +42,21 @@ definerule("plat_testsuite",
 
 			tests[#tests+1] = normalrule {
 				name = fs,
-				outleaves = { "stamp" },
+				outleaves = { e.plat.."-"..fs.."-testlog.txt" },
 				ins = {
 					bin,
 					"tests/plat/testdriver.sh",
 					"util/build+testrunner"
 				},
 				commands = {
-					"%{ins[2]} "..e.method.." %{ins[1]} 5 %{ins[3]}",
-					"touch %{outs}"
+					"(%{ins[2]} "..e.method.." %{ins[1]} 5 %{ins[3]} || echo FAILED) 2>&1 > %{outs}",
 				}
 			}
 		end
 
-		return normalrule {
+		return bundle {
 			name = e.name,
-			outleaves = { "stamp" },
-			ins = tests,
-			commands = { "touch %{outs}" }
+			srcs = tests,
 		}
 	end
 )

tests/plat/lib/test.c

@@ -26,7 +26,10 @@ void writehex(uint32_t code)
 
 void fail(uint32_t code)
 {
-    write(1, "@@FAIL 0x", 10);
+    static const char fail_msg[] = "@@FAIL 0x";
+    static const char nl_msg[] = "\n";
+
+    write(1, fail_msg, sizeof(fail_msg)-1);
     writehex(code);
-    write(1, "\n", 1);
+    write(1, nl_msg, sizeof(nl_msg)-1);
 }

tests/plat/testdriver.sh

@@ -18,6 +18,7 @@ get_test_output() {
         qemu-system-*)
             if ! command -v $method >/dev/null 2>&1 ; then
                 errcho "Warning: $method not installed, skipping test"
+                echo "@@SKIPPED"
                 exit 0
             fi
 
@@ -26,16 +27,17 @@ get_test_output() {
                 qemu-system-ppc)  img="-kernel $img" ;;
             esac
 
-            $timeoutprog -t $timeout -- $method -nographic $img > $result
+            $timeoutprog -t $timeout -- $method -nographic $img 2>&1 > $result
             ;;
 
         qemu-*)
             if ! command -v $method >/dev/null 2>&1 ; then
                 errcho "Warning: $method not installed, skipping test"
+                echo "@@SKIPPED"
                 exit 0
             fi
 
-            $method $img > $result
+            $method $img 2>&1 > $result
             ;;
 
         *)
@@ -45,6 +47,6 @@ get_test_output() {
     esac
 }
 
-get_test_output > $result
-( grep -q @@FAIL $result || ! grep -q @@FINISHED $result ) && cat $result && exit 1
+get_test_output
+( grep -q '@@FAIL\|@@SKIPPED' $result || ! grep -q @@FINISHED $result ) && cat $result && exit 1
 exit 0

util/build/build.lua

@@ -5,3 +5,4 @@ cprogram {
         "modules/src/data+lib"
     }
 }
+

util/build/testrunner.c

@@ -6,6 +6,7 @@
 #include <signal.h>
 #include <sys/wait.h>
 #include <setjmp.h>
+#include <ctype.h>
 #include "diagnostics.h"
 
 static bool timed_out = false;
@@ -57,6 +58,7 @@ int main(int argc, char* const argv[])
     FILE* childin;
     int wstatus;
     char buffer[4096];
+    char* p;
 
     parse_arguments(argc, argv);
 
@@ -90,9 +92,13 @@ int main(int argc, char* const argv[])
             break;
         fputs(buffer, stdout);
 
-        if (strcmp(buffer, "@@FINISHED\n") == 0)
+        p = buffer;
+        while (isspace(*p))
+            p++;
+
+        if (strcmp(p, "@@FINISHED\n") == 0)
             break;
-        if (strcmp(buffer, "@@FINISHED\r\n") == 0)
+        if (strcmp(p, "@@FINISHED\r\n") == 0)
             break;
     }
 
@@ -103,6 +109,9 @@ int main(int argc, char* const argv[])
     kill(pid, SIGKILL);
     waitpid(pid, &wstatus, 0);
     if (timed_out)
+    {
+        fprintf(stderr, "@@TIMEDOUT\n");
         exit(1);
+    }
     exit(WEXITSTATUS(wstatus));
 }