From 8c99e2b7ad7647197f9f0f96e82f8d777dc0edcc Mon Sep 17 00:00:00 2001
From: David Given <dg@cowlark.com>
Date: Thu, 1 Dec 2016 23:03:30 +0100
Subject: [PATCH] Run all tests, even the ones which fail, and emit a test
 summary right at the end of the build (and fail then).

---
 Makefile                 |  2 +-
 build.lua                | 16 +++++++++++++---
 first/testsummary.sh     | 11 +++++++++++
 tests/plat/_dummy.c      |  1 -
 tests/plat/build.lua     | 11 ++++-------
 tests/plat/lib/test.c    |  7 +++++--
 tests/plat/testdriver.sh |  6 +++---
 util/build/build.lua     |  1 +
 util/build/testrunner.c  | 13 +++++++++++--
 9 files changed, 49 insertions(+), 19 deletions(-)
 create mode 100755 first/testsummary.sh

diff --git a/Makefile b/Makefile
index febd87aff..4399f0c28 100644
--- a/Makefile
+++ b/Makefile
@@ -63,7 +63,7 @@ PLATDEP = $(INSDIR)/lib/ack
 
 .NOTPARALLEL:
 
-MAKECMDGOALS ?= +ack
+MAKECMDGOALS ?= +ack +tests
 BUILD_FILES = $(shell find * -name '*.lua')
 
 ifneq ($(shell which ninja),)
diff --git a/build.lua b/build.lua
index 607a83a3b..98ee9ff9b 100644
--- a/build.lua
+++ b/build.lua
@@ -46,8 +46,18 @@ installable {
 		"examples+pkg",
 		plat_packages
 	},
-	deps = {
-		test_packages
-	}
 }
 
+normalrule {
+	name = "tests",
+	ins = {
+		"first/testsummary.sh",
+		test_packages
+	},
+	outleaves = {
+		"stamp"
+	},
+	commands = {
+		"%{ins}"
+	}
+}
diff --git a/first/testsummary.sh b/first/testsummary.sh
new file mode 100755
index 000000000..4ad46d52d
--- /dev/null
+++ b/first/testsummary.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+failed=$(find "$@" ! -size 0)
+echo ""
+echo "$(echo $failed | wc -w) failed tests"
+echo ""
+for a in $failed; do
+    echo "**** $a"
+    cat $a
+    echo ""
+done
+exec test "$failed" == ""
diff --git a/tests/plat/_dummy.c b/tests/plat/_dummy.c
index a4693300f..48104b5aa 100644
--- a/tests/plat/_dummy.c
+++ b/tests/plat/_dummy.c
@@ -6,4 +6,3 @@ void _m_a_i_n(void)
     ASSERT(0 == 0);
     finished();
 }
-
diff --git a/tests/plat/build.lua b/tests/plat/build.lua
index be65668fd..f5885a190 100644
--- a/tests/plat/build.lua
+++ b/tests/plat/build.lua
@@ -42,24 +42,21 @@ definerule("plat_testsuite",
 
 			tests[#tests+1] = normalrule {
 				name = fs,
-				outleaves = { "stamp" },
+				outleaves = { e.plat.."-"..fs.."-testlog.txt" },
 				ins = {
 					bin,
 					"tests/plat/testdriver.sh",
 					"util/build+testrunner"
 				},
 				commands = {
-					"%{ins[2]} "..e.method.." %{ins[1]} 5 %{ins[3]}",
-					"touch %{outs}"
+					"(%{ins[2]} "..e.method.." %{ins[1]} 5 %{ins[3]} || echo FAILED) 2>&1 > %{outs}",
 				}
 			}
 		end
 
-		return normalrule {
+		return bundle {
 			name = e.name,
-			outleaves = { "stamp" },
-			ins = tests,
-			commands = { "touch %{outs}" }
+			srcs = tests,
 		}
 	end
 )
\ No newline at end of file
diff --git a/tests/plat/lib/test.c b/tests/plat/lib/test.c
index df00e1089..426f9944a 100644
--- a/tests/plat/lib/test.c
+++ b/tests/plat/lib/test.c
@@ -26,7 +26,10 @@ void writehex(uint32_t code)
 
 void fail(uint32_t code)
 {
-    write(1, "@@FAIL 0x", 10);
+    static const char fail_msg[] = "@@FAIL 0x";
+    static const char nl_msg[] = "\n";
+
+    write(1, fail_msg, sizeof(fail_msg)-1);
     writehex(code);
-    write(1, "\n", 1);
+    write(1, nl_msg, sizeof(nl_msg)-1);
 }
diff --git a/tests/plat/testdriver.sh b/tests/plat/testdriver.sh
index 6af14da0f..6c421f798 100755
--- a/tests/plat/testdriver.sh
+++ b/tests/plat/testdriver.sh
@@ -26,7 +26,7 @@ get_test_output() {
                 qemu-system-ppc)  img="-kernel $img" ;;
             esac
 
-            $timeoutprog -t $timeout -- $method -nographic $img > $result
+            $timeoutprog -t $timeout -- $method -nographic $img 2>&1 > $result
             ;;
 
         qemu-*)
@@ -35,7 +35,7 @@ get_test_output() {
                 exit 0
             fi
 
-            $method $img > $result
+            $method $img 2>&1 > $result
             ;;
 
         *)
@@ -45,6 +45,6 @@ get_test_output() {
     esac
 }
 
-get_test_output > $result
+get_test_output
 ( grep -q @@FAIL $result || ! grep -q @@FINISHED $result ) && cat $result && exit 1
 exit 0
diff --git a/util/build/build.lua b/util/build/build.lua
index 76623fe91..41bd0b6ad 100644
--- a/util/build/build.lua
+++ b/util/build/build.lua
@@ -5,3 +5,4 @@ cprogram {
         "modules/src/data+lib"
     }
 }
+
diff --git a/util/build/testrunner.c b/util/build/testrunner.c
index e666ec4ac..fb2add562 100644
--- a/util/build/testrunner.c
+++ b/util/build/testrunner.c
@@ -6,6 +6,7 @@
 #include <signal.h>
 #include <sys/wait.h>
 #include <setjmp.h>
+#include <ctype.h>
 #include "diagnostics.h"
 
 static bool timed_out = false;
@@ -57,6 +58,7 @@ int main(int argc, char* const argv[])
     FILE* childin;
     int wstatus;
     char buffer[4096];
+    char* p;
 
     parse_arguments(argc, argv);
 
@@ -90,9 +92,13 @@ int main(int argc, char* const argv[])
             break;
         fputs(buffer, stdout);
 
-        if (strcmp(buffer, "@@FINISHED\n") == 0)
+        p = buffer;
+        while (isspace(*p))
+            p++;
+
+        if (strcmp(p, "@@FINISHED\n") == 0)
             break;
-        if (strcmp(buffer, "@@FINISHED\r\n") == 0)
+        if (strcmp(p, "@@FINISHED\r\n") == 0)
             break;
     }
 
@@ -103,6 +109,9 @@ int main(int argc, char* const argv[])
     kill(pid, SIGKILL);
     waitpid(pid, &wstatus, 0);
     if (timed_out)
+    {
+        fprintf(stderr, "@@TIMEDOUT\n");
         exit(1);
+    }
     exit(WEXITSTATUS(wstatus));
 }