From 58a0fd8eb613b846b40cbfcb741068e1bf67a34f Mon Sep 17 00:00:00 2001
From: Kaspar Schleiser <kaspar@schleiser.de>
Date: Fri, 24 Mar 2017 14:40:38 +0100
Subject: [PATCH] murdock: add support for running tests on hardware

---
 .murdock                 | 68 ++++++++++++++++++++++++++++++++++++++--
 makefiles/murdock.inc.mk | 36 +++++++++++++++++++--
 2 files changed, 98 insertions(+), 6 deletions(-)

diff --git a/.murdock b/.murdock
index 66634df569..e0f44506e8 100755
--- a/.murdock
+++ b/.murdock
@@ -1,15 +1,42 @@
 #!/bin/sh
 
+export TEST_BOARDS_AVAILABLE=${TEST_BOARDS_AVAILABLE:-"samr21-xpro"}
 export RIOT_CI_BUILD=1
 export STATIC_TESTS=${STATIC_TESTS:-1}
 export CFLAGS_DBG=""
 export DLCACHE_DIR=${DLCACHE_DIR:-~/.dlcache}
 
+NIGHTLY=${NIGHTLY:-0}
+RUN_TESTS=${RUN_TESTS:-${NIGHTLY}}
+
+check_label() {
+    local label="${1}"
+    [ -z "${CI_PULL_LABELS}" ] && return 1
+    echo "${CI_PULL_LABELS}" | grep -q "${label}"
+    return $?
+}
+
+[ "$RUN_TESTS" != "1" ] && {
+    check_label "CI: run tests" && RUN_TESTS=1
+}
+
 error() {
     echo "$@"
     exit 1
 }
 
+# true if word "$1" is in list of words "$2", false otherwise
+# uses grep -w, thus only alphanum and "_" count as word bounderies
+# (word "def" matches "abc-def")
+is_in_list() {
+    [ $# -ne 2 ] && return 1
+
+    local needle="$1"
+    local haystack="$2"
+
+    echo "$haystack" | grep -q -w "$needle"
+}
+
 _greplist() {
     if [ $# -eq 0 ]; then
         echo cat
@@ -91,14 +118,23 @@ compile() {
     [ ! -d "$appdir" ] && error "$0: compile: error: application directory \"$appdir\" doesn't exist"
     [ ! -d "boards/$board" ] && error "$0: compile: error: board directory \"boards/$board\" doesn't exist"
 
+    # compile
     CCACHE_BASEDIR="$(pwd)" BOARD=$board RIOT_CI_BUILD=1 \
         make -C${appdir} clean all -j${JOBS:-4}
     RES=$?
 
+    # run tests
     if [ $RES -eq 0 ]; then
-        if [ "$board" = "native" -a "$appdir" = "tests/unittests" ]; then
-            make -C${appdir} test
-            RES=$?
+        if [ $RUN_TESTS -eq 1 -o "$board" = "native" ]; then
+            if [ -f "${BINDIR}/.test" ]; then
+                if [ "$board" = "native" ]; then
+                    BOARD=$board make -C${appdir} test
+                    RES=$?
+                elif is_in_list "$board" "$TEST_BOARDS_AVAILABLE"; then
+                    BOARD=$board make -C${appdir} test-murdock
+                    RES=$?
+                fi
+            fi
         fi
     fi
 
@@ -113,6 +149,32 @@ compile() {
     return $RES
 }
 
+test_job() {
+    local appdir=$1
+    local board=$2
+    local flashfile="$3"
+
+    [ ! -f "$flashfile" ] && {
+        echo "$0: _test(): flashfile \"$flashfile\" doesn't exist!"
+        return 1
+    }
+
+    dwqc \
+        ${DWQ_JOBID:+--subjob} \
+        --file $flashfile:$appdir/bin/${board}/$(basename $flashfile) \
+        --queue ${TEST_QUEUE:-$board} \
+        --maxfail 1 \
+        "./.murdock run_test $appdir $board"
+}
+
+run_test() {
+    local appdir=$1
+    local board=$2
+    print_worker
+    echo "-- executing tests for $appdir on $board:"
+    BOARD=$board make -C$appdir flash-only test
+}
+
 # execute static tests
 static_tests() {
     local repo=${CI_BASE_REPO:-https://github.com/RIOT-OS/RIOT}
diff --git a/makefiles/murdock.inc.mk b/makefiles/murdock.inc.mk
index 029446e506..50fb25774f 100644
--- a/makefiles/murdock.inc.mk
+++ b/makefiles/murdock.inc.mk
@@ -1,6 +1,36 @@
-test-murdock: link
+#
+# This file contains helper targets used by the CI.
+#
+
+# (HACK) get actual flash binary from FFLAGS.
+FLASHFILE:=$(filter $(HEXFILE) $(ELFFILE:.elf=.bin) $(ELFFILE),$(FFLAGS))
+
+#
+# This target will run "make test" on the CI cluster.
+#
+# In order to work, these requirements must be fulfilled:
+# - DWQ_REPO and DWQ_COMMIT are set correctly
+# - the user has set up autossh & proper authentication for connecting to the CI
+# (intended to be used by CI only for now)
+test-murdock:
 	cd $(RIOTBASE) && \
-		./.murdock make_test_job \
+		./.murdock test_job \
 		$$(realpath --relative-to $(RIOTBASE) $(APPDIR)) \
 		$(BOARD) \
-		$(HEXFILE)
+		$(FLASHFILE)
+
+# don't whitelist tests if there's no binary
+ifeq (1,$(RIOTNOLINK))
+  TEST_ON_CI_WHITELIST:=
+endif
+
+# create $(BINDIR)/.test file only if BOARD is in $(TEST_ON_CI_WHITELIST)
+.PHONY: $(BINDIR)/.test
+link: $(BINDIR)/.test
+$(BINDIR)/.test: $(filter clean, $(MAKECMDGOALS))
+ifneq (,$(filter $(BOARD) all, $(TEST_ON_CI_WHITELIST)))
+	$(Q)mkdir -p $(BINDIR)
+	$(Q)touch $@
+else
+	$(Q)rm -f $@
+endif