From 5d2c57f858b04fe9bf9c19f9139a2e2603ad984f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Harter?= Date: Tue, 25 Jun 2019 18:41:52 +0200 Subject: [PATCH 1/3] murdock: refactor handling of TEST_ON_CI_WHITELIST Refactor the handling to use a variable to store if a test is enabled. Add a 'test-on-ci-enabled' target that test if the test on ci is enabled. This is a first commit before changing the behavior. --- makefiles/murdock.inc.mk | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/makefiles/murdock.inc.mk b/makefiles/murdock.inc.mk index 945802f701..87eaadacd3 100644 --- a/makefiles/murdock.inc.mk +++ b/makefiles/murdock.inc.mk @@ -19,16 +19,31 @@ test-murdock: "$(BOARD):$(TOOLCHAIN)" \ $(FLASHFILE) -# don't whitelist tests if there's no binary -ifeq (1,$(RIOTNOLINK)) - TEST_ON_CI_WHITELIST:= -endif -# create $(BINDIR)/.test file only if BOARD is in $(TEST_ON_CI_WHITELIST) +# Control running tests on boards during the CI tests. +# +# Testing is enabled if all the following conditions are met: +# +# * the board is whitelisted +# * the board has enough memory and the executable is being linked +# +# TEST_ON_CI_WHITELIST can be empty, a board list or 'all' + +TEST_ON_CI_WHITELIST ?= +TEST_ON_BOARD_ENABLED ?= $(filter $(TEST_ON_CI_WHITELIST:all=%),$(BOARD)) + +TEST_ON_CI_ENABLED ?= $(if $(RIOTNOLINK),,$(TEST_ON_BOARD_ENABLED)) + +.PHONY: test-on-ci-enabled +test-on-ci-enabled: + $(Q)test -n "$(TEST_ON_CI_ENABLED)" + + +# create $(BINDIR)/.test file only if the test is enabled for this board .PHONY: $(BINDIR)/.test link: $(BINDIR)/.test $(BINDIR)/.test: $(filter clean, $(MAKECMDGOALS)) -ifneq (,$(filter $(BOARD) all, $(TEST_ON_CI_WHITELIST))) +ifneq (,$(TEST_ON_CI_ENABLED)) $(Q)mkdir -p $(BINDIR) $(Q)touch $@ else From 45a318269d7c8ee251958bd1b1df2563771eb8ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Harter?= Date: Tue, 25 Jun 2019 18:45:59 +0200 Subject: [PATCH 2/3] murdock: introduce 'TEST_ON_CI_BLACKLIST' Introduce a variable to set that a test is blacklisted. This is a move toward enabling tests by default and adding a blacklisting reason instead for a board instead of not whitelisting them which hides the problem. Currently, a test should be both whitelisted and blacklisted at the same time to have a meaning. It is planned to whitelist all by default in an upcoming pull request. --- makefiles/murdock.inc.mk | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/makefiles/murdock.inc.mk b/makefiles/murdock.inc.mk index 87eaadacd3..edf8fb530e 100644 --- a/makefiles/murdock.inc.mk +++ b/makefiles/murdock.inc.mk @@ -25,12 +25,19 @@ test-murdock: # Testing is enabled if all the following conditions are met: # # * the board is whitelisted +# * the board is not blacklisted (by default none) # * the board has enough memory and the executable is being linked # -# TEST_ON_CI_WHITELIST can be empty, a board list or 'all' +# TEST_ON_CI_WHITELIST and TEST_ON_CI_BLACKLIST can be empty, a board list or 'all' +# +# Prefer blacklisting boards that fail to whitelisting the ones that work. +# It will help tracking what is failing. +# +# Disabling a test in some case must be justified to keep track of the reason. TEST_ON_CI_WHITELIST ?= -TEST_ON_BOARD_ENABLED ?= $(filter $(TEST_ON_CI_WHITELIST:all=%),$(BOARD)) +TEST_ON_CI_BLACKLIST ?= +TEST_ON_BOARD_ENABLED ?= $(filter-out $(TEST_ON_CI_BLACKLIST:all=%),$(filter $(TEST_ON_CI_WHITELIST:all=%),$(BOARD))) TEST_ON_CI_ENABLED ?= $(if $(RIOTNOLINK),,$(TEST_ON_BOARD_ENABLED)) From bdd721fff476ae8ffe11b6240e7d0e0a6070999f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Harter?= Date: Wed, 26 Jun 2019 17:53:58 +0200 Subject: [PATCH 3/3] tests/xtimer: blacklist native instead of whitelisting Blacklist native for the xtimer tests that have timing issues. This also enables running the test on `nrf52dk` as I think it was forgotten when adding the support. --- tests/xtimer_now64_continuity/Makefile | 3 ++- tests/xtimer_usleep/Makefile | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/xtimer_now64_continuity/Makefile b/tests/xtimer_now64_continuity/Makefile index 7771d93310..c3c678dd66 100644 --- a/tests/xtimer_now64_continuity/Makefile +++ b/tests/xtimer_now64_continuity/Makefile @@ -3,7 +3,8 @@ include ../Makefile.tests_common USEMODULE += fmt USEMODULE += xtimer +TEST_ON_CI_WHITELIST += all # This test randomly fails on `native` so disable it from CI -TEST_ON_CI_WHITELIST += samr21-xpro +TEST_ON_CI_BLACKLIST += native include $(RIOTBASE)/Makefile.include diff --git a/tests/xtimer_usleep/Makefile b/tests/xtimer_usleep/Makefile index 1af1bdee92..48cf81df4e 100644 --- a/tests/xtimer_usleep/Makefile +++ b/tests/xtimer_usleep/Makefile @@ -2,8 +2,9 @@ include ../Makefile.tests_common USEMODULE += xtimer +TEST_ON_CI_WHITELIST += all # This test randomly fails on `native` so disable it from CI -TEST_ON_CI_WHITELIST += samr21-xpro +TEST_ON_CI_BLACKLIST += native # This application uses getchar and thus expects input from stdio USEMODULE += stdin