From cedfb63a8811a62383f4ccc031004c59c0d7ce63 Mon Sep 17 00:00:00 2001 From: Marian Buschsieweke Date: Fri, 8 Nov 2024 09:26:11 +0100 Subject: [PATCH 1/3] tests: no default interactive sync for native The aim is to allow faster test cycles on native for unit test style apps (that don't need interaction) by bypassing the python test framework using e.g.: make RIOT_TERMINAL=native all term This would work already before, but now is more convenient as no manual press of the `s` key is needed to start the test. For non-native boards we need the sync, as otherwise the board may finish booting before the python test automation framework can capture output. For `native` and `native64`, we actually control when the RIOT app is started and do not need to sync. On a typical machine this can reduce the test cycle by more than 4 seconds. With this change: $ time sh -c 'make BOARD=native -C tests/unittests tests-nanocoap -j && make BOARD=native RIOT_TERMINAL=native -C tests/unittests term' [...] main(): This is RIOT! (Version: 2024.10-devel-394-gd65dec-tests/no-sync-control) ................................... OK (35 tests) [...] make: Leaving directory '/home/marian.buschsieweke@ml-pa.loc/Repos/software/RIOT/master/tests/unittests' sh -c 0.30s user 0.24s system 113% cpu 0.476 total Before t his change: $ time sh -c 'make BOARD=native -C tests/unittests tests-nanocoap -j && make BOARD=native -C tests/unittests test' [...] main(): This is RIOT! (Version: 2024.10-devel-394-gd65dec-tests/no-sync-control) Help: Press s to start test, r to print it is ready READY s START ................................... OK (35 tests) [...] make: Leaving directory '/home/marian.buschsieweke@ml-pa.loc/Repos/software/RIOT/master/tests/unittests' sh -c 0.50s user 0.37s system 17% cpu 4.863 total --- tests/Makefile.tests_common | 25 ++++++++++++++----------- tests/README.md | 3 ++- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/tests/Makefile.tests_common b/tests/Makefile.tests_common index cc8dc55b5c..75c42b2664 100644 --- a/tests/Makefile.tests_common +++ b/tests/Makefile.tests_common @@ -1,17 +1,20 @@ APPLICATION ?= tests_$(notdir $(patsubst %/,%,$(CURDIR))) - -ifneq (,$(wildcard $(CURDIR)/tests*/.)) - # add interactive test configuration when testing Kconfig - DEFAULT_MODULE += test_utils_interactive_sync - # add stack metric printing configuration when testing Kconfig - DEFAULT_MODULE += test_utils_print_stack_usage -endif - -# terminate native when the test is complete -CFLAGS += -DNATIVE_AUTO_EXIT=1 - BOARD ?= native RIOTBASE ?= $(CURDIR)/../.. QUIET ?= 1 # DEVELHELP enabled by default for all tests, set 0 to disable DEVELHELP ?= 1 + +ifneq (,$(wildcard $(CURDIR)/tests*/.)) + # add interactive test configuration when automated testing is available, + # except for `native` and `native64` where we control when the RIOT app is + # started + ifeq (,$(filter native native64,$(BOARD))) + DEFAULT_MODULE += test_utils_interactive_sync + endif + # print stack usage by default when automated testing is available + DEFAULT_MODULE += test_utils_print_stack_usage +endif + +# terminate native when the test is complete +CFLAGS += -DNATIVE_AUTO_EXIT=1 diff --git a/tests/README.md b/tests/README.md index 55b8ab645a..97f3c93f62 100644 --- a/tests/README.md +++ b/tests/README.md @@ -127,7 +127,8 @@ As some board can't be reset without a manual trigger tests should be implemente with some kind of `synchronization`. This can be done in two ways: - use `test_utils_interactive_sync` when uart input/output does not need to be - disabled for the test. This is enabled by default. + disabled for the test. This is enabled by default, except for `native` and + `native64`. - set up the test in a loop so the test script will be able so sync with some kind of start condition in the test. From 77b6264da12100cc1e04e88691f7d71d97888b23 Mon Sep 17 00:00:00 2001 From: Marian Buschsieweke Date: Fri, 8 Nov 2024 20:52:14 +0100 Subject: [PATCH 2/3] tests/net/gnrc_sixlowpan_frag_sfr_congure: work around bug in test The test following test is flaky: TEST_ASSERT(msg->send_time > 0U); It assumes that 0 as time is indicating the time is invalid. But on `native`, the time will be zero for the first millisecond the test is launched. Before the interactive sync caused seemingly a reliable delay of at least one milli second, but this no longer is the case. This works around the bug by just waiting at the start of the test until the time is no longer zero, possibly spinning for one millisecond. --- tests/net/gnrc_sixlowpan_frag_sfr_congure/main.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/net/gnrc_sixlowpan_frag_sfr_congure/main.c b/tests/net/gnrc_sixlowpan_frag_sfr_congure/main.c index a3cd545d43..92433293a3 100644 --- a/tests/net/gnrc_sixlowpan_frag_sfr_congure/main.c +++ b/tests/net/gnrc_sixlowpan_frag_sfr_congure/main.c @@ -802,6 +802,10 @@ int main(void) { _tests_init(); + /* work around issue in _check_congure_snd_msg(): the assert for + * time > o will not work when the time is 0 */ + while (xtimer_now_usec() / US_PER_MS == 0) { } + TESTS_START(); TESTS_RUN(tests_gnrc_sixlowpan_frag_sfr_congure_integration()); TESTS_END(); From 562f1f02100545fa647b6dbc46803058d28946e7 Mon Sep 17 00:00:00 2001 From: Marian Buschsieweke Date: Tue, 12 Nov 2024 15:41:49 +0100 Subject: [PATCH 3/3] tests: always use interactive sync for timing tests Use `test_utils_interactive_sync` for the following tests explicitly: - tests/periph/timer_periodic - tests/sys/posix_sleep This is needed for accurate timestamping of the stdio received. The tests will not pass without on `native` / `native64`. --- tests/periph/timer_periodic/Makefile | 5 +++++ tests/sys/posix_sleep/Makefile | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/tests/periph/timer_periodic/Makefile b/tests/periph/timer_periodic/Makefile index c894723314..47756540f1 100644 --- a/tests/periph/timer_periodic/Makefile +++ b/tests/periph/timer_periodic/Makefile @@ -4,4 +4,9 @@ FEATURES_REQUIRED = periph_timer_periodic USEMODULE += fmt +# Interactive sync improves accuracy of timestamping the output +# and is also needed on native for the automatic test to +# pass. Hence, we just depend on it here. +USEMODULE += test_utils_interactive_sync + include $(RIOTBASE)/Makefile.include diff --git a/tests/sys/posix_sleep/Makefile b/tests/sys/posix_sleep/Makefile index 5dcb9ed74c..8967a1acd4 100644 --- a/tests/sys/posix_sleep/Makefile +++ b/tests/sys/posix_sleep/Makefile @@ -2,6 +2,11 @@ include ../Makefile.sys_common USEMODULE += posix_sleep +# Interactive sync improves accuracy of timestamping the output +# and is also needed on native for the automatic test to +# pass. Hence, we just depend on it here. +USEMODULE += test_utils_interactive_sync + # microbit qemu failing currently TEST_ON_CI_BLACKLIST += microbit