From b00b286a1d9faec0ef15c08b87664bc94aa8c239 Mon Sep 17 00:00:00 2001 From: Martine Lenders Date: Wed, 11 Jan 2023 12:54:54 +0100 Subject: [PATCH 1/4] boards: common: stdio_cdc_acm: let tests wait a bit for serial port --- boards/common/makefiles/stdio_cdc_acm.dep.mk | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/boards/common/makefiles/stdio_cdc_acm.dep.mk b/boards/common/makefiles/stdio_cdc_acm.dep.mk index cbd3047533..140bf7114a 100644 --- a/boards/common/makefiles/stdio_cdc_acm.dep.mk +++ b/boards/common/makefiles/stdio_cdc_acm.dep.mk @@ -8,4 +8,9 @@ ifeq (,$(filter-out stdio_cdc_acm,$(filter stdio_% slipdev_stdio,$(USEMODULE)))) USEMODULE += stdio_cdc_acm endif FEATURES_REQUIRED += highlevel_stdio + + # Enforce tests to wait a bit for the serial port after reset + TERM_DELAY ?= 2 + TESTRUNNER_CONNECT_DELAY ?= $(TERM_DELAY) + $(call target-export-variables,test,TESTRUNNER_CONNECT_DELAY) endif From 64bea31b477bf98d6220f578827a0d724b03d32b Mon Sep 17 00:00:00 2001 From: Martine Lenders Date: Wed, 11 Jan 2023 12:55:29 +0100 Subject: [PATCH 2/4] tests/congure_*: wait for serial port after reset if configured --- tests/congure_abe/tests/01-run.py | 4 ++++ tests/congure_quic/tests/01-run.py | 4 ++++ tests/congure_reno/tests/01-run.py | 4 ++++ tests/congure_test/tests/01-run.py | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/tests/congure_abe/tests/01-run.py b/tests/congure_abe/tests/01-run.py index 714ab79bf6..b098aa5c1d 100755 --- a/tests/congure_abe/tests/01-run.py +++ b/tests/congure_abe/tests/01-run.py @@ -8,6 +8,7 @@ import logging import sys +import time import unittest from riotctrl.ctrl import RIOTCtrl @@ -15,6 +16,8 @@ from riotctrl.shell.json import RapidJSONShellInteractionParser, rapidjson from riotctrl_shell.congure_test import CongureTest +from testrunner.spawn import MAKE_TERM_CONNECT_DELAY + class TestCongUREBase(unittest.TestCase): # pylint: disable=too-many-public-methods @@ -25,6 +28,7 @@ class TestCongUREBase(unittest.TestCase): def setUpClass(cls): cls.ctrl = RIOTCtrl() cls.ctrl.reset() + time.sleep(MAKE_TERM_CONNECT_DELAY) cls.ctrl.start_term() if cls.DEBUG: cls.ctrl.term.logfile = sys.stdout diff --git a/tests/congure_quic/tests/01-run.py b/tests/congure_quic/tests/01-run.py index 1b1165b06c..8d0ef8b936 100755 --- a/tests/congure_quic/tests/01-run.py +++ b/tests/congure_quic/tests/01-run.py @@ -8,6 +8,7 @@ import logging import sys +import time import unittest from riotctrl.ctrl import RIOTCtrl @@ -15,6 +16,8 @@ from riotctrl.shell.json import RapidJSONShellInteractionParser, rapidjson from riotctrl_shell.congure_test import CongureTest +from testrunner.spawn import MAKE_TERM_CONNECT_DELAY + class TestCongUREBase(unittest.TestCase): DEBUG = False @@ -25,6 +28,7 @@ class TestCongUREBase(unittest.TestCase): def setUpClass(cls): cls.ctrl = RIOTCtrl() cls.ctrl.reset() + time.sleep(MAKE_TERM_CONNECT_DELAY) cls.ctrl.start_term() if cls.DEBUG: cls.ctrl.term.logfile = sys.stdout diff --git a/tests/congure_reno/tests/01-run.py b/tests/congure_reno/tests/01-run.py index 0b08dd0880..993ff645fc 100755 --- a/tests/congure_reno/tests/01-run.py +++ b/tests/congure_reno/tests/01-run.py @@ -8,6 +8,7 @@ import logging import sys +import time import unittest from riotctrl.ctrl import RIOTCtrl @@ -15,6 +16,8 @@ from riotctrl.shell.json import RapidJSONShellInteractionParser, rapidjson from riotctrl_shell.congure_test import CongureTest +from testrunner.spawn import MAKE_TERM_CONNECT_DELAY + class TestCongUREBase(unittest.TestCase): # pylint: disable=too-many-public-methods @@ -25,6 +28,7 @@ class TestCongUREBase(unittest.TestCase): def setUpClass(cls): cls.ctrl = RIOTCtrl() cls.ctrl.reset() + time.sleep(MAKE_TERM_CONNECT_DELAY) cls.ctrl.start_term() if cls.DEBUG: cls.ctrl.term.logfile = sys.stdout diff --git a/tests/congure_test/tests/01-run.py b/tests/congure_test/tests/01-run.py index 677606cf2d..4db47e2d4e 100755 --- a/tests/congure_test/tests/01-run.py +++ b/tests/congure_test/tests/01-run.py @@ -9,6 +9,7 @@ import logging import os import sys +import time import unittest from riotctrl.ctrl import RIOTCtrl @@ -17,6 +18,8 @@ from riotctrl.shell.json import RapidJSONShellInteractionParser, rapidjson from riotctrl_shell.congure_test import CongureTest +from testrunner.spawn import MAKE_TERM_CONNECT_DELAY + class TestCongUREBase(unittest.TestCase): DEBUG = False @@ -25,6 +28,7 @@ class TestCongUREBase(unittest.TestCase): def setUpClass(cls): cls.ctrl = RIOTCtrl() cls.ctrl.reset() + time.sleep(MAKE_TERM_CONNECT_DELAY) cls.ctrl.start_term() if cls.DEBUG: cls.ctrl.term.logfile = sys.stdout From 764f7f0bf4cff93676f830e5bad441e534a20b1e Mon Sep 17 00:00:00 2001 From: Martine Lenders Date: Fri, 6 Jan 2023 14:24:17 +0100 Subject: [PATCH 3/4] gh-actions: report release-tests result to Matrix --- .github/workflows/release-test.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index ce3152f5e6..48651d16c7 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -123,6 +123,7 @@ jobs: sudo apt-get install lib32asan6 - name: Run release tests timeout-minutes: 350 + id: tests run: | RIOTBASE="$GITHUB_WORKSPACE/RIOT" TOX_ARGS="" @@ -173,6 +174,27 @@ jobs: mkdir test-reports/ junit2html ${REPORT_XML} ${REPORT_NAME}.html cp ${REPORT_XML} ${REPORT_NAME}.xml + - name: Generate result message + if: always() + id: generate_results + run: | + if [ "${{ steps.tests.conclusion }}" == "success" ]; then + nice_str=echo "✅ **PASSED**" + elif [ "${{ steps.tests.conclusion }}" == "failure" ]; then + nice_str="❌ **FAILED**" + fi + echo "nice_str=${nice_str}" >> ${GITHUB_OUTPUT} + - name: Report to Matrix channel + if: ${{ always() && steps.generate_results.outputs.nice_str != '' }} + uses: fadenb/matrix-chat-message@v0.0.6 + with: + homeserver: matrix.org + token: ${{ secrets.RIOT_CI_MATRIX_TOKEN }} + channel: '!UXNJuMzHTSAnwntbbN:utwente.io' + message: | + ${{ steps.generate_results.outputs.nice_str}}: Release tests + [${{ matrix.pytest_mark }}, ${{ matrix.sudo }}] on "*${{ github.event_name }}*": + ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - uses: actions/upload-artifact@v2 if: always() with: From 8c055f0f3e9019f32a57bc8724d4a16f66afdca3 Mon Sep 17 00:00:00 2001 From: Marian Buschsieweke Date: Thu, 12 Jan 2023 12:03:28 +0100 Subject: [PATCH 4/4] makefiles/utils/strings.mk: Fix version_is_greater_or_equal The Makefile function `version_is_greater_or_equal` is used to check if a version of GNU Make is at least the required one. However, it has the built-in assumption the version numbers have to format x.y.z, but Alpine Linux currently ships GNU Make 4.4. This results in `$(call _pad_number,3,)` which runs `printf '$03d' ''` in the shell, which is not valid. This fixes the issue by making `_pad_number` more robust by fall back to printing `0` with the given padding, if the number given to print is empty. --- makefiles/utils/strings.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/makefiles/utils/strings.mk b/makefiles/utils/strings.mk index 66467c2988..cbc280fcf0 100644 --- a/makefiles/utils/strings.mk +++ b/makefiles/utils/strings.mk @@ -7,8 +7,8 @@ lowercase = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst uppercase = $(subst a,A,$(subst b,B,$(subst c,C,$(subst d,D,$(subst e,E,$(subst f,F,$(subst g,G,$(subst h,H,$(subst i,I,$(subst j,J,$(subst k,K,$(subst l,L,$(subst m,M,$(subst n,N,$(subst o,O,$(subst p,P,$(subst q,Q,$(subst r,R,$(subst s,S,$(subst t,T,$(subst u,U,$(subst v,V,$(subst w,W,$(subst x,X,$(subst y,Y,$(subst z,Z,$1)))))))))))))))))))))))))) uppercase_and_underscore = $(call uppercase,$(subst -,_,$1)) -# Padds $2 number to $1 digits -_pad_number = $(shell printf '%0$1d' $2) +# Padds number $2 to $1 digits. If $2 is empty, zero will be printed instead. +_pad_number = $(shell printf '%0$1d' $(if $2,$2,0)) # Gets major, minor, patch from 'major.minor.patch', e.g.: 4.2.1 by index # $1: index