tests/bench_ztimer: add port of bench_xtimer

tests/bench_ztimer/Makefile

@@ -0,0 +1,101 @@
+include ../Makefile.tests_common
+
+USEMODULE += ztimer_usec ztimer_msec
+
+# this test uses 1000 timers by default. for boards that boards don't have
+# enough memory, reduce that to 100 or 20, unless NUMOF_TIMERS has been overridden.
+LOW_MEMORY_BOARDS += \
+  airfy-beacon \
+  arduino-mega2560 \
+  arduino-mkr1000 \
+  arduino-mkrfox1200 \
+  arduino-mkrwan1300 \
+  arduino-mkrzero \
+  arduino-nano-33-iot \
+  atmega1284p \
+  b-l072z-lrwan1 \
+  bastwan \
+  blackpill \
+  blackpill-128kib \
+  bluepill \
+  bluepill-128kib \
+  calliope-mini \
+  cc1312-launchpad \
+  cc1350-launchpad \
+  cc1352-launchpad \
+  cc2650-launchpad \
+  cc2650stk \
+  e104-bt5010a-tb \
+  e104-bt5011a-tb \
+  derfmega128 \
+  feather-m0 \
+  feather-m0-lora \
+  feather-m0-wifi \
+  hifive1 \
+  hifive1b \
+  i-nucleo-lrwan1 \
+  lsn50 \
+  maple-mini \
+  mega-xplained \
+  microbit \
+  microduino-corerf \
+  msb-430 \
+  msb-430h \
+  nrf51dongle \
+  nrf6310 \
+  nucleo-f030r8 \
+  nucleo-f042k6 \
+  nucleo-f070rb \
+  nucleo-f072rb \
+  nucleo-f103rb \
+  nucleo-f303k8 \
+  nucleo-f334r8 \
+  nucleo-l031k6 \
+  nucleo-l053r8 \
+  nucleo-l073rz \
+  opencm904 \
+  saml10-xpro \
+  saml11-xpro \
+  seeeduino_xiao \
+  sensebox_samd21 \
+  serpente \
+  sodaq-autonomo \
+  sodaq-explorer \
+  sodaq-one \
+  sodaq-sara-aff \
+  sodaq-sara-sff \
+  spark-core \
+  stm32f0discovery \
+  stm32l0538-disco \
+  telosb \
+  waspmote-pro \
+  wemos-zero \
+  yarm \
+  yunjia-nrf51822 \
+  z1 \
+  #
+
+SUPER_LOW_MEMORY_BOARDS += \
+  arduino-duemilanove \
+  arduino-leonardo \
+  arduino-nano \
+  arduino-uno \
+  atmega328p \
+  atmega328p-xplained-mini \
+  nucleo-f031k6 \
+  stm32f030f4-demo \
+  #
+
+ifneq (, $(filter $(BOARD), $(LOW_MEMORY_BOARDS)))
+  NUMOF_TIMERS ?= 100
+endif
+
+ifneq (, $(filter $(BOARD), $(SUPER_LOW_MEMORY_BOARDS)))
+  NUMOF_TIMERS ?= 12
+endif
+
+NUMOF_TIMERS ?= 1000
+
+CFLAGS += -DNUMOF_TIMERS=$(NUMOF_TIMERS)
+
+include $(RIOTBASE)/Makefile.include

tests/bench_ztimer/Makefile.ci

@@ -0,0 +1,17 @@
+BOARD_INSUFFICIENT_MEMORY := \
+    arduino-duemilanove \
+    arduino-nano \
+    arduino-uno \
+    atmega328p \
+    atmega328p-xplained-mini \
+    atxmega-a3bu-xplained \
+    bluepill-stm32f030c8 \
+    im880b \
+    nucleo-l011k4 \
+    olimexino-stm32 \
+    samd10-xmini \
+    slstk3400a \
+    stk3200 \
+    stm32g0316-disco \
+    zigduino \
+    #

tests/bench_ztimer/README.md

@@ -0,0 +1,77 @@
+# Introduction
+
+This test executes some benchmarks for ztimer's set() / remove() / now()
+operations.
+
+# Details
+
+This set of benchmarks measures ztimer's list operation efficiency.
+Depending on the available memory, the individual benchmarks that are using
+multiple timers are run with either 1000 (the default), 100 or 20 timers.
+Each benchmark is repeated REPEAT times (default 1000).
+As only the operations are benchmarked, it is asserted that no timer ever
+actually triggers.
+
+### set() one
+
+This repeatedly sets one timer in an otherwise empty list.
+All but the first iteration will cause ztimer to implicitly remove the timer
+first.
+All iterations will cause the underlying periph timer to be updated.
+
+### remove() one
+
+This repeatedly removes one timer from the list. In all but the first iteration,
+this list will be empty.
+
+
+### set() + remove() one
+
+This repeatedly first sets, then removes one timer. The list is empty
+before and after each iteration.
+All iterations will cause the underlying periph timer to be updated.
+
+### set() many increasing targets
+
+This adds NUMOF timers with increasing target times. Each iteration will add a
+timer at the end of ztimer's timer list.
+Only the first iteration will cause the underlying periph timer to be updated.
+After this test, the list is populated with NUMOF timers.
+
+### re-set() first
+
+This repatedly re-sets the first timer in the list (to the same target time).
+All iterations will cause the underlying periph timer to be updated.
+
+### re-set() middle
+
+This repatedly re-sets the timer in the middle of the list (to the same target
+time).
+
+### re-set() last
+
+This repatedly re-sets the last timer in the list (to the same target time).
+
+### remove() + set() first, middle, last
+
+Same as the previous three, but does a remove() before set().
+
+### remove() many decreasing
+
+This removes all timers from the list, starting with the last.
+
+### ztimer_now()
+
+This simply calls ztimer_now() in a loop.
+
+
+# How to interpret results
+
+The aim is to measure the time spent in ztimer's list operations.
+Lower values are better.
+The first/middle/last tests give an idea of the best case / average case /
+worst case when running the operation with NUMOF timers.
+Note that every set() on an already set timer will trigger an implicit remove(),
+thus the timer list has to be iterated twice.
+The tests that do a remove() before set() show whether ztimer correctly
+identifies an unset timer.

tests/bench_ztimer/main.c

@@ -0,0 +1,296 @@
+/*
+ * Copyright (C) 2019 Kaspar Schleiser <kaspar@schleiser.de>
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @ingroup     tests
+ * @{
+ *
+ * @file
+ * @brief       ztimer set / remove / now benchmark application
+ *
+ * @author      Kaspar Schleiser <kaspar@schleiser.de>
+ *
+ * @}
+ */
+
+#include <stdio.h>
+
+#include "test_utils/expect.h"
+
+#include "msg.h"
+#include "thread.h"
+#include "ztimer.h"
+
+#ifndef ZTIMER
+#define ZTIMER ZTIMER_MSEC
+#endif
+
+#ifndef NUMOF_TIMERS
+#define NUMOF_TIMERS   (1000U)
+#endif
+
+#ifndef REPEAT
+#define REPEAT   (1000U)
+#endif
+
+#ifndef BASE
+#define BASE    (10000000LU)
+#endif
+
+#ifndef SPREAD
+#define SPREAD  (10LU)
+#endif
+
+static ztimer_t _timers[NUMOF_TIMERS];
+
+/* This variable is set by any timer that actually triggers.  As the test is
+ * only testing set/remove/now operations, timers are not supposed to trigger.
+ * Thus, after every test there's an 'expect(!_triggers)'
+ */
+static unsigned _triggers;
+
+/*
+ * The test assumes that first, middle and last will always end up in at the
+ * same index within the timer queue.  In order to compensate for the time that
+ * previous operations take themselves, the interval is corrected. The
+ * variables "start" and "_base" are used for that.
+ */
+uint32_t _base;
+
+static void _callback(void *arg) {
+    unsigned *triggers = arg;
+    *triggers += 1;
+}
+
+/* returns the interval for timer 'n' that has to be set in order to insert it
+ * into position n */
+static uint32_t _timer_val(unsigned n)
+{
+    return _base + (SPREAD * n);
+}
+
+/* set timer 'n' to its intended position 'n' */
+static void _timer_set(unsigned n)
+{
+    ztimer_set(ZTIMER, &_timers[n], _timer_val(n));
+}
+
+/* remove timer 'n' */
+static void _timer_remove(unsigned n)
+{
+    ztimer_remove(ZTIMER, &_timers[n]);
+}
+
+static void _print_result(const char *desc, unsigned n, uint32_t total)
+{
+    printf("%30s %8"PRIu32" / %u = %"PRIu32"\n", desc, total, n, total/n);
+}
+
+int main(void)
+{
+    puts("ztimer benchmark application.\n");
+
+    unsigned n;
+    uint32_t before, diff, start;
+
+    /* initializing timer structs */
+    for (unsigned int n = 0; n < NUMOF_TIMERS; n++) {
+        _timers[n].callback = _callback;
+        _timers[n].arg = &_triggers;
+    }
+
+    start = ztimer_now(ZTIMER_USEC);
+
+    /*
+     * test setting one set timer REPEAT times
+     *
+     */
+    _base = BASE;
+    before = ztimer_now(ZTIMER_USEC);
+    for (n = 0; n < REPEAT; n++) {
+        _timer_set(0);
+    }
+
+    diff = ztimer_now(ZTIMER_USEC) - before;
+
+    _print_result("set() one", REPEAT, diff);
+    expect(!_triggers);
+
+    /*
+     * test removing one unset timer REPEAT times
+     *
+     */
+    before = ztimer_now(ZTIMER_USEC);
+    for (n = 0; n < REPEAT; n++) {
+        _timer_remove(0);
+    }
+
+    diff = ztimer_now(ZTIMER_USEC) - before;
+
+    _print_result("remove() one", REPEAT, diff);
+    expect(!_triggers);
+
+    /*
+     * test setting / removing one timer REPEAT times
+     *
+     */
+    before = ztimer_now(ZTIMER_USEC);
+    _base = BASE  - (before - start);
+    for (n = 0; n < REPEAT; n++) {
+        _timer_set(0);
+        _timer_remove(0);
+    }
+
+    diff = ztimer_now(ZTIMER_USEC) - before;
+
+    _print_result("set() + remove() one", REPEAT, diff);
+    expect(!_triggers);
+
+    /*
+     * test setting NUMOF_TIMERS timers with increasing targets
+     *
+     */
+    before = ztimer_now(ZTIMER_USEC);
+    _base = BASE  - (before - start);
+    for (unsigned int n = 0; n < NUMOF_TIMERS; n++) {
+        _timer_set(n);
+    }
+
+    diff = ztimer_now(ZTIMER_USEC) - before;
+
+    _print_result("set() many increasing target", NUMOF_TIMERS, diff);
+    expect(!_triggers);
+
+    /*
+     * test re-setting first timer REPEAT times
+     *
+     */
+    before = ztimer_now(ZTIMER_USEC);
+    _base = BASE  - (before - start);
+    for (n = 0; n < REPEAT; n++) {
+        _timer_set(0);
+    }
+
+    diff = ztimer_now(ZTIMER_USEC) - before;
+
+    _print_result("re-set()  first", REPEAT, diff);
+    expect(!_triggers);
+
+    /*
+     * test setting middle timer REPEAT times
+     *
+     */
+    before = ztimer_now(ZTIMER_USEC);
+    _base = BASE  - (before - start);
+    for (n = 0; n < REPEAT; n++) {
+        _timer_set(NUMOF_TIMERS/2);
+    }
+
+    diff = ztimer_now(ZTIMER_USEC) - before;
+
+    _print_result("re-set() middle", REPEAT, diff);
+    expect(!_triggers);
+
+    /*
+     * test setting last timer REPEAT times
+     *
+     */
+    before = ztimer_now(ZTIMER_USEC);
+    _base = BASE  - (before - start);
+    for (n = 0; n < REPEAT; n++) {
+        _timer_set(NUMOF_TIMERS - 1);
+    }
+
+    diff = ztimer_now(ZTIMER_USEC) - before;
+
+    _print_result("re-set()   last", REPEAT, diff);
+    expect(!_triggers);
+
+    /*
+     * test removing / setting first timer REPEAT times
+     *
+     */
+    before = ztimer_now(ZTIMER_USEC);
+    _base = BASE  - (before - start);
+    for (n = 0; n < REPEAT; n++) {
+        _timer_remove(0);
+        _timer_set(0);
+    }
+
+    diff = ztimer_now(ZTIMER_USEC) - before;
+
+    _print_result("remove() + set()  first", REPEAT, diff);
+    expect(!_triggers);
+
+    /*
+     * test removing / setting middle timer REPEAT times
+     *
+     */
+    before = ztimer_now(ZTIMER_USEC);
+    _base = BASE  - (before - start);
+    for (n = 0; n < REPEAT; n++) {
+        _timer_remove(NUMOF_TIMERS/2);
+        _timer_set(NUMOF_TIMERS/2);
+    }
+
+    diff = ztimer_now(ZTIMER_USEC) - before;
+
+    _print_result("remove() + set() middle", REPEAT, diff);
+    expect(!_triggers);
+
+    /*
+     * test removing / setting last timer REPEAT times
+     *
+     */
+    before = ztimer_now(ZTIMER_USEC);
+    _base = BASE  - (before - start);
+    for (n = 0; n < REPEAT; n++) {
+        _timer_remove(NUMOF_TIMERS - 1);
+        _timer_set(NUMOF_TIMERS - 1);
+    }
+
+    diff = ztimer_now(ZTIMER_USEC) - before;
+
+    _print_result("remove() + set()   last", REPEAT, diff);
+    expect(!_triggers);
+
+    /*
+     * test removing NUMOF_TIMERS timers (latest first)
+     *
+     */
+    before = ztimer_now(ZTIMER_USEC);
+    for (n = 0; n < NUMOF_TIMERS; n++) {
+        _timer_remove(NUMOF_TIMERS - n - 1);
+    }
+
+    diff = ztimer_now(ZTIMER_USEC) - before;
+
+    _print_result("remove() many decreasing", NUMOF_TIMERS, diff);
+    expect(!_triggers);
+
+    /*
+     * test ztimer_now()
+     *
+     */
+    before = ztimer_now(ZTIMER_USEC);
+    n = REPEAT;
+    while (n--) {
+        ztimer_now(ZTIMER);
+    }
+
+    diff = ztimer_now(ZTIMER_USEC) - before;
+
+    _print_result("ztimer_now()", REPEAT, diff);
+    expect(!_triggers);
+
+    _print_result("sizeof(ztimer_t)", NUMOF_TIMERS, sizeof(_timers));
+
+    puts("done.");
+
+    return 0;
+}

tests/bench_ztimer/tests/01-run.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2019 Freie Universität Berlin
+#
+# This file is subject to the terms and conditions of the GNU Lesser
+# General Public License v2.1. See the file LICENSE in the top level
+# directory for more details.
+
+import sys
+from testrunner import run
+
+
+def testfunc(child):
+    child.expect_exact("ztimer benchmark application.\r\n")
+    for i in range(13):
+        child.expect(r"\s+[\w() _\+]+\s+\d+ / \d+ = \d+\r\n")
+
+    child.expect_exact("done.\r\n")
+
+
+if __name__ == "__main__":
+    sys.exit(run(testfunc))