Merge pull request #16126 from kfessel/p-mix-schedrr

sys/sched_rr: Add a round robin scheduler module

.gitignore

@@ -9,8 +9,9 @@ doc/doxygen/man
 doc/doxygen/*.log
 doc/doxygen/*.db
 doc/doxygen/*.tmp
-# Built binaries
+# bin (e.g.:build directory) and .bin files
-*bin
+bin
+*.bin
 # Build directory
 /build
 # AFL findings

examples/thread_duel/Makefile

@@ -0,0 +1,27 @@
+# name of your application
+APPLICATION = thread_duel
+
+# If no BOARD is found in the environment, use this default:
+BOARD ?= native
+
+# This has to be the absolute path to the RIOT base directory:
+RIOTBASE ?= $(CURDIR)/../..
+
+# This defaults to build with round_robin using ztimer
+RR ?= 1
+
+USEMODULE += ztimer_usec
+
+ifeq (1,$(RR))
+  USEMODULE += sched_round_robin
+endif
+
+# Comment this out to disable code in RIOT that does safety checking
+# which is not needed in a production environment but helps in the
+# development process:
+DEVELHELP ?= 1
+
+# Change this to 0 show compiler invocation lines by default:
+QUIET ?= 1
+
+include $(RIOTBASE)/Makefile.include

examples/thread_duel/Makefile.ci

@@ -0,0 +1,12 @@
+BOARD_INSUFFICIENT_MEMORY := \
+    arduino-duemilanove \
+    arduino-leonardo \
+    arduino-nano \
+    arduino-uno \
+    atmega328p \
+    atmega328p-xplained-mini \
+    nucleo-f031k6 \
+    nucleo-l011k4 \
+    samd10-xmini \
+    stm32f030f4-demo \
+    #

examples/thread_duel/README.md

@@ -0,0 +1,45 @@
+Thread-Duel
+============
+
+This is a thread duel application to show RIOTs abilities to run multiple-threads
+concurrently, even if they are neither cooperative nor dividable into different scheduler priorities,
+by using the optional round-robin scheduler module.
+
+Every thread will do some work (busy waiting).
+After the work is done it counts up by the amount of work it did and then it rests.
+There are different resting strategies and these have a huge
+influence on thread fairness and scheduling.
+
+Resting strategies for the threads of this example are:
+- `nice_wait`: does nice breaks giving other threads time to use the CPU
+- `bad_wait`: takes breaks by busy waiting and therefore hogging the CPU
+- `yield_wait`: takes no explicit breaks but yields (to higher or equal priority threads)
+- `no_wait`: never takes a break
+
+After completing a batch of work (and rest) a thread will print information on the work done.
+(Printing is done in steps to avoid flooding)
+
+If one thread (all are same priority) follows `bad_wait` or `no_wait` strategy,
+scheduling without round robin will see all CPU time be hogged by that one thread
+(or the first one to get it).
+
+In this example Round Robin scheduling is enabled by default,
+to disable compile with `RR=0`
+
+Change the behaviour of the different threads by adding
+`CFLAGS='-DTHREAD_1={<rest_strategy>,<work>}'`
+
+e.g.:
+```
+CFLAGS='-DTHREAD_1={yield_wait,3} -DTHREAD_2={bad_wait,2}' RR=0 make
+```
+Will set:
+- thread 1 to follow `yield_waiting` strategy and
+to complete 3 works in one batch after that, it will yield.
+- thread 2 will do 2 work and follow `bad_wait` (hog the cpu while waiting).
+the Round Robin scheduling is not activated.
+
+The result will be:
+- CPU hogged by thread 2, which will only do work for 20% of the time (really bad)
+- thread 1 will have done 3 work but will never even get the chance to print that
+- thread 3 will never have done any work.

examples/thread_duel/main.c

@@ -0,0 +1,153 @@
+/*
+ * Copyright (C) 2020 TUBA Freiberg
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+
+#include "thread.h"
+#include "sched.h"
+
+#include "ztimer.h"
+#include "timex.h"
+
+#include "sched_round_robin.h"
+
+#define PRINT_STEPS 10
+#define WORK_SCALE  1000
+#define STEPS_PER_SET 10
+
+__attribute__((unused))
+static void bad_wait(uint32_t us)
+{
+    /* keep the CPU busy waiting for some time to pass simulate working */
+    ztimer_spin(ZTIMER_USEC, us);
+}
+
+static void (* const do_work)(uint32_t us) = bad_wait;
+
+__attribute__((unused))
+static void nice_wait(uint32_t us)
+{
+    /* be nice give the CPU some time to do other things or rest */
+    ztimer_sleep(ZTIMER_USEC, us);
+}
+
+__attribute__((unused))
+static void yield_wait(uint32_t unused)
+{
+    (void) unused;
+    /* do not wait just yield */
+    thread_yield();
+}
+
+__attribute__((unused))
+static void no_wait(uint32_t unused)
+{
+    (void) unused;
+    /* do not wait */
+}
+
+/* worker_config is a small configuration structure for the thread_worker */
+struct worker_config {
+    void (*waitfn)(uint32_t);  /**< the resting strategy */
+    uint32_t workload;         /**< the amount of work to do per set */
+};
+
+/*
+ * the following are threads that count and wait with different strategies and
+ * print their current count in steps.
+ * the ration of active (doing hard work like checking the timer)
+ * to passive (wait to be informed when a certain time is there) waiting
+ * is determined by there value given to the thread.
+ * no_wait and yield_wait threads are restless an therefore never pause.
+ */
+
+void * thread_worker(void * d)
+{
+    nice_wait(200 *  US_PER_MS);  /* always be nice at start */
+#ifdef DEVELHELP
+    const char *name = thread_get_active()->name;
+#else
+    int16_t pid = thread_getpid();
+#endif
+
+    uint32_t w = 0;
+    struct worker_config *wc =  d;
+    /* Each set consists of STEPS_PER_SET steps which are divided into work (busy waiting)
+     * and resting.
+     * E.g. if there are 10 steps per set, the maximum workload is 10, which means no rest.
+     * If the given value is out of range work ratio is set to half of STEPS_PER_SET */
+    uint32_t work = wc->workload;
+    if (work > STEPS_PER_SET) {
+        work = STEPS_PER_SET / 2;
+    }
+    uint32_t rest = (STEPS_PER_SET - work);
+    uint32_t step = 0;
+
+    /* work some time and rest */
+    for (;;) {
+        if (w - step >= PRINT_STEPS) {
+#ifdef DEVELHELP
+            printf("%s: %" PRIu32 ", %" PRIu32 "\n", name, w, work);
+#else
+            printf("T-Pid %i:%" PRIu32 ", %" PRIu32 "\n", pid, w, work);
+#endif
+            step = w;
+        }
+        do_work(work * WORK_SCALE);
+        w += work;
+        wc->waitfn(rest * WORK_SCALE);
+    }
+}
+/*
+ * nice_wait ->  a thread does nice breaks giving other threads time to do something
+ * bad_wait ->   a thread that waits by spinning (intensely looking at the clock)
+ * yield_wait -> a restless thread that yields before continuing with the next work package
+ * no_wait ->    a restless thread always working until it is preempted
+ */
+/* yield_wait and nice_wait threads are able to work in "parallel" without sched_round_robin */
+
+#ifndef  THREAD_1
+#define  THREAD_1 {no_wait, 5}
+#endif
+
+#ifndef  THREAD_2
+#define  THREAD_2 {no_wait, 5}
+#endif
+
+#ifndef  THREAD_3
+#define  THREAD_3 {no_wait, 5}
+#endif
+
+/*a TINY Stack should be enough*/
+#ifndef WORKER_STACKSIZE
+#define WORKER_STACKSIZE (THREAD_STACKSIZE_TINY+THREAD_EXTRA_STACKSIZE_PRINTF)
+#endif
+
+int main(void)
+{
+    {
+        static char stack[WORKER_STACKSIZE];
+        static struct worker_config wc = THREAD_1;   /* 0-10 workness */
+        thread_create(stack, sizeof(stack), 7, THREAD_CREATE_STACKTEST,
+                      thread_worker, &wc, "T1");
+    }
+    {
+        static char stack[WORKER_STACKSIZE];
+        static struct worker_config wc = THREAD_2;   /* 0-10 workness */
+        thread_create(stack, sizeof(stack), 7, THREAD_CREATE_STACKTEST,
+                      thread_worker, &wc, "T2");
+    }
+    {
+        static char stack[WORKER_STACKSIZE];
+        static struct worker_config wc = THREAD_3;   /* 0-10 workness */
+        thread_create(stack, sizeof(stack), 7, THREAD_CREATE_STACKTEST,
+                      thread_worker, &wc, "T3");
+    }
+}

sys/Makefile.dep

@@ -357,6 +357,15 @@ ifneq (,$(filter schedstatistics,$(USEMODULE)))
   USEMODULE += sched_cb
 endif
 
+ifneq (,$(filter sched_round_robin,$(USEMODULE)))
+# this depends on either ztimer_usec or ztimer_msec if neither is used
+# prior to this msec is preferred
+  ifeq (,$(filter ztimer_usec,$(USEMODULE))$(filter ztimer_msec,$(USEMODULE)))
+    USEMODULE += ztimer_msec
+  endif
+  USEMODULE += sched_runq_callback
+endif
+
 ifneq (,$(filter saul_reg,$(USEMODULE)))
   USEMODULE += saul
 endif

sys/auto_init/auto_init.c

@@ -49,6 +49,11 @@ void auto_init(void)
         extern void init_schedstatistics(void);
         init_schedstatistics();
     }
+    if (IS_USED(MODULE_SCHED_ROUND_ROBIN)) {
+        LOG_DEBUG("Auto init sched_round_robin.\n");
+        extern void sched_round_robin_init(void);
+        sched_round_robin_init();
+    }
     if (IS_USED(MODULE_DUMMY_THREAD)) {
         extern void dummy_thread_create(void);
         dummy_thread_create();

sys/include/sched_round_robin.h

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2020 TUBA Freiberg
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @defgroup    sched_round_robin Round Robin Scheduler
+ * @ingroup     sys
+ * @brief       This module module provides round robin scheduling for all
+ *              runable threads within each not masked priority.
+ *              Priority 0 is masked by default.
+ *              This implementation tries to find a balance between
+ *              low resources (static memory: a timer and an uint8),
+ *              fairness in terms of CPU time share and simplicity.
+ *              But it does round robin the runqueue when the timer ticks
+ *              even if the thread just got the CPU.
+ *
+ *              This module might be used if threads are not divisible
+ *              into priorities and cooperation can not be ensured.
+ *
+ * @{
+ *
+ * @file
+ * @brief       Round Robin Scheduler
+ *
+ * @author      Karl Fessel <karl.fessel@ovgu.de>
+ *
+ */
+#ifndef SCHED_ROUND_ROBIN_H
+#define SCHED_ROUND_ROBIN_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if !defined(SCHED_RR_TIMEOUT) || defined(DOXYGEN)
+/**
+ * @brief   Time between round robin calls in Units of SCHED_RR_TIMERBASE
+ *
+ * @details Defaults to 10ms
+ */
+#if MODULE_ZTIMER_MSEC
+#define SCHED_RR_TIMEOUT 10
+#else
+#define SCHED_RR_TIMEOUT 10000
+#endif
+#endif
+
+#if !defined(SCHED_RR_TIMERBASE) || defined(DOXYGEN)
+/**
+ * @brief   ztimer to use for the round robin scheduler
+ *
+ * @details Defaults to ZTIMER_MSEC if available else it uses ZTIMER_USEC
+ */
+#if MODULE_ZTIMER_MSEC
+#define SCHED_RR_TIMERBASE ZTIMER_MSEC
+#else
+#define SCHED_RR_TIMERBASE ZTIMER_USEC
+#endif
+#endif
+
+#if !defined(SCHED_RR_MASK) || defined(DOXYGEN)
+/**
+ * @brief   Masks off priorities that should not be scheduled default: 0 is masked
+ *
+ * @details Priority 0 (highest) should always be masked.
+ *          Threads with that priority may not be programmed
+ *          with the possibility of being scheduled in mind.
+ *          Parts of this scheduler assume 0 current_rr_priority is uninitialised.
+ */
+#define SCHED_RR_MASK (1 << 0)
+#endif
+
+/**
+ *  @brief Initialises the Round Robin Scheduler
+ */
+void sched_round_robin_init(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SCHED_ROUND_ROBIN_H */
+/** @} */

sys/sched_round_robin/Kconfig

@@ -0,0 +1,19 @@
+# Copyright (c) 2021 TUBA Freiberg
+#
+# This file is subject to the terms and conditions of the GNU Lesser
+# General Public License v2.1. See the file LICENSE in the top level
+# directory for more details.
+#
+
+config  MODULE_SCHED_ROUND_ROBIN
+    bool "round robin scheduling support"
+    depends on MODULE_ZTIMER_MSEC || MODULE_ZTIMER_USEC
+    depends on TEST_KCONFIG
+    select MODULE_SCHED_RUNQUEUE_API
+
+if MODULE_SCHED_ROUND_ROBIN
+config SCHED_RR_TIMEOUT
+    int "timeout for round robin scheduling"
+    default 10000
+
+endif

sys/sched_round_robin/Makefile

@@ -0,0 +1 @@
+include $(RIOTBASE)/Makefile.base

sys/sched_round_robin/sched_round_robin.c

@@ -0,0 +1,118 @@
+/*
+ * Copyright (C) 2021 TUBA Freiberg
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+/**
+ * @ingroup     sys
+ * @{
+ *
+ * @file
+ * @brief       Round Robin Scheduler implementation
+ *
+ * @author      Karl Fessel <karl.fessel@ovgu.de>
+ *
+ * @}
+ */
+
+#include "sched.h"
+#include "thread.h"
+#include "ztimer.h"
+#include "sched_round_robin.h"
+
+#define ENABLE_DEBUG 0
+#include "debug.h"
+
+static void _sched_round_robin_cb(void *d);
+
+static ztimer_t _rr_timer = { .callback = _sched_round_robin_cb };
+
+/*
+ * Assuming simple reads from and writes to a byte to be atomic on every board
+ * Value 0 is assumed to show this system is uninitialised.
+ * The timer will not be started for prio = 0;
+ */
+static uint8_t _current_rr_priority = 0;
+
+void sched_runq_callback(uint8_t prio);
+
+void _sched_round_robin_cb(void *d)
+{
+    (void)d;
+    /*
+     * reorder current Round Robin priority
+     * (put the current thread at the end of the run queue of its priority)
+     * and setup the scheduler to schedule when returning from the IRQ
+     */
+    uint8_t prio = _current_rr_priority;
+    if (prio != 0xff) {
+        DEBUG_PUTS("Round_Robin");
+        sched_runq_advance(prio);
+        _current_rr_priority = 0xff;
+    }
+    thread_t *active_thread = thread_get_active();
+    if (active_thread) {
+        uint8_t active_priority = active_thread->priority;
+        if (active_priority == prio) {
+            thread_yield_higher();
+            /* thread change will call the runqueue_change_cb */
+        }
+        else {
+            sched_runq_callback(active_priority);
+        }
+    }
+}
+
+static inline void _sched_round_robin_remove(void)
+{
+    _current_rr_priority = 0xff;
+    ztimer_remove(SCHED_RR_TIMERBASE, &_rr_timer);
+}
+
+static inline void _sched_round_robin_set(uint8_t prio)
+{
+    if (prio == 0) {
+        return;
+    }
+    _current_rr_priority = prio;
+    ztimer_set(SCHED_RR_TIMERBASE, &_rr_timer, SCHED_RR_TIMEOUT);
+}
+
+void sched_runq_callback(uint8_t prio)
+{
+    if (SCHED_RR_MASK & (1 << prio) || prio == 0) {
+        return;
+    }
+
+    if (_current_rr_priority == prio) {
+        if (sched_runq_is_empty(prio)) {
+            _sched_round_robin_remove();
+            thread_t *active_thread = thread_get_active();
+            if (active_thread) {
+                prio = active_thread->priority;
+            }
+            else {
+                return;
+            }
+        }
+    }
+
+    if (_current_rr_priority == 0xff  &&
+        !(SCHED_RR_MASK & (1 << prio)) &&
+        sched_runq_more_than_one(prio)) {
+        _sched_round_robin_set(prio);
+    }
+}
+
+void sched_round_robin_init(void)
+{
+    /* init _current_rr_priority */
+    _current_rr_priority = 0xff;
+    /* check if applicable to active priority */
+    thread_t *active_thread = thread_get_active();
+    if (active_thread) {
+        sched_runq_callback(active_thread->priority);
+    }
+}

tests/sys_sched_round_robin/Makefile

@@ -0,0 +1,10 @@
+include ../Makefile.tests_common
+
+# Set to 1 to disable the round-robin scheduling module
+NORR ?= 0
+
+ifneq (1,$(NORR))
+  USEMODULE += sched_round_robin
+endif
+
+include $(RIOTBASE)/Makefile.include

tests/sys_sched_round_robin/Makefile.ci

@@ -0,0 +1,14 @@
+BOARD_INSUFFICIENT_MEMORY := \
+    arduino-duemilanove \
+    arduino-leonardo \
+    arduino-nano \
+    arduino-uno \
+    atmega328p \
+    atmega328p-xplained-mini\
+    nucleo-f031k6 \
+    nucleo-f042k6 \
+    nucleo-l011k4 \
+    samd10-xmini \
+    stk3200 \
+    stm32f030f4-demo \
+    #

tests/sys_sched_round_robin/README.md

@@ -0,0 +1,54 @@
+Round Robing Scheduling Test
+========================
+
+This application is a simple test case for round-robin scheduling.
+Two threads are started with the same priority.
+The first thread is a busy loop and is started first.
+The second thread unlocks a mutex allowing the main thread to continue and exit.
+
+Without Round Robin scheduling the busy loop thread would run indefinitely,
+with round-robin in eventually getting de-scheduled allowing the main thread to run and exit.
+
+Usage
+=====
+
+By default `sched_round_robin` is included:
+
+`make tests/sys_sched_round_robin flash term`
+
+```
+...Board Initialisation...
+
+Help: Press s to start test, r to print it is ready
+s
+START
+main(): This is RIOT! (Version: ...)
+starting threads
+double locking mutex
+mutex_thread yield
+bad thread looping
+unlock mutex
+[SUCCESS]
+
+```
+
+It can be excluded from the build by setting the command-line argument `NORR=1`:
+
+
+`NORR=1 make tests/sys_sched_round_robin flash term`
+
+```
+...Board Initialisation...
+
+Help: Press s to start test, r to print it is ready
+s
+START
+main(): This is RIOT! (Version: ...)
+starting threads
+double locking mutex
+mutex_thread yield
+bad thread looping
+```
+
+This will loop endlessly as the bad thread does not release the CPU,
+`make test` will timeout in that case.

tests/sys_sched_round_robin/main.c

@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2020 TUBA Freiberg
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @ingroup     tests
+ * @{
+ * @file
+ * @brief       Test sys/sched_round_robin
+ * @author      Karl Fessel <karl.fessel@ovgu.de>
+ * @}
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+
+#include "thread.h"
+
+static kernel_pid_t main_pid;
+
+
+void * thread_wakeup_main(void *d)
+{
+    (void) d;
+    puts("wakup_thread yield");
+    thread_yield();
+    while (puts("wakeup main"), thread_wakeup(main_pid) == (int)STATUS_NOT_FOUND) {
+        thread_yield();
+    };
+    return NULL;
+}
+
+void * thread_bad(void *d)
+{
+    (void) d;
+    puts("bad thread looping");
+    for (;;) {
+        /* I'm a bad thread I do nothing and I do that all the time */
+    }
+}
+
+/* each thread gets a stack */
+static char stack[2][THREAD_STACKSIZE_DEFAULT];
+
+/* shared priority of the threads - lower than main waiting for it to sleep */
+static const uint8_t shared_prio = THREAD_PRIORITY_MAIN + 1;
+
+int main(void)
+{
+    puts("starting threads");
+    main_pid = thread_getpid();
+    thread_create(stack[0], sizeof(stack[0]), shared_prio, THREAD_CREATE_STACKTEST,
+                  thread_wakeup_main, NULL, "TWakeup");
+    thread_create(stack[1], sizeof(stack[1]), shared_prio, THREAD_CREATE_STACKTEST,
+                  thread_bad, NULL, "TBad");
+    puts("main is going to sleep");
+    thread_sleep();
+
+    /* success: main got woken up again which means "TWakup" got cpu time
+     * even though "TBad" was trying to hog the whole CPU */
+    puts("[SUCCESS]");
+}

tests/sys_sched_round_robin/tests/01-run.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2021 TUBA Freiberg
+#
+# This file is subject to the terms and conditions of the GNU Lesser
+# General Public License v2.1. See the file LICENSE in the top level
+# directory for more details.
+
+import sys
+from testrunner import run
+
+
+def testfunc(child):
+    child.expect_exact("starting threads")
+    child.expect_exact("main is going to sleep")
+    child.expect_exact("wakeup main")
+    child.expect_exact("[SUCCESS]")
+
+
+if __name__ == "__main__":
+    sys.exit(run(testfunc))