sys/net/gnrc/tx_sync: new module

The new `gnrc_tx_sync` module allows users of the GNRC network stack to
synchronize with the actual transmission of outgoing packets. This is directly
integrated into gnrc_sock. Hence, if `gnrc_tx_sync` is used, calls to e.g.
sock_udp_send() will block until the network stack has processed the message.

Use cases:
1. Prevent packet drop when sending at high rate
    - If the application is sending faster than the stack can handle, the
      message queues will overflow and outgoing packets are lost
2. Passing auxiliary data about the transmission back the stack
    - When e.g. the number of required retransmissions, the transmission time
      stamp, etc. should be made available to a user of an UDP sock, a
      synchronization mechanism is needed
3. Simpler error reporting without footguns
    - The current approach of using `core/msg` for passing up error messages is
      difficult to use if other message come in. Currently, gnrc_sock is
      busy-waiting and fetching messages from the message queue until the number
      of expected status reports is received. It will enqueue all
      non-status-report messages again at the end of the queue. This has
      multiple issues:
        - Busy waiting is especially in lower power scenarios with time slotted
          MAC protocols harmful, as the CPU will remain active and consume
          power even though the it could sleep until the TX slot is reached
        - The status reports from the network stack are send to the user thread
          blocking. If the message queue of the user thread is full, the network
          stack would block until the user stack can fetch the messages. If
          another higher priority thread would start sending a message, it
          would busy wait for its status reports to completely come in. Hence,
          the first thread doesn't get CPU time to fetch messages and unblock
          the network stack. As a result, the system would lock up completely.
    - Just adding the error/status code to the gnrc_tx_sync_t would preallocate
      and reserve memory for the error reporting. That way gnrc_sock does not
      need to search through the message queue for status reports and the
      network stack does not need to block for the user thread fetching it.

sys/include/net/gnrc/nettype.h

@@ -48,6 +48,11 @@ extern "C" {
  * @note    Expand at will.
  */
 typedef enum {
+    /**
+     * @brief   TX synchronization data for passing up error data or
+     *          auxiliary data
+     */
+    GNRC_NETTYPE_TX_SYNC = -3,
     /**
      * @brief   Protocol is as defined in @ref gnrc_netif_hdr_t. Not usable with
      *          @ref net_gnrc_netreg

sys/include/net/gnrc/tx_sync.h

@@ -0,0 +1,139 @@
+/*
+ * Copyright (C) 2020 Otto-von-Guericke-Universität Magdeburg
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @defgroup    net_gnrc_tx_sync    Helpers for synchronizing with transmission.
+ * @ingroup     net_gnrc
+ * @brief       This allows upper layers to wait for a transmission to complete
+ *              (or fail) and for passing up data about the transmission.
+ * @{
+ *
+ * @file
+ * @brief   Definitions for TX sync.
+ *
+ * @author  Marian Buschsieweke <marian.buschsieweke@ovgu.de>
+ */
+#ifndef NET_GNRC_TX_SYNC_H
+#define NET_GNRC_TX_SYNC_H
+
+#include <errno.h>
+#include <stdint.h>
+
+#include "mutex.h"
+#include "net/gnrc/pktbuf.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief   TX synchronization data */
+typedef struct {
+    mutex_t signal;     /**< Mutex used for synchronization */
+} gnrc_tx_sync_t;
+
+/**
+ * @brief   Helper to initialize a gnrc_tx_sync_t structure
+ */
+static inline gnrc_tx_sync_t gnrc_tx_sync_init(void)
+{
+    gnrc_tx_sync_t result = { .signal = MUTEX_INIT_LOCKED };
+    return result;
+}
+
+/**
+ * @brief   Build a TX sync snip
+ *
+ * @param[in,out]   tx_sync     TX sync structure the snip should hold
+ *
+ * @return  The TX sync snip holing @p tx_sync
+ * @retval  NULL    Allocation Failed
+ *
+ * @note    No need to initialize @p tx_sync, this function will do it for you.
+ */
+static inline gnrc_pktsnip_t * gnrc_tx_sync_build(gnrc_tx_sync_t *tx_sync)
+{
+    *tx_sync = gnrc_tx_sync_init();
+    gnrc_pktsnip_t *snip = gnrc_pktbuf_add(NULL, NULL, 0, GNRC_NETTYPE_TX_SYNC);
+    if (!snip) {
+        return NULL;
+    }
+    snip->data = tx_sync;
+    return snip;
+}
+
+/**
+ * @brief   Appends a newly allocated tx sync pktsnip to the end of the packet
+ *
+ * @param[in]       pkt     Packet to append TX sync pktsnip to
+ * @param[in,out]   tx_sync TX sync structure to initialize and append
+ *
+ * @retval  0       Success
+ * @retval  -ENOMEM Allocation failed
+ *
+ * @note    No need to initialize @p tx_sync, this function will do it for you.
+ */
+static inline int gnrc_tx_sync_append(gnrc_pktsnip_t *pkt,
+                                      gnrc_tx_sync_t *tx_sync)
+{
+    gnrc_pktsnip_t *snip = gnrc_tx_sync_build(tx_sync);
+    if (!snip) {
+        return -ENOMEM;
+    }
+    gnrc_pkt_append(pkt, snip);
+    return 0;
+}
+
+/**
+ * @brief   Split off the TX sync snip and return it
+ * @param[in,out]   pkt     Packet to split off the TX sync snip
+ * @return  The TX sync snip that no longer is part of @p pkt
+ * @retval  NULL            @p pkt contains no TX sync snip
+ */
+gnrc_pktsnip_t * gnrc_tx_sync_split(gnrc_pktsnip_t *pkt);
+
+/**
+ * @brief   Signal TX completion via the given tx sync packet snip
+ *
+ * @param[in]       pkt     The tx sync packet snip of the packet that was transmitted
+ *
+ * @pre     Module gnrc_netttype_tx_sync is sued
+ * @pre     `pkt->type == GNRC_NETTYPE_TX_SYNC`
+ */
+static inline void gnrc_tx_complete(gnrc_pktsnip_t *pkt)
+{
+    assert(IS_USED(MODULE_GNRC_TX_SYNC) && (pkt->type == GNRC_NETTYPE_TX_SYNC));
+    gnrc_tx_sync_t *sync = pkt->data;
+    mutex_unlock(&sync->signal);
+}
+
+/**
+ * @brief   Block until transmission of the corresponding packet has completed
+ *          or failed
+ *
+ * @param[in,out]   sync    TX sync structure used for synchronization
+ *
+ * @pre     @p sync has been added to the packet to synchronize with, e.g. via
+ *          @ref gnrc_tx_sync_append
+ * @pre     The packet linked to @p sync has been passed to the network stack
+ *          for transmission. Otherwise this will block forever.
+ *
+ * @note    If the transmission has already completed, this function will not
+ *          block and return immediately instead.
+ */
+static inline void gnrc_tx_sync(gnrc_tx_sync_t *sync)
+{
+    mutex_lock(&sync->signal);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* NET_GNRC_TX_SYNC_H */
+/** @} */

sys/net/gnrc/Makefile

@@ -136,5 +136,8 @@ endif
 ifneq (,$(filter gnrc_tcp,$(USEMODULE)))
   DIRS += transport_layer/tcp
 endif
+ifneq (,$(filter gnrc_tx_sync,$(USEMODULE)))
+  DIRS += tx_sync
+endif
 
 include $(RIOTBASE)/Makefile.base

sys/net/gnrc/netif/gnrc_netif.c

@@ -47,6 +47,7 @@
 
 #include "net/gnrc/netif.h"
 #include "net/gnrc/netif/internal.h"
+#include "net/gnrc/tx_sync.h"
 
 #define ENABLE_DEBUG 0
 #include "debug.h"
@@ -1534,7 +1535,13 @@ static void _send(gnrc_netif_t *netif, gnrc_pktsnip_t *pkt, bool push_back)
      * layer implementations in case `gnrc_netif_pktq` is included */
     gnrc_pktbuf_hold(pkt, 1);
 #endif /* IS_USED(MODULE_GNRC_NETIF_PKTQ) */
+    gnrc_pktsnip_t *tx_sync = IS_USED(MODULE_GNRC_TX_SYNC)
+                            ? gnrc_tx_sync_split(pkt) : NULL;
     res = netif->ops->send(netif, pkt);
+    if (tx_sync != NULL) {
+        uint32_t err = (res < 0) ? -res : GNRC_NETERR_SUCCESS;
+        gnrc_pktbuf_release_error(tx_sync, err);
+    }
 #if IS_USED(MODULE_GNRC_NETIF_PKTQ)
     if (res == -EBUSY) {
         int put_res;

sys/net/gnrc/network_layer/sixlowpan/frag/gnrc_sixlowpan_frag.c

@@ -19,13 +19,14 @@
 #include <assert.h>
 #include <inttypes.h>
 
-#include "net/gnrc/pktbuf.h"
 #include "net/gnrc/netapi.h"
+#include "net/gnrc/netif.h"
 #include "net/gnrc/netif/hdr.h"
+#include "net/gnrc/pktbuf.h"
 #include "net/gnrc/sixlowpan/frag.h"
 #include "net/gnrc/sixlowpan/frag/rb.h"
 #include "net/gnrc/sixlowpan/internal.h"
-#include "net/gnrc/netif.h"
+#include "net/gnrc/tx_sync.h"
 #include "net/sixlowpan.h"
 #include "utlist.h"
 
@@ -111,7 +112,6 @@ static gnrc_pktsnip_t *_build_frag_pkt(gnrc_pktsnip_t *pkt,
     frag_hdr->disp_size = byteorder_htons(fbuf->datagram_size);
     frag_hdr->tag = byteorder_htons(fbuf->tag);
 
-
     return gnrc_pkt_prepend(frag, netif);
 }
 
@@ -177,7 +177,8 @@ static uint16_t _send_1st_fragment(gnrc_netif_t *iface,
 
 static uint16_t _send_nth_fragment(gnrc_netif_t *iface,
                                    gnrc_sixlowpan_frag_fb_t *fbuf,
-                                   size_t payload_len)
+                                   size_t payload_len,
+                                   gnrc_pktsnip_t **tx_sync)
 {
     gnrc_pktsnip_t *frag, *pkt = fbuf->pkt;
     sixlowpan_frag_n_t *hdr;
@@ -227,6 +228,9 @@ static uint16_t _send_nth_fragment(gnrc_netif_t *iface,
     if ((offset + local_offset) < payload_len) {
         gnrc_netif_hdr_t *netif_hdr = frag->data;
         netif_hdr->flags |= GNRC_NETIF_HDR_FLAGS_MORE_DATA;
+    } else if (IS_USED(MODULE_GNRC_TX_SYNC) && (*tx_sync != NULL)) {
+        gnrc_pkt_append(frag, *tx_sync);
+        *tx_sync = NULL;
     }
     DEBUG("6lo frag: send subsequent fragment (datagram size: %u, "
           "datagram tag: %" PRIu16 ", offset: %" PRIu8 " (%u bytes), "
@@ -242,6 +246,7 @@ void gnrc_sixlowpan_frag_send(gnrc_pktsnip_t *pkt, void *ctx, unsigned page)
     assert(ctx != NULL);
     gnrc_sixlowpan_frag_fb_t *fbuf = ctx;
     gnrc_netif_t *iface;
+    gnrc_pktsnip_t *tx_sync = NULL;
     uint16_t res;
     /* payload_len: actual size of the packet vs
      * datagram_size: size of the uncompressed IPv6 packet */
@@ -262,6 +267,10 @@ void gnrc_sixlowpan_frag_send(gnrc_pktsnip_t *pkt, void *ctx, unsigned page)
     }
 #endif
 
+    if (IS_USED(MODULE_GNRC_TX_SYNC)) {
+        tx_sync = gnrc_tx_sync_split((pkt) ? pkt : fbuf->pkt);
+    }
+
     /* Check whether to send the first or an Nth fragment */
     if (fbuf->offset == 0) {
         if ((res = _send_1st_fragment(iface, fbuf, payload_len)) == 0) {
@@ -272,7 +281,7 @@ void gnrc_sixlowpan_frag_send(gnrc_pktsnip_t *pkt, void *ctx, unsigned page)
     }
     /* (offset + (datagram_size - payload_len) < datagram_size) simplified */
     else if (fbuf->offset < payload_len) {
-        if ((res = _send_nth_fragment(iface, fbuf, payload_len)) == 0) {
+        if ((res = _send_nth_fragment(iface, fbuf, payload_len, &tx_sync)) == 0) {
             /* error sending subsequent fragment */
             DEBUG("6lo frag: error sending subsequent fragment"
                   "(offset = %u)\n", fbuf->offset);
@@ -283,6 +292,11 @@ void gnrc_sixlowpan_frag_send(gnrc_pktsnip_t *pkt, void *ctx, unsigned page)
         goto error;
     }
     fbuf->offset += res;
+    if (IS_USED(MODULE_GNRC_TX_SYNC) && tx_sync) {
+        /* re-attach tx_sync to allow releasing it at end
+         * of transmission, or transmission failure */
+        gnrc_pkt_append((pkt) ? pkt : fbuf->pkt, tx_sync);
+    }
     if (!gnrc_sixlowpan_frag_fb_send(fbuf)) {
         DEBUG("6lo frag: message queue full, can't issue next fragment "
               "sending\n");
@@ -293,6 +307,9 @@ void gnrc_sixlowpan_frag_send(gnrc_pktsnip_t *pkt, void *ctx, unsigned page)
 error:
     gnrc_pktbuf_release(fbuf->pkt);
     fbuf->pkt = NULL;
+    if (IS_USED(MODULE_GNRC_TX_SYNC) && tx_sync) {
+        gnrc_pktbuf_release(tx_sync);
+    }
 }
 
 void gnrc_sixlowpan_frag_recv(gnrc_pktsnip_t *pkt, void *ctx, unsigned page)

sys/net/gnrc/pktbuf/gnrc_pktbuf.c

@@ -15,6 +15,7 @@
 
 #include "mutex.h"
 #include "net/gnrc/pktbuf.h"
+#include "net/gnrc/tx_sync.h"
 
 #include "pktbuf_internal.h"
 
@@ -94,7 +95,13 @@ void gnrc_pktbuf_release_error(gnrc_pktsnip_t *pkt, uint32_t err)
         tmp = pkt->next;
         if (pkt->users == 1) {
             pkt->users = 0; /* not necessary but to be on the safe side */
-            gnrc_pktbuf_free_internal(pkt->data, pkt->size);
+            if (!IS_USED(MODULE_GNRC_TX_SYNC)
+                || (pkt->type != GNRC_NETTYPE_TX_SYNC)) {
+                gnrc_pktbuf_free_internal(pkt->data, pkt->size);
+            }
+            else {
+                gnrc_tx_complete(pkt);
+            }
             gnrc_pktbuf_free_internal(pkt, sizeof(gnrc_pktsnip_t));
         }
         else {

sys/net/gnrc/sock/gnrc_sock.c

@@ -23,6 +23,7 @@
 #include "net/gnrc/ipv6.h"
 #include "net/gnrc/ipv6/hdr.h"
 #include "net/gnrc/netreg.h"
+#include "net/gnrc/tx_sync.h"
 #include "net/udp.h"
 #include "utlist.h"
 #include "xtimer.h"
@@ -205,11 +206,22 @@ ssize_t gnrc_sock_send(gnrc_pktsnip_t *payload, sock_ip_ep_t *local,
 #ifdef MODULE_GNRC_NETERR
     unsigned status_subs = 0;
 #endif
+#if IS_USED(MODULE_GNRC_TX_SYNC)
+    gnrc_tx_sync_t tx_sync;
+#endif
 
     if (local->family != remote->family) {
         gnrc_pktbuf_release(payload);
         return -EAFNOSUPPORT;
     }
+
+#if IS_USED(MODULE_GNRC_TX_SYNC)
+    if (gnrc_tx_sync_append(payload, &tx_sync)) {
+        gnrc_pktbuf_release(payload);
+        return -ENOMEM;
+    }
+#endif
+
     switch (local->family) {
 #ifdef SOCK_HAS_IPV6
         case AF_INET6: {
@@ -271,6 +283,11 @@ ssize_t gnrc_sock_send(gnrc_pktsnip_t *payload, sock_ip_ep_t *local,
         gnrc_pktbuf_release(pkt);
         return -EBADMSG;
     }
+
+#if IS_USED(MODULE_GNRC_TX_SYNC)
+    gnrc_tx_sync(&tx_sync);
+#endif
+
 #ifdef MODULE_GNRC_NETERR
     uint32_t last_status = GNRC_NETERR_SUCCESS;
 

sys/net/gnrc/tx_sync/Makefile

@@ -0,0 +1,3 @@
+MODULE := gnrc_tx_sync
+
+include $(RIOTBASE)/Makefile.base

sys/net/gnrc/tx_sync/gnrc_tx_sync.c

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2020 Otto-von-Guericke-Universität Magdeburg
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @{
+ *
+ * @file
+ * @brief   Implementation of the TX synchronization helpers
+ *
+ * @author  Marian Buschsieweke <marian.buschsieweke@ovgu.de>
+ * @}
+ */
+
+#include <assert.h>
+
+#include "net/gnrc/tx_sync.h"
+
+gnrc_pktsnip_t * gnrc_tx_sync_split(gnrc_pktsnip_t *pkt)
+{
+    if (!pkt) {
+        return NULL;
+    }
+    gnrc_pktsnip_t *next;
+
+    while ((next = pkt->next) != NULL) {
+        if (next->type == GNRC_NETTYPE_TX_SYNC) {
+            pkt->next = NULL;
+            /* TX sync snip must be last snip in packet */
+            assert(next->next == NULL);
+            return next;
+        }
+        pkt = next;
+    }
+
+    return NULL;
+}