tests/riotboot/flashwrite: add automatic test

tests/riotboot_flashwrite/Makefile

@@ -1,61 +1,77 @@
-DEVELHELP ?= 0
+# If no BOARD is found in the environment, use this default:
 BOARD ?= samr21-xpro
+
 include ../Makefile.tests_common
 
 # Include packages that pull up and auto-init the link layer.
-# NOTE: 6LoWPAN will be included if IEEE802.15.4 devices are present
-USEMODULE += gnrc_netdev_default
 USEMODULE += auto_init_gnrc_netif
 # Specify the mandatory networking modules for IPv6 and UDP
 USEMODULE += gnrc_ipv6_default
-USEMODULE += gnrc_ipv6_router_default
 USEMODULE += sock_udp
 # Additional networking modules that can be dropped if not needed
 USEMODULE += gnrc_icmpv6_echo
 
+# Required for nanocoap server
 USEMODULE += nanocoap_sock
 
 # include this for printing IP addresses
+USEMODULE += shell
 USEMODULE += shell_commands
 
-# Comment this out to enable code in RIOT that does safety checking
-# which is not needed in a production environment but helps in the
-# development process:
-#DEVELHELP = 1
-
-# Use different settings when compiling for one of the following (low-memory)
-# boards
-LOW_MEMORY_BOARDS := nucleo-f334r8
-
-# uncomment these to use ethos
-#USEMODULE += stdio_ethos gnrc_uhcpc
-#
-## ethos baudrate can be configured from make command
-#ETHOS_BAUDRATE ?= 115200
-#CFLAGS += -DETHOS_BAUDRATE=$(ETHOS_BAUDRATE)
-
-ifneq (,$(filter $(BOARD),$(LOW_MEMORY_BOARDS)))
-  $(info Using low-memory configuration for microcoap_server.)
-  ## low-memory tuning values
-  USEMODULE += prng_minstd
-endif
-
 # include riotboot modules
 USEMODULE += riotboot_flashwrite
 FEATURES_REQUIRED += riotboot
 
-# Change this to 0 show compiler invocation lines by default:
+# Change this to 0 to not use ethos and instead include 6LoWPAN if
-QUIET ?= 1
+# IEEE802.15.4 devices are present
+USE_ETHOS ?= 1
+
+ifeq (1,$(USE_ETHOS))
+  USEMODULE += stdio_ethos
+  USEMODULE += gnrc_uhcpc
+
+  # ethos baudrate can be configured from make command
+  ETHOS_BAUDRATE ?= 115200
+  CFLAGS += -DETHOS_BAUDRATE=$(ETHOS_BAUDRATE)
+
+  # make sure ethos and uhcpd are built
+  TERMDEPS += host-tools
+
+  # For local testing, run
+  #
+  #     $ cd dist/tools/ethos; sudo ./setup_network.sh riot0 2001:db8::0/64
+  #
+  #... in another shell and keep it running.
+  export TAP ?= riot0
+  TERMPROG = $(RIOTTOOLS)/ethos/ethos
+  TERMFLAGS = $(TAP) $(PORT)
+else
+  USEMODULE += gnrc_netdev_default
+endif
+
+# Ensure both slot bin files are always generated and linked to avoid compiling
+# during the test. This ensures that "BUILD_IN_DOCKER=1 make test"
+# can rely on them being present without having to trigger re-compilation.
+BUILD_FILES += $(SLOT_RIOT_ELFS:%.elf=%.bin)
+
+# The test needs the linked slot binaries without header in order to be able to
+# create final binaries with specific APP_VER values. The CI RasPi test workers
+# don't compile themselves, thus add the required files here so they will be
+# submitted along with the test jobs.
+TEST_EXTRA_FILES += $(SLOT_RIOT_ELFS)
 
 include $(RIOTBASE)/Makefile.include
 
-ifneq (,$(filter $(BOARD),$(LOW_MEMORY_BOARDS)))
+# lower pktbuf size to something sufficient for this application
-  # lower pktbuf buffer size
+# Set GNRC_PKTBUF_SIZE via CFLAGS if not being set via Kconfig.
-  # Set GNRC_PKTBUF_SIZE via CFLAGS if not being set via Kconfig.
+ifndef CONFIG_GNRC_PKTBUF_SIZE
-  ifndef CONFIG_GNRC_PKTBUF_SIZE
+  CFLAGS += -DCONFIG_GNRC_PKTBUF_SIZE=2000
-    CFLAGS += -DCONFIG_GNRC_PKTBUF_SIZE=1000
-  endif
 endif
 
 # Set a custom channel if needed
 include $(RIOTMAKE)/default-radio-settings.inc.mk
+
+.PHONY: host-tools
+
+host-tools:
+	$(Q)env -u CC -u CFLAGS make -C $(RIOTTOOLS)

tests/riotboot_flashwrite/README.md

@@ -8,11 +8,38 @@ over network without *any* kind of authentication or other security!
 Please see the README of examples/nanocoap_server for instructions on how to
 set up a network for testing.
 
-# How to test
+## Requirements
+
+This test uses [aiocoap](https://pypi.org/project/aiocoap/) to send the firmware to the device over coap.
+
+### How to test over Ethos
+
+First set up the network:
+
+    $ sudo dist/tools/ethos/setup_network.sh riot0 2001:db8::/64
+
+Then provide de device and test:
+
+    $ BOARD=<board> make flash test-with-config
+
+### How to test over the air (802.15.4)
+
+On another device setup a BR and start `start_network.sh` on that device serial
+port.
+
+    $ BOARD=<board> make -C examples/gnrc_border_router flash
+
+    $ sudo dist/tools/ethos/start_network.sh /dev/ttyACMx riot0 2001:db8::/64
+
+Then provide the device and test:
+
+    $ USE_ETHOS=0 BOARD=<board> make flash test-with-config
+
+### Manual test
 
 First, compile and flash with riotboot enabled:
 
-    $ BOARD=<board> make riotboot/flash
+    $ BOARD=<board> make flash
 
 Confirm it booted from slot 0 (it should print "Current slot=0"), then
 recompile in order to get an image for the second slot with a newer version

tests/riotboot_flashwrite/main.c

@@ -19,43 +19,111 @@
 
 #include <stdio.h>
 
+#include "thread.h"
+#include "irq.h"
 #include "net/nanocoap_sock.h"
-#include "xtimer.h"
+
+#include "shell.h"
+
 #include "riotboot/slot.h"
+#include "riotboot/flashwrite.h"
 
 #define COAP_INBUF_SIZE (256U)
 
+/* Extend stacksize of nanocoap server thread */
+static char _nanocoap_server_stack[THREAD_STACKSIZE_DEFAULT + THREAD_EXTRA_STACKSIZE_PRINTF];
+#define NANOCOAP_SERVER_QUEUE_SIZE     (8)
+static msg_t _nanocoap_server_msg_queue[NANOCOAP_SERVER_QUEUE_SIZE];
+
 #define MAIN_QUEUE_SIZE     (8)
 static msg_t _main_msg_queue[MAIN_QUEUE_SIZE];
 
-/* import "ifconfig" shell command, used for printing addresses */
+static void *_nanocoap_server_thread(void *arg)
-extern int _gnrc_netif_config(int argc, char **argv);
-
-int main(void)
 {
-    puts("riotboot_flashwrite test application");
+    (void)arg;
-
-    int current_slot = riotboot_slot_current();
-    printf("Current slot=%d\n", current_slot);
-    riotboot_slot_print_hdr(current_slot);
 
     /* nanocoap_server uses gnrc sock which uses gnrc which needs a msg queue */
-    msg_init_queue(_main_msg_queue, MAIN_QUEUE_SIZE);
+    msg_init_queue(_nanocoap_server_msg_queue, NANOCOAP_SERVER_QUEUE_SIZE);
-
-    puts("");
-
-    puts("Waiting for address autoconfiguration...");
-    xtimer_sleep(3);
-
-    /* print network addresses */
-    puts("Configured network interfaces:");
-    _gnrc_netif_config(0, NULL);
 
     /* initialize nanocoap server instance */
     uint8_t buf[COAP_INBUF_SIZE];
     sock_udp_ep_t local = { .port=COAP_PORT, .family=AF_INET6 };
     nanocoap_server(&local, buf, sizeof(buf));
 
-    /* should be never reached */
+    return NULL;
+}
+
+static int cmd_print_riotboot_hdr(int argc, char **argv)
+{
+    (void)argc;
+    (void)argv;
+
+    int current_slot = riotboot_slot_current();
+    if (current_slot != -1) {
+        /* Sometimes, udhcp output messes up the following printfs.  That
+         * confuses the test script. As a workaround, just disable interrupts
+         * for a while.
+         */
+        unsigned state = irq_disable();
+        riotboot_slot_print_hdr(current_slot);
+        irq_restore(state);
+    }
+    else {
+        puts("[FAILED] You're not running riotboot");
+    }
+    return 0;
+}
+
+static int cmd_print_current_slot(int argc, char **argv)
+{
+    (void)argc;
+    (void)argv;
+    /* Sometimes, udhcp output messes up the following printfs.  That
+     * confuses the test script. As a workaround, just disable interrupts
+     * for a while.
+     */
+    unsigned state = irq_disable();
+    printf("Running from slot %d\n", riotboot_slot_current());
+    irq_restore(state);
+    return 0;
+}
+
+static int cmd_riotboot_invalidate(int argc, char **argv)
+{
+    if (argc < 2) {
+        puts("usage: riotboot-invalidate <slot number>");
+    }
+    riotboot_flashwrite_invalidate(atoi(argv[1]));
+    return 0;
+}
+
+static const shell_command_t shell_commands[] = {
+    { "current-slot", "Print current slot number", cmd_print_current_slot },
+    { "riotboot-hdr", "Print current slot header", cmd_print_riotboot_hdr },
+    { "riotboot-invalidate", "Invalidate slot <slot number>", cmd_riotboot_invalidate },
+    { NULL, NULL, NULL }
+};
+
+int main(void)
+{
+    puts("riotboot_flashwrite test application");
+
+    cmd_print_current_slot(0, NULL);
+    cmd_print_riotboot_hdr(0, NULL);
+
+    /* start nanocoap server thread */
+    thread_create(_nanocoap_server_stack, sizeof(_nanocoap_server_stack),
+                  THREAD_PRIORITY_MAIN - 1,
+                  THREAD_CREATE_STACKTEST,
+                  _nanocoap_server_thread, NULL, "nanocoap server");
+
+    /* the shell contains commands that receive packets via GNRC and thus
+       needs a msg queue */
+    msg_init_queue(_main_msg_queue, MAIN_QUEUE_SIZE);
+
+    puts("Starting the shell");
+    char line_buf[SHELL_DEFAULT_BUFSIZE];
+    shell_run(shell_commands, line_buf, SHELL_DEFAULT_BUFSIZE);
+
     return 0;
 }

tests/riotboot_flashwrite/tests-with-config/01-run.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2019 Inria
+#
+# This file is subject to the terms and conditions of the GNU Lesser
+# General Public License v2.1. See the file LICENSE in the top level
+# directory for more details.
+
+import os
+import subprocess
+import sys
+import time
+
+from testrunner import run
+from testrunner import utils
+
+
+UPDATING_TIMEOUT = 10
+
+USE_ETHOS = int(os.getenv("USE_ETHOS", "1"))
+TAP = os.getenv("TAP", "riot0")
+BINDIR = os.getenv("BINDIR")
+
+
+def wait_for_update(child):
+    return child.expect([r"_flashwrite_handler\(\): received data: offset=\d+ "
+                         r"len=\d+ blockwise=\d+ more=\d+\r\n",
+                         r"_flashwrite_handler\(\): finish\r\n"],
+                        timeout=UPDATING_TIMEOUT)
+
+
+def make_notify(client_url, slot, version):
+    cmd = [
+        "aiocoap-client",
+        "-m",
+        "POST",
+        "coap://{}/flashwrite".format(client_url),
+        "--payload",
+        "@tests_riotboot_flashwrite-slot{}.{}.riot.bin".format(slot, version),
+        "--payload-initial-szx",
+        "2",
+    ]
+    return subprocess.Popen(cmd, cwd=BINDIR)
+
+
+def make_riotboot_slots(version):
+    cmd = [
+        "make",
+        "USE_ETHOS={}".format(USE_ETHOS),
+        "RIOTBOOT_SKIP_COMPILE=1",
+        "APP_VER={}".format(version),
+        "riotboot",
+    ]
+    assert not subprocess.call(cmd)
+
+
+def get_ipv6_addr(child):
+    child.expect_exact('>')
+    child.sendline('ifconfig')
+    if USE_ETHOS == 0:
+        # Get device global address
+        child.expect(
+            r"inet6 addr: (?P<gladdr>[0-9a-fA-F:]+:[A-Fa-f:0-9]+)"
+            "  scope: global  VAL"
+        )
+        addr = child.match.group("gladdr").lower()
+    else:
+        # Get device local address
+        child.expect_exact("Link type: wired")
+        child.expect(
+            r"inet6 addr: (?P<lladdr>[0-9a-fA-F:]+:[A-Fa-f:0-9]+)"
+            "  scope: link  VAL"
+        )
+        addr = "{}%{}".format(child.match.group("lladdr").lower(), TAP)
+    return addr
+
+
+def ping6(client):
+    print("pinging node...")
+    ping_ok = False
+    for _ in range(10):
+        try:
+            subprocess.check_call(["ping", "-q", "-c1", "-w1", client])
+            ping_ok = True
+            break
+        except subprocess.CalledProcessError:
+            pass
+
+    if not ping_ok:
+        print("pinging node failed. aborting test.")
+        sys.exit(1)
+    else:
+        print("pinging node succeeded.")
+    return ping_ok
+
+
+def get_reachable_addr(child):
+    # Give some time for the network interface to be configured
+    time.sleep(1)
+    # Get address
+    client_addr = get_ipv6_addr(child)
+    # Verify address is reachable
+    ping6(client_addr)
+    return "[{}]".format(client_addr)
+
+
+def app_version(child):
+    utils.test_utils_interactive_sync_shell(child, 5, 1)
+    # get version of currently running image
+    # "Image Version: 0x00000000"
+    child.sendline('riotboot-hdr')
+    child.expect(r"Image Version: (?P<app_ver>0x[0-9a-fA-F:]+)\r\n")
+    app_ver = int(child.match.group("app_ver"), 16)
+    return app_ver
+
+
+def running_slot(child):
+    utils.test_utils_interactive_sync_shell(child, 5, 1)
+    # get version of currently running image
+    # "Image Version: 0x00000000"
+    child.sendline('current-slot')
+    child.expect(r"Running from slot (\d+)\r\n")
+    slot = int(child.match.group(1))
+    return slot
+
+
+def testfunc(child):
+    # Get current app_ver and slot
+    current_app_ver = app_version(child)
+    current_slot = running_slot(child)
+    # Verify client is reachable and get address
+    client = get_reachable_addr(child)
+
+    for version in [current_app_ver + 1, current_app_ver + 2]:
+        # Create newer slots bins
+        make_riotboot_slots(version)
+        # Trigger update process
+        make_notify(client, current_slot ^ 1, version)
+        child.expect(
+            r"riotboot_flashwrite: initializing update to target slot (\d+)\r\n",
+        )
+        target_slot = int(child.match.group(1))
+        # Wait for update to complete
+        while wait_for_update(child) == 0:
+            pass
+        child.sendline('reboot')
+        child.expect_exact("Starting the shell")
+        # Verify running slot
+        current_slot = running_slot(child)
+        assert target_slot == current_slot, "BOOTED FROM SAME SLOT"
+        # Verify client is reachable and get address
+        client = get_reachable_addr(child)
+
+    child.sendline("riotboot-invalidate {}".format(current_slot))
+    child.sendline('reboot')
+    child.expect_exact("Starting the shell")
+    assert running_slot(child) != current_slot, "DID NOT INVALIDATE"
+
+    print("TEST PASSED")
+
+
+if __name__ == "__main__":
+    sys.exit(run(testfunc, echo=True))