Merge #19315

19315: cpu/native: add host fs access via VFS r=benpicco a=benpicco



Co-authored-by: Benjamin Valentin <benpicco@beuth-hochschule.de>
Co-authored-by: Benjamin Valentin <benjamin.valentin@ml-pa.com>

boards/native/Makefile.dep

@@ -12,10 +12,10 @@ ifneq (,$(filter periph_can,$(FEATURES_USED)))
   USEPKG += libsocketcan
 endif
 
-# default to using littlefs2 on the virtual flash if no other fs was selected
+# default to host fs pass-through if no other fs was selected
 ifneq (,$(filter vfs_default,$(USEMODULE)))
-  ifeq (,$(filter lwext%_vfs spiffs littlefs fatfs_vfs,$(USEMODULE)))
-    USEMODULE += littlefs2
+  ifeq (,$(filter lwext%_vfs spiffs littlefs% fatfs_vfs,$(USEMODULE)))
+    USEMODULE += fs_native
   endif
   USEMODULE += mtd
 endif

boards/native/board_init.c

@@ -66,6 +66,10 @@ VFS_AUTO_MOUNT(fatfs, VFS_MTD(mtd0_dev), VFS_DEFAULT_NVM(0), 0);
 #elif defined(MODULE_LWEXT4)
 VFS_AUTO_MOUNT(lwext4, VFS_MTD(mtd0_dev), VFS_DEFAULT_NVM(0), 0);
 
+/* host fs pass-through */
+#elif defined(MODULE_FS_NATIVE)
+VFS_AUTO_MOUNT(native, { .hostpath = FS_NATIVE_DIR }, VFS_DEFAULT_NVM(0), 0);
+
 #endif
 #endif /* MODULE_VFS_DEFAULT */
 

boards/native/include/board.h

@@ -97,6 +97,15 @@ void _native_LED_RED_TOGGLE(void);
 extern mtd_dev_t *mtd0;
 #endif
 
+/**
+ * @name    Host FS access configuration
+ * @{
+ */
+#ifndef FS_NATIVE_DIR
+#define FS_NATIVE_DIR           "native"  /**< Folder on the host fs exported to RIOT */
+#endif
+/** @} */
+
 #if defined(MODULE_SPIFFS) || DOXYGEN
 /**
  * @name    SPIFFS default configuration

cpu/native/Makefile

@@ -22,6 +22,10 @@ ifneq (,$(filter mtd_native,$(USEMODULE)))
   DIRS += mtd
 endif
 
+ifneq (,$(filter fs_native,$(USEMODULE)))
+  DIRS += fs
+endif
+
 ifneq (,$(filter backtrace,$(USEMODULE)))
   DIRS += backtrace
 endif

cpu/native/fs/Makefile

@@ -0,0 +1,3 @@
+MODULE := fs_native
+
+include $(RIOTBASE)/Makefile.base

cpu/native/fs/native_fs.c

@@ -0,0 +1,291 @@
+/*
+ * Copyright (C) 2023 ML!PA Consulting GmbH
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @ingroup     sys_fs_native
+ * @{
+ *
+ * @file
+ * @brief       native integration with vfs
+ *
+ * @author      Benjamin Valentin <benjamin.valentin@ml-pa.com>
+ *
+ * @}
+ */
+
+#include <assert.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "mutex.h"
+#include "native_internal.h"
+#include "fs/native_fs.h"
+#include "vfs.h"
+
+#define ENABLE_DEBUG 0
+#include <debug.h>
+
+/**
+ * @brief Assign each native instance a sub-sirectory based on `_native_id`
+ */
+#ifndef CONFIG_NATIVE_ISOLATE_FS
+#define CONFIG_NATIVE_ISOLATE_FS 0
+#endif
+
+/* Not using static inline functions here because they are also assigned to. */
+#define FD(filep) filp->private_data.value
+#define DIRP(dirp) dirp->private_data.ptr
+
+/* Reentrancy guard required by the various static locals of the implementation */
+static mutex_t _lock;
+
+static void _do_prefix(vfs_mount_t *mountp, const char *name, char *buffer, size_t len)
+{
+    const native_desc_t *fs_desc = mountp->private_data;
+    size_t res;
+
+    assert(len > strlen(fs_desc->hostpath));
+
+    if (CONFIG_NATIVE_ISOLATE_FS) {
+        res = snprintf(buffer, len, "%s/%u%s", fs_desc->hostpath, _native_id, name);
+    } else {
+        res = snprintf(buffer, len, "%s%s", fs_desc->hostpath, name);
+    }
+
+#ifdef NDEBUG
+    if (res > len) {
+        puts("fs_native: path larger than PATH_MAX");
+        exit(ENOBUFS);
+    }
+#else
+    assert(res <= len);
+#endif
+}
+
+static char *_prefix_path(vfs_mount_t *mountp, const char *name)
+{
+    static char buffer[PATH_MAX];
+
+    _do_prefix(mountp, name, buffer, sizeof(buffer));
+
+    return buffer;
+}
+
+static int _mount(vfs_mount_t *mountp)
+{
+    int res;
+    mutex_lock(&_lock);
+
+    /* create common root dir first */
+    if (CONFIG_NATIVE_ISOLATE_FS) {
+        char *parent = _prefix_path(mountp, "");
+        /* remove node specific suffix */
+        char *end = strrchr(parent, '/');
+        *end = '\0';
+
+        /* Ignoring errors: they're caught by the subsequent real_mkdir */
+        real_mkdir(parent, 0777);
+    }
+
+    real_mkdir(_prefix_path(mountp, ""), 0777);
+    res = errno == EEXIST ? 0 : -errno;
+
+    mutex_unlock(&_lock);
+    return res;
+}
+
+static int _unmount(vfs_mount_t *mountp)
+{
+    mutex_lock(&_lock);
+
+    /* Ignoring errors: directories with any content are left in place. */
+    real_rmdir(_prefix_path(mountp, ""));
+
+    mutex_unlock(&_lock);
+    return 0;
+}
+
+static int _mkdir(vfs_mount_t *mountp, const char *name, mode_t mode)
+{
+    int res = 0;
+    mutex_lock(&_lock);
+
+    if (real_mkdir(_prefix_path(mountp, name), mode) < 0) {
+        res = -errno;
+    }
+
+    mutex_unlock(&_lock);
+    return res;
+}
+
+static int _rmdir(vfs_mount_t *mountp, const char *name)
+{
+    int res = 0;
+    mutex_lock(&_lock);
+
+    if (real_rmdir(_prefix_path(mountp, name)) < 0) {
+        res = -errno;
+    }
+
+    mutex_unlock(&_lock);
+    return res;
+}
+
+static int _statvfs(vfs_mount_t *mountp, const char *restrict path, struct statvfs *restrict buf)
+{
+    int res = 0;
+    mutex_lock(&_lock);
+
+    if (real_statvfs(_prefix_path(mountp, path), buf) < 0) {
+        res = -errno;
+    }
+
+    mutex_unlock(&_lock);
+    return res;
+}
+
+static int _open(vfs_file_t *filp, const char *name, int flags, mode_t mode)
+{
+    int res = 0;
+    mutex_lock(&_lock);
+
+    if ((FD(filep) = real_open(_prefix_path(filp->mp, name), flags, mode)) < 0) {
+        res = -errno;
+    }
+
+    mutex_unlock(&_lock);
+    return res;
+}
+
+static ssize_t _read(vfs_file_t *filp, void *dest, size_t nbytes)
+{
+    return real_read(FD(filep), dest, nbytes);
+}
+
+static ssize_t _write(vfs_file_t *filp, const void *src, size_t nbytes)
+{
+    return real_write(FD(filep), src, nbytes);
+}
+
+static off_t _lseek(vfs_file_t *filp, off_t off, int whence)
+{
+    return real_lseek(FD(filep), off, whence);
+}
+
+static int _fstat(vfs_file_t *filp, struct stat *buf)
+{
+    return real_fstat(FD(filep), buf);
+}
+
+static int _fsync(vfs_file_t *filp)
+{
+    return real_fsync(FD(filep));
+}
+
+static int _close(vfs_file_t *filp)
+{
+    return real_close(FD(filep));
+}
+
+static int _unlink(vfs_mount_t *mountp, const char *name)
+{
+    int res = 0;
+    mutex_lock(&_lock);
+
+    if (real_unlink(_prefix_path(mountp, name)) < 0) {
+        res = -errno;
+    }
+
+    mutex_unlock(&_lock);
+    return res;
+}
+
+static int _rename(vfs_mount_t *mountp, const char *from_path, const char *to_path)
+{
+    static char path_a[PATH_MAX];
+    static char path_b[PATH_MAX];
+
+    int res = 0;
+    mutex_lock(&_lock);
+
+    _do_prefix(mountp, from_path, path_a, sizeof(path_a));
+    _do_prefix(mountp, to_path, path_b, sizeof(path_b));
+
+    if (real_rename(path_a, path_b) < 0) {
+        res = -errno;
+    }
+
+    mutex_unlock(&_lock);
+    return res;
+}
+
+static int _opendir(vfs_DIR *dirp, const char *dirname)
+{
+    int res = 0;
+    mutex_lock(&_lock);
+
+    if ((DIRP(dirp) = real_opendir(_prefix_path(dirp->mp, dirname))) == NULL) {
+        res = -errno;
+    }
+
+    mutex_unlock(&_lock);
+    return res;
+}
+
+static int _readdir(vfs_DIR *dirp, vfs_dirent_t *entry)
+{
+    struct dirent *dirent = real_readdir(DIRP(dirp));
+
+    if (dirent == NULL) {
+        return 0;
+    }
+
+    entry->d_ino = dirent->d_ino;
+    strncpy(entry->d_name, (char *)dirent->d_name, sizeof(entry->d_name));
+
+    return 1;
+}
+
+static int _closedir(vfs_DIR *dirp)
+{
+    return real_closedir(DIRP(dirp));
+}
+
+static const vfs_file_system_ops_t native_fs_ops = {
+    .mount = _mount,
+    .umount = _unmount,
+    .unlink = _unlink,
+    .mkdir = _mkdir,
+    .rmdir = _rmdir,
+    .rename = _rename,
+    .stat = vfs_sysop_stat_from_fstat,
+    .statvfs = _statvfs,
+};
+
+static const vfs_file_ops_t native_file_ops = {
+    .open = _open,
+    .close = _close,
+    .read = _read,
+    .write = _write,
+    .lseek = _lseek,
+    .fstat = _fstat,
+    .fsync = _fsync,
+};
+
+static const vfs_dir_ops_t native_dir_ops = {
+    .opendir = _opendir,
+    .readdir = _readdir,
+    .closedir = _closedir,
+};
+
+const vfs_file_system_t native_file_system = {
+    .fs_op = &native_fs_ops,
+    .f_op = &native_file_ops,
+    .d_op = &native_dir_ops,
+};

cpu/native/include/native_internal.h

@@ -59,7 +59,9 @@
 #include <time.h>
 #include <sys/time.h>
 #include <sys/stat.h>
+#include <sys/statvfs.h>
 #include <sys/uio.h>
+#include <dirent.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -90,6 +92,10 @@ void _native_init_syscalls(void);
  */
 extern ssize_t (*real_read)(int fd, void *buf, size_t count);
 extern ssize_t (*real_write)(int fd, const void *buf, size_t count);
+extern off_t (*real_lseek)(int fd, off_t offset, int whence);
+extern off_t (*real_fstat)(int fd, struct stat *statbuf);
+extern int (*real_statvfs)(const char *restrict path, struct statvfs *restrict buf);
+extern int (*real_fsync)(int fd);
 extern size_t (*real_fread)(void *ptr, size_t size, size_t nmemb, FILE *stream);
 extern void (*real_clearerr)(FILE *stream);
 extern __attribute__((noreturn)) void (*real_exit)(int status);
@@ -124,6 +130,11 @@ extern int (*real_gettimeofday)(struct timeval *t, ...);
 extern int (*real_ioctl)(int fildes, int request, ...);
 extern int (*real_listen)(int socket, int backlog);
 extern int (*real_open)(const char *path, int oflag, ...);
+extern int (*real_mkdir)(const char *pathname, mode_t mode);
+extern int (*real_rmdir)(const char *pathname);
+extern DIR *(*real_opendir)(const char *name);
+extern struct dirent *(*real_readdir)(DIR *dirp);
+extern int (*real_closedir)(DIR *dirp);
 extern int (*real_pause)(void);
 extern int (*real_pipe)(int[2]);
 /* The ... is a hack to save includes: */
@@ -136,6 +147,7 @@ extern int (*real_setsockopt)(int socket, ...);
 extern int (*real_socket)(int domain, int type, int protocol);
 extern int (*real_printf)(const char *format, ...);
 extern int (*real_unlink)(const char *);
+extern int (*real_rename)(const char *, const char *);
 extern long int (*real_random)(void);
 extern const char* (*real_gai_strerror)(int errcode);
 extern FILE* (*real_fopen)(const char *path, const char *mode);

cpu/native/periph/pm.c

@@ -70,6 +70,10 @@ void pm_off(void)
 #endif
 #ifdef MODULE_PERIPH_GPIO_LINUX
     gpio_linux_teardown();
+#endif
+#ifdef MODULE_VFS_DEFAULT
+    extern void auto_unmount_vfs(void);
+    auto_unmount_vfs();
 #endif
     real_exit(EXIT_SUCCESS);
 }
@@ -85,6 +89,10 @@ void pm_reboot(void)
 #ifdef MODULE_PERIPH_GPIO_LINUX
     gpio_linux_teardown();
 #endif
+#ifdef MODULE_VFS_DEFAULT
+    extern void auto_unmount_vfs(void);
+    auto_unmount_vfs();
+#endif
 
     if (real_execve(_native_argv[0], _native_argv, NULL) == -1) {
         err(EXIT_FAILURE, "reboot: execve");

cpu/native/syscalls.c

@@ -105,6 +105,16 @@ int (*real_fgetc)(FILE *stream);
 mode_t (*real_umask)(mode_t cmask);
 ssize_t (*real_writev)(int fildes, const struct iovec *iov, int iovcnt);
 ssize_t (*real_send)(int sockfd, const void *buf, size_t len, int flags);
+off_t (*real_lseek)(int fd, off_t offset, int whence);
+off_t (*real_fstat)(int fd, struct stat *statbuf);
+int (*real_fsync)(int fd);
+int (*real_mkdir)(const char *pathname, mode_t mode);
+int (*real_rmdir)(const char *pathname);
+DIR *(*real_opendir)(const char *name);
+struct dirent *(*real_readdir)(DIR *dirp);
+int (*real_closedir)(DIR *dirp);
+int (*real_rename)(const char *, const char *);
+int (*real_statvfs)(const char *restrict path, struct statvfs *restrict buf);
 
 void _native_syscall_enter(void)
 {
@@ -339,8 +349,9 @@ char *make_message(const char *format, va_list argp)
             free(message);
             return NULL;
         }
-        if (n < size)
+        if (n < size) {
             return message;
+        }
         size = n + 1;
         if ((temp = realloc(message, size)) == NULL) {
             free(message);
@@ -548,4 +559,14 @@ void _native_init_syscalls(void)
     *(void **)(&real_ftell) = dlsym(RTLD_NEXT, "ftell");
     *(void **)(&real_fputc) = dlsym(RTLD_NEXT, "fputc");
     *(void **)(&real_fgetc) = dlsym(RTLD_NEXT, "fgetc");
+    *(void **)(&real_mkdir) = dlsym(RTLD_NEXT, "mkdir");
+    *(void **)(&real_rmdir) = dlsym(RTLD_NEXT, "rmdir");
+    *(void **)(&real_lseek) = dlsym(RTLD_NEXT, "lseek");
+    *(void **)(&real_fstat) = dlsym(RTLD_NEXT, "fstat");
+    *(void **)(&real_fsync) = dlsym(RTLD_NEXT, "fsync");
+    *(void **)(&real_rename) = dlsym(RTLD_NEXT, "rename");
+    *(void **)(&real_opendir) = dlsym(RTLD_NEXT, "opendir");
+    *(void **)(&real_readdir) = dlsym(RTLD_NEXT, "readdir");
+    *(void **)(&real_closedir) = dlsym(RTLD_NEXT, "closedir");
+    *(void **)(&real_statvfs) = dlsym(RTLD_NEXT, "statvfs");
 }

sys/include/fs/native_fs.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2023 ML!PA Consulting GmbH
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @defgroup    sys_fs_native native fs integration
+ * @ingroup     cpu_native
+ * @brief       Access to the host fs from RIOT native
+ *
+ * @{
+ *
+ * @file
+ * @brief       native integration with vfs
+ *
+ * @author      Benjamin Valentin <benjamin.valentin@ml-pa.com>
+ */
+
+#ifndef FS_NATIVE_FS_H
+#define FS_NATIVE_FS_H
+
+#include <stdalign.h>
+
+#include "vfs.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief   native filesystem access
+ */
+typedef struct {
+    const char *hostpath;   /**< base directory, with no trailing slash */
+} native_desc_t;
+
+/**
+ * @brief   The native vfs driver
+ */
+extern const vfs_file_system_t native_file_system;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* FS_NATIVE_FS_H */
+/** @} */

sys/include/vfs_default.h

@@ -44,6 +44,9 @@
 #if IS_USED(MODULE_LWEXT4)
 #include "fs/lwext4_fs.h"
 #endif
+#if IS_USED(MODULE_FS_NATIVE)
+#include "fs/native_fs.h"
+#endif
 
 #ifdef __cplusplus
 extern "C" {

sys/net/application_layer/gcoap/fileserver.c

@@ -312,7 +312,7 @@ static ssize_t _put_file(coap_pkt_t *pdu, uint8_t *buf, size_t len,
     }
     ret = O_WRONLY;
     ret |= (create ? O_CREAT | O_APPEND : 0);
-    if ((fd = vfs_open(request->namebuf, ret, 0)) < 0) {
+    if ((fd = vfs_open(request->namebuf, ret, 0666)) < 0) {
         ret = fd;
         goto unlink_on_error;
     }
@@ -511,7 +511,7 @@ static ssize_t _put_directory(coap_pkt_t *pdu, uint8_t *buf, size_t len,
         if (request->options.exists.if_match) {
             return gcoap_fileserver_error_handler(pdu, buf, len, COAP_CODE_PRECONDITION_FAILED);
         }
-        if ((err = vfs_mkdir(request->namebuf, 0)) < 0) {
+        if ((err = vfs_mkdir(request->namebuf, 0777)) < 0) {
             return gcoap_fileserver_error_handler(pdu, buf, len, err);
         }
         gcoap_resp_init(pdu, buf, len, COAP_CODE_CREATED);

sys/vfs/vfs.c

@@ -1193,6 +1193,13 @@ void auto_init_vfs(void)
     }
 }
 
+void auto_unmount_vfs(void)
+{
+    for (unsigned i = 0; i < MOUNTPOINTS_NUMOF; ++i) {
+        vfs_umount(&vfs_mountpoints_xfa[i], true);
+    }
+}
+
 int vfs_mount_by_path(const char *path)
 {
     for (unsigned i = 0; i < MOUNTPOINTS_NUMOF; ++i) {

tests/gcoap_fileserver/Makefile

@@ -34,6 +34,8 @@ ifeq (native, $(BOARD))
   USEMODULE += socket_zep_hello
   USEMODULE += netdev
   TERMFLAGS += -z 127.0.0.1:17754 # Murdock has no IPv6 support
+  # make sure each instance gets their own fs
+  CFLAGS += -DCONFIG_NATIVE_ISOLATE_FS=1
 else
   USEMODULE += netdev_default
 endif

tests/gcoap_fileserver/tests/01-run.py

@@ -110,6 +110,10 @@ def test_linear_topology(factory, zep_dispatch):
     # make sure the content matches
     assert B.cmd("md5sum /nvm0/song.txt").split()[2] == B.cmd("md5sum /nvm0/song2.txt").split()[2]
 
+    # clean up after run
+    B.cmd("vfs rm /nvm0/song.txt")
+    B.cmd("vfs rm /nvm0/song2.txt")
+
     # terminate nodes
     for n in nodes:
         n.stop_term()