Merge pull request #20024 from mguetschow/makefile-subfolders

Build system: support for application subfolders

Makefile.base

@@ -104,20 +104,30 @@ ASSMOBJ     := $(ASSMSRC:%.S=$(BINDIR)/$(MODULE)/%.o)
 OBJ := $(OBJC) $(OBJCXX) $(ASMOBJ) $(ASSMOBJ) $(GENOBJC)
 DEP := $(OBJC:.o=.d) $(OBJCXX:.o=.d) $(ASSMOBJ:.o=.d)
 
+SRC_ALL := $(SRC) $(SRCXX) $(ASMSRC) $(ASSMSRC)
+SUBDIRS_IN_DIRS := $(filter $(DIRS), $(abspath $(sort $(dir $(SRC_ALL)))))
+ifneq (,$(SUBDIRS_IN_DIRS))
+  $(warning Files of the following subdirectories are selected \
+    both as RIOT modules (using DIRS) and directly as sourcefiles (using SRC): \
+    $(patsubst $(CURDIR)/%,./%, $(SUBDIRS_IN_DIRS)). \
+    Please select a single approach for each subfolder to prevent linking errors.)
+endif
+SUBDIRS := $(filter-out $(BINDIR)/$(MODULE)/, $(dir $(OBJ)))
+
 include $(RIOTMAKE)/blob.inc.mk
 include $(RIOTMAKE)/tools/fixdep.inc.mk
 
-$(BINDIR)/$(MODULE)/:
+$(BINDIR)/$(MODULE)/ $(SUBDIRS):
 	$(Q)mkdir -p $@
 
-OLD_OBJECTS = $(wildcard $(BINDIR)/$(MODULE)/*.o)
+OLD_OBJECTS = $(wildcard $(BINDIR)/$(MODULE)/*.o $(BINDIR)/$(MODULE)/**/*.o)
 
 # do not clean objects from bindist modules
 ifeq (,$(filter $(MODULE),$(BIN_USEMODULE)))
   OBJECTS_TO_REMOVE = $(filter-out $(OBJ),$(OLD_OBJECTS))
 endif
 
-$(MODULE).module compile-commands $(OBJ): | $(BINDIR)/$(MODULE)/
+$(MODULE).module compile-commands $(OBJ): | $(BINDIR)/$(MODULE)/ $(SUBDIRS)
 
 $(MODULE).module: $(OBJ) $(if $(OBJECTS_TO_REMOVE),$(MODULE).cleanup) | $(DIRS:%=ALL--%)
 

Makefile.include

@@ -720,6 +720,7 @@ COMPILE_COMMANDS_FLAGS ?= --clangd
 compile-commands: $(COMPILE_COMMANDS_PATH)
 %/compile_commands.json: $(BUILDDEPS)
 	$(Q)DIRS="$(DIRS)" APPLICATION_BLOBS="$(BLOBS)" \
+    APPLICATION_SRC="$(SRC)" APPLICATION_SRCXX="$(SRCXX)" APPLICATION_ASMSRC="$(ASMSRC)" APPLICATION_ASSMSRC="$(ASSMSRC)" \
 	  "$(MAKE)" -C $(APPDIR) -f $(RIOTMAKE)/application.inc.mk compile-commands
 	$(Q)$(RIOTTOOLS)/compile_commands/compile_commands.py $(COMPILE_COMMANDS_FLAGS) $(BINDIR) \
 	  > $@
@@ -744,6 +745,7 @@ $(ELFFILE): $(BASELIBS) $(ARCHIVES) $(LD_SCRIPTS)
 
 $(APPLICATION_MODULE).module: pkg-build $(BUILDDEPS)
 	$(Q)DIRS="$(DIRS)" APPLICATION_BLOBS="$(BLOBS)" \
+    APPLICATION_SRC="$(SRC)" APPLICATION_SRCXX="$(SRCXX)" APPLICATION_ASMSRC="$(ASMSRC)" APPLICATION_ASSMSRC="$(ASSMSRC)" \
 	  "$(MAKE)" -C $(APPDIR) -f $(RIOTMAKE)/application.inc.mk
 $(APPLICATION_MODULE).module: FORCE
 

doc/doxygen/src/creating-an-application.md

@@ -92,6 +92,21 @@ USEMODULE += gnrc_udp
 
 Modules typically pull in all required dependencies.
 
+## Including source files in subfolders
+
+By default, all source files in an application's (or any RIOT module's) directory
+are automatically compiled as part of the application. In order to organize source
+code in a directory structure, two different approaches can be used:
+
+1. Make each subdirectory a separate RIOT module with a unique name inside its
+Makefile, either by adding the directory's path to `DIRS` or with the [out-of-tree
+module support](#external-modules).
+2. Add the source files within subdirectories to `SRC`, either explicitly or with
+Makefile wildcards.
+
+Both approaches are illustrated and explained in `examples/subfolders`.
+
+
 # Helper tools
 
 To help you start writing an application within RIOT, the build system provides
@@ -210,7 +225,7 @@ configuration (e.g. configuring some of the pins configured as ADC as
 additional PWM outputs instead) a copy of the upstream board that is then
 customized to the application needs is the best course of action.
 
-## External Modules
+## External Modules                                         {#external-modules}
 
 Similar to the external boards, external modules can be written in a similar way
 as regular in-tree modules.

examples/subfolders/Makefile

@@ -0,0 +1,33 @@
+# name of your application
+APPLICATION = subfolders
+
+# If no BOARD is found in the environment, use this default:
+BOARD ?= native
+
+# This has to be the absolute path to the RIOT base directory:
+RIOTBASE ?= $(CURDIR)/../..
+
+# Add subfolders as modules
+DIRS += module
+USEMODULE += my_module # name as defined in module/Makefile
+
+# Add source files in subfolders manually
+SRC += main.c
+SRC += folder/a.c folder/subfolder/b.c
+
+# Alternative method to add files in subfolders using wildcards
+# SRC += $(wildcard *.c folder/*.c folder/**/*.c)
+
+# Adding subfolders both via SRC and DIRS will generate a warning
+# and likely fail during linking
+# DIRS += folder
+
+# Comment this out to disable code in RIOT that does safety checking
+# which is not needed in a production environment but helps in the
+# development process:
+DEVELHELP ?= 1
+
+# Change this to 0 show compiler invocation lines by default:
+QUIET ?= 1
+
+include $(RIOTBASE)/Makefile.include

examples/subfolders/README.md

@@ -0,0 +1,86 @@
+# Application Example with Subfolders
+
+This example demonstrates the usage of subfolders in a RIOT application
+(or in a RIOT module in general) show-casing two possible approaches: RIOT
+modules and simple subfolders.
+
+## Details
+
+Consider the following folder structure of this example.
+The source files in `module` are incorporated as a RIOT module,
+while the source files in `folder` are considered part of the application itself.
+
+```
+.
+├── folder
+│   ├── a.c
+│   └── subfolder
+│       └── b.c
+├── main.c
+├── Makefile
+├── module
+│   ├── a.c
+│   ├── b.c
+│   └── Makefile
+└── README.md
+```
+
+### RIOT modules
+
+At a minimum, each module in RIOT requires a `Makefile` with the following content:
+
+```Makefile
+MODULE := my_module
+
+include $(RIOTBASE)/Makefile.base
+```
+
+If `MODULE` is not specified, the name of the module's directory is automatically used,
+leaving only the last line as minimal content.
+It is important to note that module names have to be unique both among _all_ RIOT modules,
+i.e., including the modules that are part of RIOT itself.
+
+If not manually specified via `SRC`, all source files which reside
+directly in the module's directory are considered part of the module.
+RIOT modules are also described in greater detail [in the documentation](https://doc.riot-os.org/creating-modules.html).
+
+Two lines need to be added to the application's Makefile in order to compile and use the module:
+
+```Makefile
+DIRS += module
+USEMODULE += my_module
+```
+
+The string added to `DIRS` has to match the directory name,
+while the string added to `USEMODULE` has to match the module's name as defined above.
+
+
+### Subfolders
+
+Compared to the module approach, no additional Makefile is needed in the subfolder.
+The application's Makefile needs to add _all_ source files explicitly,
+including the ones residing directly in the application directory:
+
+```Makefile
+SRC += main.c
+SRC += folder/a.c folder/subfolder/b.c
+```
+
+To avoid listing all source files individually, it is of course possible
+to use normal GNU make functions such as `wildcard`:
+
+```Makefile
+SRC += $(wildcard *.c folder/*.c folder/**/*.c)
+```
+
+
+## Which approach should I use?
+
+In general, modules in RIOT are well-defined units of code that provide a set of features to your application.
+If this matches your use-case, i.e., you have all your application tests separated into a subfolder,
+RIOT modules are probably the best approach.
+It is good practice to prefix all your application modules to avoid name clashes.
+
+If however you barely want to organize your files in a sensible folder structure,
+but always require all source files to be part of your application,
+the more straight-forward subfolder approach is probably the better pick.

examples/subfolders/folder/a.c

@@ -0,0 +1,14 @@
+/*
+ * Copyright (C) 2023 TU Dresden
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+#include <stdio.h>
+
+void folder_a(void)
+{
+    puts("./folder/a.c");
+}

examples/subfolders/folder/subfolder/b.c

@@ -0,0 +1,14 @@
+/*
+ * Copyright (C) 2023 TU Dresden
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+#include <stdio.h>
+
+void folder_b(void)
+{
+    puts("./folder/subfolder/b.c");
+}

examples/subfolders/main.c

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2023 TU Dresden
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @ingroup     examples
+ * @{
+ *
+ * @file
+ * @brief       Application showcasing the use of subfolders in RIOT applications
+ *
+ * @author      Mikolai Gütschow <mikolai.guetschow@tu-dresden.de>
+ *
+ * @}
+ */
+
+#include <stdio.h>
+
+void module_a(void);
+void module_b(void);
+void folder_a(void);
+void folder_b(void);
+
+int main(void)
+{
+    puts("./main.c");
+    // call functions from RIOT module
+    module_a();
+    module_b();
+    // call functions from subfolder
+    folder_a();
+    folder_b();
+
+    return 0;
+}

examples/subfolders/module/Makefile

@@ -0,0 +1,3 @@
+MODULE := my_module
+
+include $(RIOTBASE)/Makefile.base

examples/subfolders/module/a.c

@@ -0,0 +1,14 @@
+/*
+ * Copyright (C) 2023 TU Dresden
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+#include <stdio.h>
+
+void module_a(void)
+{
+    puts("./module/a.c");
+}

examples/subfolders/module/b.c

@@ -0,0 +1,14 @@
+/*
+ * Copyright (C) 2023 TU Dresden
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+#include <stdio.h>
+
+void module_b(void)
+{
+    puts("./module/b.c");
+}

makefiles/application.inc.mk

@@ -3,17 +3,20 @@ MODULE = $(APPLICATION_MODULE)
 DIRS += $(RIOTCPU)/$(CPU) $(BOARDDIR)
 DIRS += $(RIOTBASE)/core $(RIOTBASE)/core/lib $(RIOTBASE)/drivers $(RIOTBASE)/sys
 
-# For regular modules, adding files to BLOBS to their Makefile is sufficient to
-# create the corresponding headers.
+# For regular modules, adding files to BLOBS, SRC, SRCXX, ASMSRC or ASSMSRC
+# in their Makefile is sufficient to explicitely set the variables.
 #
 # Application modules are different, as they use this makefile to build, thus
 # application level variables are not available unless exported.
 #
-# But exporting e.g., BLOBS, would pre-set the variable for all
-# submakefiles.
+# But exporting would pre-set the variables for all submakefiles.
 #
-# As workaround, $(RIOTBASE)/Makefile.include passes BLOBS to this
-# Makefile as APPLICATION_BLOBS.
-BLOBS = $(APPLICATION_BLOBS)
+# As workaround, $(RIOTBASE)/Makefile.include passes the above-listed variables
+# to this Makefile as APPLICATION_*.
+BLOBS   = $(APPLICATION_BLOBS)
+SRC     = $(APPLICATION_SRC)
+SRCXX   = $(APPLICATION_SRCXX)
+ASMSRC  = $(APPLICATION_ASMSRC)
+ASSMSRC = $(APPLICATION_ASSMSRC)
 
 include $(RIOTBASE)/Makefile.base