Merge pull request #17211 from NikLeberg/feature/external_pkg_dirs

buildsystem: add EXTERNAL_PKG_DIRS functionality

Kconfig

@@ -34,9 +34,13 @@ rsource "sys/Kconfig"
 rsource "pkg/Kconfig"
 
 menu "External Modules"
-osource "$(KCONFIG_EXTERNAL_CONFIGS)"
+osource "$(KCONFIG_EXTERNAL_MODULE_CONFIGS)"
 endmenu # External Modules
 
+menu "External Packages"
+osource "$(KCONFIG_EXTERNAL_PKG_CONFIGS)"
+endmenu # External Packages
+
 comment "RIOT is in a migration phase."
 comment "Some configuration options may not be here. Use CFLAGS instead."
 

Makefile.include

@@ -49,6 +49,7 @@ EXTERNAL_BOARD_DIRS             ?=
 RIOTMAKE                        ?= $(RIOTBASE)/makefiles
 RIOTKCONFIG                     ?= $(RIOTBASE)/kconfigs
 RIOTPKG                         ?= $(RIOTBASE)/pkg
+EXTERNAL_PKG_DIRS               ?=
 RIOTTOOLS                       ?= $(RIOTBASE)/dist/tools
 RIOTPROJECT                     ?= $(shell git rev-parse --show-toplevel 2>/dev/null || pwd)
 BUILD_DIR                       ?= $(RIOTBASE)/build
@@ -98,6 +99,9 @@ ifeq ($(INSIDE_DOCKER),0)
   ifeq ($(origin EXTERNAL_MODULE_DIRS),command line)
     $(error EXTERNAL_MODULE_DIRS must be passed as environment variable, and not as command line argument)
   endif
+  ifeq ($(origin EXTERNAL_PKG_DIRS),command line)
+    $(error EXTERNAL_PKG_DIRS must be passed as environment variable, and not as command line argument)
+  endif
 endif
 
 # Deprecation of configuring 'RIOTBOARD'
@@ -142,6 +146,9 @@ EXTERNAL_BOARD_DIRS             := $(foreach dir,\
 EXTERNAL_MODULE_DIRS            := $(foreach dir,\
                                              $(EXTERNAL_MODULE_DIRS),\
                                              $(abspath $(dir)))
+EXTERNAL_PKG_DIRS               := $(foreach dir,\
+                                             $(EXTERNAL_PKG_DIRS),\
+                                             $(abspath $(dir)))
 
 # Ensure that all directories are set and don't contain spaces.
 ifneq (, $(filter-out 1, $(foreach v,$(__DIRECTORY_VARIABLES),$(words $($(v))))))
@@ -428,8 +435,9 @@ ifeq (1,$(TEST_KCONFIG))
   KCONFIG_PACKAGES := $(call lowercase,$(patsubst CONFIG_PACKAGE_%,%,$(filter CONFIG_PACKAGE_%,$(.VARIABLES))))
   USEPKG := $(KCONFIG_PACKAGES)
 
-  # Locate used packages in $(RIOTPKG).
-  PKG_PATHS := $(sort $(foreach dir,$(RIOTPKG),\
+  # Locate used packages in $(RIOTPKG) or $(EXTERNAL_PKG_DIRS).
+  PKGDIRS := $(RIOTPKG) $(EXTERNAL_PKG_DIRS)
+  PKG_PATHS := $(sort $(foreach dir,$(PKGDIRS),\
     $(foreach pkg,$(USEPKG),$(dir $(wildcard $(dir)/$(pkg)/Makefile)))))
 
   EXTERNAL_MODULE_PATHS := $(dir $(EXTERNAL_MODULE_KCONFIGS))

makefiles/dependency_resolution.inc.mk

@@ -6,8 +6,9 @@
 EXTERNAL_MODULE_PATHS := $(sort $(foreach dir,$(EXTERNAL_MODULE_DIRS),\
   $(foreach mod,$(USEMODULE),$(dir $(wildcard $(dir)/$(mod)/Makefile)))))
 
-# Locate used packages in $(RIOTPKG).
-PKG_PATHS := $(sort $(foreach dir,$(RIOTPKG),\
+# Locate used packages in $(RIOTPKG) or $(EXTERNAL_PKG_DIRS).
+PKGDIRS := $(RIOTPKG) $(EXTERNAL_PKG_DIRS)
+PKG_PATHS := $(sort $(foreach dir,$(PKGDIRS),\
   $(foreach pkg,$(USEPKG),$(dir $(wildcard $(dir)/$(pkg)/Makefile)))))
 
 # Back up current state to detect changes

makefiles/kconfig.mk

@@ -27,7 +27,10 @@ export KCONFIG_AUTOHEADER_HEADER
 export KCONFIG_GENERATED_DEPENDENCIES = $(GENERATED_DIR)/Kconfig.dep
 
 # This file will contain external module configurations
-export KCONFIG_EXTERNAL_CONFIGS = $(GENERATED_DIR)/Kconfig.external_modules
+export KCONFIG_EXTERNAL_MODULE_CONFIGS = $(GENERATED_DIR)/Kconfig.external_modules
+
+# This file will contain external package configurations
+export KCONFIG_EXTERNAL_PKG_CONFIGS = $(GENERATED_DIR)/Kconfig.external_pkgs
 
 # Add configurations that only work when running the Kconfig test so far,
 # because they activate modules.
@@ -183,8 +186,8 @@ EXTERNAL_MODULE_KCONFIGS ?= $(sort $(foreach dir,$(EXTERNAL_MODULE_DIRS),\
                               $(wildcard $(dir)/*/Kconfig)))
 # Build a Kconfig file that source all external modules configuration
 # files. Every EXTERNAL_MODULE_DIRS with a Kconfig file is written to
-# KCONFIG_EXTERNAL_CONFIGS as 'osource dir/Kconfig'
-$(KCONFIG_EXTERNAL_CONFIGS): FORCE | $(GENERATED_DIR)
+# KCONFIG_EXTERNAL_MODULE_CONFIGS as 'osource dir/Kconfig'
+$(KCONFIG_EXTERNAL_MODULE_CONFIGS): FORCE | $(GENERATED_DIR)
 	$(Q)\
 	if [ -n "$(EXTERNAL_MODULE_KCONFIGS)" ] ; then  \
 		printf "%s\n" $(EXTERNAL_MODULE_KCONFIGS) \
@@ -195,6 +198,24 @@ $(KCONFIG_EXTERNAL_CONFIGS): FORCE | $(GENERATED_DIR)
 		| $(LAZYSPONGE) $(LAZYSPONGE_FLAGS) $@ ; \
 	fi
 
+# All directories in EXTERNAL_PKG_DIRS which have a subdirectory containing a
+# Kconfig file.
+EXTERNAL_PKG_KCONFIGS ?= $(sort $(foreach dir,$(EXTERNAL_PKG_DIRS),\
+                              $(wildcard $(dir)/*/Kconfig)))
+# Build a Kconfig file that sources all external packages configuration
+# files. Every directory with a Kconfig file is written to KCONFIG_PKG_CONFIGS
+# as 'osource dir/Kconfig'
+$(KCONFIG_EXTERNAL_PKG_CONFIGS): FORCE | $(GENERATED_DIR)
+	$(Q)\
+	if [ -n "$(EXTERNAL_PKG_KCONFIGS)" ] ; then  \
+		printf "%s\n" $(EXTERNAL_PKG_KCONFIGS) \
+		| awk '{ printf "osource \"%s\"\n", $$0 }' \
+		| $(LAZYSPONGE) $(LAZYSPONGE_FLAGS) $@ ; \
+	else \
+		printf "# no external packages" \
+		| $(LAZYSPONGE) $(LAZYSPONGE_FLAGS) $@ ; \
+	fi
+
 # When the 'clean' target is called, the files inside GENERATED_DIR should be
 # regenerated. For that, we conditionally change GENERATED_DIR from an 'order
 # only' requisite to a normal one.
@@ -211,7 +232,7 @@ GENERATED_DIR_DEP := $(if $(CLEAN),,|) $(GENERATED_DIR)
 # Generates a .config file by merging multiple sources specified in
 # MERGE_SOURCES. This will also generate KCONFIG_OUT_DEP with the list of used
 # Kconfig files.
-$(KCONFIG_OUT_CONFIG): $(KCONFIG_EXTERNAL_CONFIGS)
+$(KCONFIG_OUT_CONFIG): $(KCONFIG_EXTERNAL_MODULE_CONFIGS) $(KCONFIG_EXTERNAL_PKG_CONFIGS)
 $(KCONFIG_OUT_CONFIG): $(GENERATED_DEPENDENCIES_DEP) $(GENCONFIG) $(MERGE_SOURCES) $(GENERATED_DIR_DEP)
 	$(Q) $(GENCONFIG) \
 	  --config-out=$(KCONFIG_OUT_CONFIG) \

pkg/doc.txt

@@ -72,5 +72,34 @@
  *  * commit your changes using `git commit`
  *  * create the patch files using `git format-patch -n riot-port`
  *  * move the resulting patch files to the patches directory of your package.
+ *
+ * Packages outside of RIOTPKG
+ * ---------------------------
+ * It can be beneficial to create packages outside of the RIOT tree. For example
+ * if one is working on new packages that aren't ready to be committed to
+ * upstream or if an application needs its own unique packages. For this, one
+ * can use the `EXTERNAL_PKG_DIRS` make variable. It works similar to the way
+ * [external modules](src/creating-modules.html#modules-outside-of-riotbase) are
+ * handled. In your application's Makefile, in addition to adding the package
+ * name to `USEPKG` as shown above, add the path to a folder that contains your
+ * external packages:
+ *
+ * ~~~~~~~~ {.mk}
+ * EXTERNAL_PKG_DIRS += <PATH_TO_FOLDER_CONTAINING_PACKAGES>
+ * ~~~~~~~~
+ *
+ * The path is allowed to be relative to the application's directory.
+ *
+ * ***NOTE:*** The name of an external package must be unique (both in regard to
+ * other external packages, as well to native RIOT packages). Additionally, the
+ * directory containing the packages must match the package name, e.g. package
+ * `foo`must be located in `<PATH_IN_EXTERNAL_PKG_DIRS>/foo`.
+ *
+ * An example can be found in
+ * [`tests/external_pkg_dirs`](https://github.com/RIOT-OS/RIOT/tree/master/tests/external_pkg_dirs)
+ *
+ * RIOT welcomes new and useful packages. If you'd like to share your work,
+ * consider [contributing](https://github.com/RIOT-OS/RIOT/tree/master/CONTRIBUTING.md).
+ *
  * @}
  */

tests/external_pkg_dirs/Makefile

@@ -0,0 +1,6 @@
+include ../Makefile.tests_common
+
+EXTERNAL_PKG_DIRS += external_pkgs
+USEPKG += external_pkg
+
+include $(RIOTBASE)/Makefile.include

tests/external_pkg_dirs/README.md

@@ -0,0 +1,16 @@
+external_pkg_dirs
+=================
+
+Test application for the EXTERNAL_PKG_DIRS feature of the buildsystem.
+Two external packages are provided in `external_pkgs/`: `external_pkg` and
+`external_pkg_not_used`. If the first package is not properly included, a define
+from `CFLAGS` is missing and a precompiler error is triggered. If the second
+package somehow ends up included it triggers a makefile error.
+
+Usage
+=====
+
+Set `EXTERNAL_PKG_DIRS` inside Makefile to point to other paths where the
+buildsystem can look for packages. Similar functionality to `EXTERNAL_PKG_DIRS`.
+Be careful to not name these externally provided packages the same as existing
+packages in `$(RIOTBASE)/pkg/`.

tests/external_pkg_dirs/app.config.test

@@ -0,0 +1 @@
+CONFIG_PACKAGE_EXTERNAL_PKG=y

tests/external_pkg_dirs/external_pkgs/external_pkg/Kconfig

@@ -0,0 +1,9 @@
+# Copyright (c) 2022 Niklaus Leuenberger
+#
+# This file is subject to the terms and conditions of the GNU Lesser
+# General Public License v2.1. See the file LICENSE in the top level
+# directory for more details.
+
+config PACKAGE_EXTERNAL_PKG
+    bool "external_pkg package"
+    depends on TEST_KCONFIG

tests/external_pkg_dirs/external_pkgs/external_pkg/Makefile

@@ -0,0 +1,16 @@
+PKG_NAME = external_pkg
+
+include $(RIOTBASE)/Makefile.base
+
+.PHONY: all prepare clean distclean
+
+all: prepare
+
+prepare:
+	@:
+
+clean::
+	rm -rf $(BINDIR)/pkg/$(PKG_NAME)
+
+distclean::
+	rm -rf $(BINDIR)/pkg/$(PKG_NAME)

tests/external_pkg_dirs/external_pkgs/external_pkg/Makefile.include

@@ -0,0 +1 @@
+CFLAGS += -DTEST_EXTERNAL_PKG

tests/external_pkg_dirs/external_pkgs/external_pkg_not_used/Kconfig

@@ -0,0 +1,9 @@
+# Copyright (c) 2022 Niklaus Leuenberger
+#
+# This file is subject to the terms and conditions of the GNU Lesser
+# General Public License v2.1. See the file LICENSE in the top level
+# directory for more details.
+
+config PACKAGE_EXTERNAL_PKG_NOT_USED
+    bool "external_pkg_not_used package"
+    depends on TEST_KCONFIG

tests/external_pkg_dirs/external_pkgs/external_pkg_not_used/Makefile

@@ -0,0 +1,19 @@
+PKG_NAME = external_pkg_not_used
+
+include $(RIOTBASE)/Makefile.base
+
+.PHONY: all prepare clean distclean
+
+all:
+	$(error target all for external_pkg_not_used executed)
+
+prepare:
+	$(error target prepare for external_pkg_not_used executed)
+
+clean::
+	rm -rf $(BINDIR)/pkg/$(PKG_NAME)
+	$(error target clean for external_pkg_not_used executed)
+
+distclean::
+	rm -rf $(BINDIR)/pkg/$(PKG_NAME)
+	$(error target distclean for external_pkg_not_used executed)

tests/external_pkg_dirs/external_pkgs/external_pkg_not_used/Makefile.include

@@ -0,0 +1 @@
+CFLAGS += -DTEST_EXTERNAL_PKG_NOT_USED

tests/external_pkg_dirs/main.c

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2021 Niklaus Leuenberger
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @ingroup     tests
+ * @{
+ *
+ * @file
+ * @brief       This is a test for the EXTERNAL_PKG_DIRS variable.
+ *
+ * @author      Niklaus Leuenberger <niklaus.leuenb@gmail.com>
+ *
+ * @}
+ */
+
+#include <stdio.h>
+
+#ifndef TEST_EXTERNAL_PKG
+    #error "Required external package not included."
+#endif
+
+#ifdef TEST_EXTERNAL_PKG_NOT_USED
+    #error "External package included that shouldn't be."
+#endif
+
+int main(void)
+{
+    puts("If it compiles, it works!");
+    return 0;
+}

tests/kconfig/Makefile

@@ -4,6 +4,10 @@ USEMODULE += external_module_1
 USEMODULE += external_module_2
 EXTERNAL_MODULE_DIRS += external_modules
 
+USEPKG += external_pkg_1
+USEPKG += external_pkg_2
+EXTERNAL_PKG_DIRS += external_pkgs
+
 ifeq (1, $(TEST_KCONFIG))
   KCONFIG_ADD_CONFIG += $(APPDIR)/app.config
 endif

tests/kconfig/app.config

@@ -9,3 +9,7 @@ CONFIG_APP_MSG_2=y
 # enable configuration of external modules via kconfig
 CONFIG_KCONFIG_EXTERNAL_MODULE_1=y
 CONFIG_KCONFIG_EXTERNAL_MODULE_2=y
+
+# enable configuration of external packages via kconfig
+CONFIG_KCONFIG_EXTERNAL_PKG_1=y
+CONFIG_KCONFIG_EXTERNAL_PKG_2=y

tests/kconfig/app.config.test

@@ -1,2 +1,4 @@
 CONFIG_MODULE_EXTERNAL_MODULE_1=y
 CONFIG_MODULE_EXTERNAL_MODULE_2=y
+CONFIG_PACKAGE_EXTERNAL_PKG_1=y
+CONFIG_PACKAGE_EXTERNAL_PKG_2=y

tests/kconfig/external_pkgs/external_pkg_1/Kconfig

@@ -0,0 +1,17 @@
+menuconfig KCONFIG_EXTERNAL_PKG_1
+    bool "Configure external package message 1"
+    default y
+    help
+        This will enable configuring the external package message
+
+if KCONFIG_EXTERNAL_PKG_1
+
+config EXTERNAL_PKG_1_MESSAGE
+    string "External package 1 text"
+    default "External package message 1 defined in Kconfig file"
+
+endif # KCONFIG_EXTERNAL_PKG_1
+
+config PACKAGE_EXTERNAL_PKG_1
+    bool "Select external pkg 1"
+    depends on TEST_KCONFIG

tests/kconfig/external_pkgs/external_pkg_1/Makefile

@@ -0,0 +1,16 @@
+PKG_NAME = external_pkg_1
+
+include $(RIOTBASE)/Makefile.base
+
+.PHONY: all prepare clean distclean
+
+all: prepare
+
+prepare:
+	@:
+
+clean::
+	rm -rf $(BINDIR)/pkg/$(PKG_NAME)
+
+distclean::
+	rm -rf $(BINDIR)/pkg/$(PKG_NAME)

tests/kconfig/external_pkgs/external_pkg_1/Makefile.include

@@ -0,0 +1,3 @@
+# Use an immediate variable to evaluate `MAKEFILE_LIST` now
+USEPKG_INCLUDES_external_pkg_1 := $(LAST_MAKEFILEDIR)/include
+INCLUDES += -I$(USEPKG_INCLUDES_external_pkg_1)

tests/kconfig/external_pkgs/external_pkg_1/include/external_pkg_1.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2022 Niklaus Leuenberger
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @defgroup
+ * @ingroup
+ * @brief
+ * @{
+ *
+ * @file
+ * @brief
+ *
+ * @author      Francisco Molina <francois-xavier.molina@inria.fr>
+ * @author      NikLeberg <niklaus.leuenb+github@gmail.com>
+ *
+ * @}
+ */
+
+#ifndef EXTERNAL_PKG_1_H
+#define EXTERNAL_PKG_1_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef CONFIG_EXTERNAL_PKG_1_MESSAGE
+#define CONFIG_EXTERNAL_PKG_1_MESSAGE    "this should not show"
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+/** @} */
+#endif /* EXTERNAL_PKG_1_H */

tests/kconfig/external_pkgs/external_pkg_2/Kconfig

@@ -0,0 +1,17 @@
+menuconfig KCONFIG_EXTERNAL_PKG_2
+    bool "Configure external package message 2"
+    default y
+    help
+        This will enable configuring the external package message
+
+if KCONFIG_EXTERNAL_PKG_2
+
+config EXTERNAL_PKG_2_MESSAGE
+    string "External package 2 text"
+    default "External package message 2 defined in Kconfig file"
+
+endif # KCONFIG_EXTERNAL_PKG_2
+
+config PACKAGE_EXTERNAL_PKG_2
+    bool "Select external pkg 2"
+    depends on TEST_KCONFIG

tests/kconfig/external_pkgs/external_pkg_2/Makefile

@@ -0,0 +1,16 @@
+PKG_NAME = external_pkg_2
+
+include $(RIOTBASE)/Makefile.base
+
+.PHONY: all prepare clean distclean
+
+all: prepare
+
+prepare:
+	@:
+
+clean::
+	rm -rf $(BINDIR)/pkg/$(PKG_NAME)
+
+distclean::
+	rm -rf $(BINDIR)/pkg/$(PKG_NAME)

tests/kconfig/external_pkgs/external_pkg_2/Makefile.include

@@ -0,0 +1,3 @@
+# Use an immediate variable to evaluate `MAKEFILE_LIST` now
+USEPKG_INCLUDES_external_pkg_2 := $(LAST_MAKEFILEDIR)/include
+INCLUDES += -I$(USEPKG_INCLUDES_external_pkg_2)

tests/kconfig/external_pkgs/external_pkg_2/include/external_pkg_2.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2022 Niklaus Leuenberger
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @defgroup
+ * @ingroup
+ * @brief
+ * @{
+ *
+ * @file
+ * @brief
+ *
+ * @author      Francisco Molina <francois-xavier.molina@inria.fr>
+ * @author      NikLeberg <niklaus.leuenb+github@gmail.com>
+ *
+ * @}
+ */
+
+#ifndef EXTERNAL_PKG_2_H
+#define EXTERNAL_PKG_2_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef CONFIG_EXTERNAL_PKG_2_MESSAGE
+#define CONFIG_EXTERNAL_PKG_2_MESSAGE    "this should not show"
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+/** @} */
+#endif /* EXTERNAL_PKG_2_H */

tests/kconfig/main.c

@@ -24,6 +24,8 @@
 #include "kernel_defines.h"
 #include "external_module_1.h"
 #include "external_module_2.h"
+#include "external_pkg_1.h"
+#include "external_pkg_2.h"
 
 int main(void)
 {
@@ -36,5 +38,8 @@ int main(void)
     puts(CONFIG_EXTERNAL_MODULE_1_MESSAGE);
     puts(CONFIG_EXTERNAL_MODULE_2_MESSAGE);
 
+    puts(CONFIG_EXTERNAL_PKG_1_MESSAGE);
+    puts(CONFIG_EXTERNAL_PKG_2_MESSAGE);
+
     return 0;
 }

tests/kconfig/tests/01-run.py

@@ -15,6 +15,8 @@ def testfunc(child):
     child.expect_exact("MSG_2 is active")
     child.expect_exact("External Message 1 defined in Kconfig file")
     child.expect_exact("External Message 2 defined in Kconfig file")
+    child.expect_exact("External package message 1 defined in Kconfig file")
+    child.expect_exact("External package message 2 defined in Kconfig file")
 
 
 if __name__ == "__main__":