1
0
mirror of https://github.com/RIOT-OS/RIOT.git synced 2024-12-29 04:50:03 +01:00

Merge pull request #14727 from leandrolanzieri/pr/kconfig/refactor_integration

makefiles/kconfig: refactor integration and add genconfig script
This commit is contained in:
Cenk Gündoğan 2020-08-10 12:17:29 +02:00 committed by GitHub
commit b65c6abad5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 271 additions and 93 deletions

View File

@ -1,5 +1,3 @@
genconfig.py
kconfiglib.py
menuconfig.py
merge_config.py
__pycache__

View File

@ -7,9 +7,7 @@ include $(RIOTBASE)/pkg/pkg.mk
all:
$(Q)cp $(PKG_SOURCE_DIR)/kconfiglib.py $(PKG_SOURCE_DIR)/menuconfig.py \
$(PKG_SOURCE_DIR)/genconfig.py $(PKG_SOURCE_DIR)/examples/merge_config.py \
.
remove:
$(Q)$(RM) -r $(PKG_SOURCE_DIR) kconfiglib.py menuconfig.py genconfig.py \
merge_config.py
$(Q)$(RM) -r $(PKG_SOURCE_DIR) kconfiglib.py menuconfig.py

173
dist/tools/kconfiglib/genconfig.py vendored Executable file
View File

@ -0,0 +1,173 @@
#!/usr/bin/env python3
# Copyright (c) 2018-2019, Ulf Magnusson
# 2020 HAW Hamburg
# SPDX-License-Identifier: ISC
"""
This script is used to merge multiple configuration sources and generate
different outputs related to Kconfig:
- Generate a header file with #defines from the configuration, matching the
format of include/generated/autoconf.h in the Linux kernel.
- Write the configuration output as a .config file. See --config-out.
- The --sync-deps, --file-list, and --env-list options generate information that
can be used to avoid needless rebuilds/reconfigurations.
Before writing a header or configuration file, Kconfiglib compares the old
contents of the file against the new contents. If there's no change, the write
is skipped. This avoids updating file metadata like the modification time, and
might save work depending on your build setup.
A custom header string can be inserted at the beginning of generated
configuration and header files by setting the KCONFIG_CONFIG_HEADER and
KCONFIG_AUTOHEADER_HEADER environment variables, respectively. The string is
not automatically made a comment (this is by design, to allow anything to be
added), and no trailing newline is added, so add '/* */', '#', and newlines as
appropriate.
"""
import argparse
import logging
import os
import kconfiglib
DEFAULT_SYNC_DEPS_PATH = "deps/"
class NoConfigurationFile(Exception):
"""
Raised when an operation that requires a configuration input file is
executed but the file is not specified.
"""
pass
def merge_configs(kconf, configs=[]):
# Enable warnings for assignments to undefined symbols
kconf.warn_assign_undef = True
# (This script uses alldefconfig as the base. Other starting states could be
# set up here as well. The approach in examples/allnoconfig_simpler.py could
# provide an allnoconfig starting state for example.)
# Disable warnings generated for multiple assignments to the same symbol within
# a (set of) configuration files. Assigning a symbol multiple times might be
# done intentionally when merging configuration files.
kconf.warn_assign_override = False
kconf.warn_assign_redun = False
# Create a merged configuration by loading the fragments with replace=False.
for config in configs:
logging.debug(kconf.load_config(config, replace=False))
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=__doc__)
parser.add_argument(
"--header-path",
metavar="HEADER_FILE",
help="""
Path to write the generated header file to. If not specified the header file is
not written.
""")
parser.add_argument(
"--config-out",
metavar="CONFIG_FILE",
help="""
Write the configuration to CONFIG_FILE. If not specified the file is not
written.
""")
parser.add_argument(
"--kconfig-filename",
metavar="KCONFIG_FILENAME",
nargs="?",
default="Kconfig",
help="Top-level Kconfig file (default: Kconfig)")
parser.add_argument(
"--sync-deps",
metavar="OUTPUT_DIR",
nargs="?",
const=DEFAULT_SYNC_DEPS_PATH,
help="""
Enable generation of symbol dependency information for incremental builds,
optionally specifying the output directory (default: {}). See the docstring of
Kconfig.sync_deps() in Kconfiglib for more information.
""".format(DEFAULT_SYNC_DEPS_PATH))
parser.add_argument(
"--file-list",
metavar="FILE_LIST_FILE",
help="""
Write a makefile listing all the Kconfig files used, and adding them as
dependencies of HEADER_FILE. The paths are absolute. Files appear in the order
they're 'source'd.
""")
parser.add_argument(
"--env-list",
metavar="ENV_LIST_FILE",
help="""
Write a list of all environment variables referenced in Kconfig files to
ENV_LIST_FILE, with one variable per line. Each line has the format NAME=VALUE.
Only environment variables referenced with the preprocessor $(VAR) syntax are
included, and not variables referenced with the older $VAR syntax (which is
only supported for backwards compatibility).
""")
parser.add_argument(
"-d", "--debug",
action="store_true",
help="Enable debug messages")
parser.add_argument(
"--config-sources",
metavar="CONFIG_SOURCES",
nargs='*',
help="List of configuration files to merge and apply. May be empty.")
args = parser.parse_args()
log_level = logging.DEBUG if args.debug else logging.ERROR
logging.basicConfig(format='[genconfig.py]:%(levelname)s-%(message)s',
level=log_level)
kconf = kconfiglib.Kconfig(args.kconfig_filename)
merge_configs(kconf, args.config_sources)
if args.config_out is not None:
logging.debug(kconf.write_config(args.config_out, save_old=False))
if args.header_path is not None:
logging.debug(kconf.write_autoconf(args.header_path))
if args.sync_deps is not None:
logging.debug("Incremental build header files generated at '{}'".format(args.sync_deps))
kconf.sync_deps(args.sync_deps)
if args.file_list is not None:
if args.config_out is None:
raise NoConfigurationFile("Can't generate Kconfig dependency file without configuration file")
logging.debug("Kconfig dependencies written to '{}'".format(args.file_list))
with open(args.file_list, "w", encoding="utf-8") as f:
f.write("{}: \\\n".format(args.config_out))
for path in kconf.kconfig_filenames:
f.write(" {} \\\n".format(os.path.abspath(path)))
if args.env_list is not None:
logging.debug("Kconfig environmental variables written to '{}'".format(args.env_list))
with open(args.env_list, "w", encoding="utf-8") as f:
for env_var in kconf.env_vars:
f.write("{}={}\n".format(env_var, os.environ[env_var]))
if __name__ == "__main__":
main()

View File

@ -92,12 +92,6 @@ will not longer be overridable by means of CFLAGS (e.g. set on the
compilation command or on a Makefile). Consider this if you are getting a
'redefined warning'.
## A note on the usage of the 'clean' command
When using Kconfig as the configurator for RIOT, configuration symbols may be
used in Makefiles through the build system. For this to work properly make
sure that when cleaning an application you call `make clean && make all`,
instead of `make clean all`.
---
# Integration into the build system {#kconfig-integration-into-build-system}
@ -134,27 +128,32 @@ Kconfig symbol as documented in [Appendix A](#kconfig-appendix-a).
### 2. Merging all configuration sources {#kconfig-steps-merge-configs}
In this step configuration values are taken from multiple sources and merged
into a single `merged.config` configuration file. This file is temporary and is
into a single `out.config` configuration file. This file is temporary and is
removed on clean. If the user needs to save a particular configuration
set, a backup has to be saved (this can be done using the menuconfig interface)
so it can be loaded later in this step.
To accomplish merging of multiple input files, the `mergeconfig` script is
To accomplish merging of multiple input files, the `genconfig` script is
used. Note that **the order matters**: existing configuration values are
merged in the order expressed in the input section, where the last value
assigned to a parameter has the highest priority. If no configuration files are
available all default values will be applied.
`merged.config` is the only configuration input for the `autoconf.h` in the
`out.config` is the only configuration input for the `autoconf.h` in the
[generation step](#kconfig-steps-header-gen).
Additionally this step generates a file `out.config.d` which holds the
information of all the used Kconfig files in Makefile format. This file is
included by the build system and allows to re-trigger the generation of
`out.conf` whenever a Kconfig file is modified.
#### Input
- Optional:
- `$ (APPDIR)/app.config`: Application specific default configurations.
- `$ (APPDIR)/user.config`: Configurations saved by user.
#### Output
- `$ (GENERATED_DIR)/merged.config` file.
- `$ (GENERATED_DIR)/out.config` file.
### 3. Menuconfig execution (optional)
Menuconfig is a graphical interface for software configuration. It is used for
@ -162,7 +161,7 @@ the configuration of the Linux kernel. This section explains the process
that occurs when RIOT is being configured using the menuconfig interface.
The main `Kconfig` file is used in this step to show the configurable
parameters of the system. Kconfig will filter innaplicable parameters (i.e.
parameters of the system. Kconfig will filter inapplicable parameters (i.e.
parameters exposed by modules that are not being used) based on the file
`$ (GENERATED_DIR)/Kconfig.dep` generated in step 1.
@ -176,26 +175,29 @@ information see
Note that if Kconfig is not used to configure a module, the corresponding
header files default values will be used.
`merged.config` is one of the inputs for menuconfig. This means that any
`out.config` is one of the inputs for menuconfig. This means that any
configuration that the application defines in the `app.config` or a backup
configuration from the user in `user.config` are taken into account on the
first run (see [Appendix C](#kconfig-appendix-c)).
In this step the user chooses configuration values (or selects the minimal
configuration) and saves it to the `merged.config` file. Here the user can
configuration) and saves it to the `out.config` file. Here the user can
choose to save a backup configuration file for later at a different location
(e.g. a `user.config` file in the application folder).
If any changes occur to `out.config`, the
[generation of autoconf.h](#kconfig-steps-header-gen) is executed automatically.
#### Input
- `/Kconfig` file.
- Optional:
- `$ (APPDIR)/app.config`
- `$ (APPDIR)/user.config`
- `$ (GENERATED_DIR)/merged.config`
- `$ (GENERATED_DIR)/out.config`
#### Output
- Updated `$ (GENERATED_DIR)/merged.config` file.
- `$ (GENERATED_DIR)/merged.config.old` backup file.
- Updated `$ (GENERATED_DIR)/out.config` file.
- `$ (GENERATED_DIR)/out.config.old` backup file.
### 4. Generation of the autoconf.h header {#kconfig-steps-header-gen}
With the addition of Kconfig a dependency has been added to the build
@ -205,30 +207,31 @@ that should be used to configure modules in RIOT:
`CONFIG_<module>_<parameter>`.
In order to generate the `autoconf.h` file the `genconfig` script is used.
Inputs for this script are the main `Kconfig` file and `merged.config`
Inputs for this script are the main `Kconfig` file and `out.config`
configuration file, which holds the selected values for the exposed parameters.
#### Input:
- `$ (GENERATED_DIR)/merged.config` file.
- `$ (GENERATED_DIR)/out.config` file.
- Main `Kconfig` file exposing configuration of modules.
#### Output:
- `$ (GENERATED_DIR)/autoconf.h` configuration header file.
- `$ (GENERATED_DIR)/out.config` file.
- Optional:
- `$ (GENERATED_DIR)/deps/*/*.h` header files that allow incremental builds
### Summary of files
These files are defined in `kconfig.mk`.
| File | Description |
| ----------------- | ----------- |
| `Kconfig` | Defines configuration options of modules. |
| `Kconfig.dep` | Holds a list of the modules that are being compiled. |
| `app.config` | Holds default application configuration values. |
| `user.config` | Holds configuration values applied by the user. |
| `merged.config` | Holds configuration from multiple sources. Used to generate header. |
| `autoconf.h` | Header file containing the macros that applied the selected configuration. |
| `out.config` | Configuration file containing all the symbols defined in `autoconf.h`. |
| File | Description |
| ---------------| ----------- |
| `Kconfig` | Defines configuration options of modules. |
| `Kconfig.dep` | Holds a list of the modules that are being compiled. |
| `app.config` | Holds default application configuration values. |
| `user.config` | Holds configuration values applied by the user. |
| `out.config` | Configuration file containing all the symbols defined in `autoconf.h`. |
| `out.config.d` | Dependency file of `out.config` containing the list of Kconfig files used to generate it. |
| `autoconf.h` | Header file containing the macros that applied the selected configuration. |
## Kconfig symbols in Makefiles
As '.config' files have Makefile syntax they can be included when building,
@ -254,10 +257,10 @@ application's Makefile. The symbols will not be defined until after including
---
# Transition phase {#kconfig-transition-phase}
## Making configuration via Kconfig optional {#kconfig-configuration-optional}
During transition to the usage of Kconfig as the main configurator for RIOT,
the default behavior will be the traditional one: expose configuration options
in header files and use CFLAGS as inputs. To allow optional configuration via
Kconfig, a convention will be used when writing Kconfig files.
During transition to the usage of Kconfig as the main configuration tool for
RIOT, the default behavior will be the traditional one: expose configuration
options in header files and use CFLAGS as inputs. To allow optional
configuration via Kconfig, a convention will be used when writing Kconfig files.
Modules should be contained in their own `menuconfig` entries, this way the user
can choose to enable the configuration via Kconfig for an specific module.
@ -548,14 +551,18 @@ the menuconfig graphical interface or writing '.config' files by hand.
As explained in the
['Configuration sources merging step'](#kconfig-steps-merge-configs)
of the configuration process, configuration from multiple sources are loaded to
create a single `merged.config` file, and the order of merging matters: last
create a single `out.config` file, and the order of merging matters: last
file has priority.
While editing values directly via '.config' files `merged.config` will be
re-built. Once the user decides to edit `merged.config` directly using
menuconfig, the file will not be re-built anymore, and any changes by manually
editing the source files will have no effect. To go back to manual edition
a `make clean` has to be issued in the application directory.
While editing values directly via '.config' files `out.config` will be
re-built. The user can also use menuconfig interface to modify the configuration
file (this is the recommended way, as it gives access to much more information
regarding dependencies and default values of the symbols). Menuconfig will
change `out.config` directly (a backup file `out.config.old` will be kept).
**It is recommended to save backups of the configurations, as any change on the
configuration sources would re-trigger the merging process and overwrite
`out.config`.**
## Appendix D: A few key aspects while exposing a macro to Kconfig {#kconfig-appendix-d}
A macro that holds a 0 or 1 is modelled in Kconfig as a `bool` symbol. References to this macro

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 13 KiB

After

Width:  |  Height:  |  Size: 14 KiB

View File

@ -29,24 +29,22 @@ KCONFIG_APP_CONFIG = $(APPDIR)/app.config
# Default and user overwritten configurations
KCONFIG_USER_CONFIG = $(APPDIR)/user.config
# This file will contain merged configurations from MERGE_SOURCES and is the
# one that is used to generate the 'riotconf.h' header
KCONFIG_MERGED_CONFIG = $(GENERATED_DIR)/merged.config
# This is the output of the generated configuration. It always mirrors the
# content of KCONFIG_GENERATED_AUTOCONF_HEADER_C, and it is used to load
# configuration symbols to the build system.
KCONFIG_OUT_CONFIG = $(GENERATED_DIR)/out.config
# This file is generated by the GENCONFIG tool. It is similar to the .d
# files generated by GCC, and the idea is the same. We want to re-trigger the
# generation of KCONFIG_OUT_CONFIG and KCONFIG_GENERATED_AUTOCONF_HEADER_C
# whenever a change occurs on one of the previously used Kconfig files.
KCONFIG_OUT_DEP = $(KCONFIG_OUT_CONFIG).d
# Include configuration symbols if available. This allows to check for Kconfig
# symbols in makefiles. Make tries to 'remake' all included files (see
# https://www.gnu.org/software/make/manual/html_node/Remaking-Makefiles.html).
-include $(KCONFIG_OUT_CONFIG)
# Flag that indicates that the configuration KCONFIG_MERGED_CONFIG has been
# edited
KCONFIG_EDITED_CONFIG = $(GENERATED_DIR)/.editedconfig
# Add configurations to merge, in ascendent priority (i.e. a file overrides the
# previous ones).
MERGE_SOURCES += $(wildcard $(KCONFIG_APP_CONFIG))
@ -69,7 +67,10 @@ $(GENERATED_DIR): $(if $(MAKE_RESTARTS),,$(CLEAN))
# configuration via Kconfig is disabled by default). Should this change, the
# check would not longer be valid, and Kconfig would have to run on every
# build.
SHOULD_RUN_KCONFIG ?= $(or $(wildcard $(APPDIR)/*.config), $(wildcard $(APPDIR)/Kconfig), $(wildcard $(KCONFIG_MERGED_CONFIG)), $(filter menuconfig, $(MAKECMDGOALS)))
SHOULD_RUN_KCONFIG ?= $(or $(wildcard $(APPDIR)/*.config), \
$(wildcard $(APPDIR)/Kconfig), \
$(if $(CLEAN),,$(wildcard $(KCONFIG_OUT_CONFIG))), \
$(filter menuconfig, $(MAKECMDGOALS)))
ifneq (,$(SHOULD_RUN_KCONFIG))
@ -80,11 +81,24 @@ KCONFIG_SYNC_DEPS ?=
BUILDDEPS += $(KCONFIG_GENERATED_AUTOCONF_HEADER_C)
# Include configuration header when building
CFLAGS += -include '$(KCONFIG_GENERATED_AUTOCONF_HEADER_C)'
CFLAGS += -imacros '$(KCONFIG_GENERATED_AUTOCONF_HEADER_C)'
USEMODULE_W_PREFIX = $(addprefix MODULE_,$(USEMODULE))
USEPKG_W_PREFIX = $(addprefix PKG_,$(USEPKG))
.PHONY: menuconfig
# Opens the menuconfig interface for configuration of modules using the Kconfig
# system. It will try to update the autoconf.h, which will update if needed
# (i.e. out.config changed).
menuconfig: $(MENUCONFIG) $(KCONFIG_OUT_CONFIG)
$(Q)KCONFIG_CONFIG=$(KCONFIG_OUT_CONFIG) $(MENUCONFIG) $(KCONFIG)
$(MAKE) $(KCONFIG_GENERATED_AUTOCONF_HEADER_C)
# These rules are not included when only calling `make clean` in
# order to keep the $(BINDIR) directory clean.
ifneq (clean, $(MAKECMDGOALS))
# Build a Kconfig file defining all used modules and packages. This is done by
# defining symbols like 'MODULE_<MODULE_NAME>' or PKG_<PACKAGE_NAME> which
# default to 'y'. Then, every module and package Kconfig menu will depend on
@ -95,43 +109,34 @@ $(KCONFIG_GENERATED_DEPENDENCIES): FORCE | $(GENERATED_DIR)
printf "config %s\n\tbool\n\tdefault y\n", toupper($$0)}' \
| $(LAZYSPONGE) $(LAZYSPONGE_FLAGS) $@
.PHONY: menuconfig
# Conditionally depend on KCONFIG_MERGED_CONFIG. Only trigger configuration
# merging process if there are configuration files to merge. This avoids the
# usage of aditional bash `if [ ]` in the target recipe.
MERGE_CONFIG_DEP = $(if $(strip $(MERGE_SOURCES)),$(KCONFIG_MERGED_CONFIG))
# Opens the menuconfig interface for configuration of modules using the Kconfig
# system.
menuconfig: $(MENUCONFIG) $(MERGE_CONFIG_DEP) $(KCONFIG_EDITED_CONFIG)
$(Q)KCONFIG_CONFIG=$(KCONFIG_MERGED_CONFIG) $(MENUCONFIG) $(KCONFIG)
# Marks that the configuration file has been edited via some interface, such as
# menuconfig
$(KCONFIG_EDITED_CONFIG): FORCE
$(Q)touch $(KCONFIG_EDITED_CONFIG)
# Generates a merged configuration file from the given sources, only when the
# configuration has not been updated by some interface like menuconfig
$(KCONFIG_MERGED_CONFIG): $(MERGECONFIG) $(KCONFIG_GENERATED_DEPENDENCIES) $(MERGE_SOURCES)
$(Q)\
if ! test -f $(KCONFIG_EDITED_CONFIG); then \
$(MERGECONFIG) $(KCONFIG) $@ $(MERGE_SOURCES); \
fi
# Build a header file with all the Kconfig configurations. genconfig will avoid
# any unnecessary rewrites of the header file if no configurations changed.
# The rule is not included when only `make clean` is called in order to keep the
# $(BINDIR) folder clean
ifneq (clean,$(MAKECMDGOALS))
$(KCONFIG_OUT_CONFIG) $(KCONFIG_GENERATED_AUTOCONF_HEADER_C) &: $(KCONFIG_GENERATED_DEPENDENCIES) $(GENCONFIG) $(MERGE_CONFIG_DEP)
$(Q) \
KCONFIG_CONFIG=$(KCONFIG_MERGED_CONFIG) $(GENCONFIG) \
# Generates a .config file by merging multiple sources specified in
# MERGE_SOURCES. This will also generate KCONFIG_OUT_DEP with the list of used
# Kconfig files.
$(KCONFIG_OUT_CONFIG): $(KCONFIG_GENERATED_DEPENDENCIES) $(GENCONFIG) $(MERGE_SOURCES) | $(GENERATED_DIR)
$(Q) $(GENCONFIG) \
--config-out=$(KCONFIG_OUT_CONFIG) \
--file-list $(KCONFIG_OUT_DEP) \
--kconfig-filename $(KCONFIG) \
--config-sources $(MERGE_SOURCES) && \
touch $(KCONFIG_OUT_CONFIG)
endif # eq (clean, $(MAKECMDGOALS))
# Generates the configuration header file which holds the same information
# as KCONFIG_OUT_CONFIG, and is used to inject the configurations during
# compilation.
#
# This will optionally generate the 'dummy' header files needed for incremental
# builds.
$(KCONFIG_GENERATED_AUTOCONF_HEADER_C): $(KCONFIG_OUT_CONFIG)
$(Q) $(GENCONFIG) \
--header-path $(KCONFIG_GENERATED_AUTOCONF_HEADER_C) \
$(if $(KCONFIG_SYNC_DEPS),--sync-deps $(KCONFIG_SYNC_DIR)) \
$(KCONFIG)
endif
--kconfig-filename $(KCONFIG) \
--config-sources $(KCONFIG_OUT_CONFIG) && \
touch $(KCONFIG_GENERATED_AUTOCONF_HEADER_C)
# Try to load the list of Kconfig files used
-include $(KCONFIG_OUT_DEP)
endif

View File

@ -1,8 +1,7 @@
# Define tools to use
MENUCONFIG ?= $(RIOTTOOLS)/kconfiglib/riot_menuconfig.py
BASE_MENUCONFIG ?= $(RIOTTOOLS)/kconfiglib/menuconfig.py
GENCONFIG ?= $(RIOTTOOLS)/kconfiglib/genconfig.py
MERGECONFIG ?= $(RIOTTOOLS)/kconfiglib/merge_config.py
GENCONFIG := $(RIOTTOOLS)/kconfiglib/genconfig.py
$(BASE_MENUCONFIG):
@echo "[INFO] Kconfiglib not found - getting it"
@ -10,5 +9,3 @@ $(BASE_MENUCONFIG):
@echo "[INFO] Kconfiglib downloaded"
$(GENCONFIG): $(BASE_MENUCONFIG)
$(MERGECONFIG): $(BASE_MENUCONFIG)