From a07dac9ad1b0ec2175f6554bab1deb6e4674adcc Mon Sep 17 00:00:00 2001 From: Marian Buschsieweke Date: Tue, 2 Mar 2021 21:26:20 +0100 Subject: [PATCH] build system: add new compile-commands make target By running make compile-commands a `compile_commands.json` in the RIOT base directory. With the environment variable `COMPILE_COMMANDS` the path of this file can be changed to a custom location. The `compile_commands.json` will contain the exact compile command, but as additional flag `-I/usr/$(TARGET)/include` is added to work around `clangd` not being able to locate the newlib system headers. The additional includes can be overwritten using the environment variable `COMPILE_COMMANDS_EXTRA_INCLUDES`. --- Makefile.base | 23 +- Makefile.include | 9 + dist/tools/compile_commands/README.md | 8 + .../compile_commands/compile_commands.py | 292 ++++++++++++++++++ doc/doxygen/src/getting-started.md | 22 ++ 5 files changed, 352 insertions(+), 2 deletions(-) create mode 100644 dist/tools/compile_commands/README.md create mode 100755 dist/tools/compile_commands/compile_commands.py diff --git a/Makefile.base b/Makefile.base index 53f858e99a..9b12efeeb9 100644 --- a/Makefile.base +++ b/Makefile.base @@ -17,7 +17,8 @@ DIRS := $(sort $(abspath $(DIRS))) _MOD := $(shell basename $(CURDIR)) MODULE ?= $(_MOD) -.PHONY: all clean $(DIRS:%=ALL--%) $(DIRS:%=CLEAN--%) $(MODULE).module +.PHONY: all clean $(DIRS:%=ALL--%) $(DIRS:%=CLEAN--%) $(MODULE).module \ + compile-commands $(DIRS:%=COMPILE-COMMANDS--%) all: $(MODULE).module ..nothing @@ -32,6 +33,9 @@ $(DIRS:%=ALL--%): $(DIRS:%=CLEAN--%): $(QQ)"$(MAKE)" -C $(@:CLEAN--%=%) clean +$(DIRS:%=COMPILE-COMMANDS--%): + $(QQ)"$(MAKE)" -C $(@:COMPILE-COMMANDS--%=%) compile-commands + ## submodules ifeq (1, $(SUBMODULES)) # don't use *.c as SRC if SRC is empty (e.g., no module selected) @@ -69,6 +73,21 @@ ifneq (,$(SRCXX)) endif endif +compile-commands: | $(DIRS:%=COMPILE-COMMANDS--%) + $(file >$(BINDIR)/$(MODULE)/compile_cmds.txt,SRC: $(sort $(SRC) $(SRC_NO_LTO))) + $(file >>$(BINDIR)/$(MODULE)/compile_cmds.txt,SRC_NO_LTO: $(sort $(SRC_NO_LTO))) + $(file >>$(BINDIR)/$(MODULE)/compile_cmds.txt,SRCXX: $(sort $(SRCXX))) + $(file >>$(BINDIR)/$(MODULE)/compile_cmds.txt,CURDIR: $(CURDIR)) + $(file >>$(BINDIR)/$(MODULE)/compile_cmds.txt,CFLAGS: $(CFLAGS)) + $(file >>$(BINDIR)/$(MODULE)/compile_cmds.txt,LTOFLAGS: $(LTOFLAGS)) + $(file >>$(BINDIR)/$(MODULE)/compile_cmds.txt,INCLUDES: $(INCLUDES)) + $(file >>$(BINDIR)/$(MODULE)/compile_cmds.txt,CXXFLAGS: $(CXXFLAGS)) + $(file >>$(BINDIR)/$(MODULE)/compile_cmds.txt,CXXINCLUDES: $(CXXINCLUDES)) + $(file >>$(BINDIR)/$(MODULE)/compile_cmds.txt,CC: $(CC)) + $(file >>$(BINDIR)/$(MODULE)/compile_cmds.txt,CXX: $(CXX)) + $(file >>$(BINDIR)/$(MODULE)/compile_cmds.txt,TARGET_ARCH: $(TARGET_ARCH)) + $(file >>$(BINDIR)/$(MODULE)/compile_cmds.txt,TARGET_ARCH_LLVM: $(TARGET_ARCH_LLVM)) + # include makefile snippets for packages in $(USEPKG) that modify GENSRC: -include $(USEPKG:%=$(RIOTPKG)/%/Makefile.gensrc) @@ -89,7 +108,7 @@ include $(RIOTMAKE)/tools/fixdep.inc.mk $(BINDIR)/$(MODULE)/: $(Q)mkdir -p $@ -$(MODULE).module $(OBJ): | $(BINDIR)/$(MODULE)/ +$(MODULE).module compile-commands $(OBJ): | $(BINDIR)/$(MODULE)/ $(MODULE).module: $(OBJ) | $(DIRS:%=ALL--%) diff --git a/Makefile.include b/Makefile.include index 9bb29c5ad7..a5c49da719 100644 --- a/Makefile.include +++ b/Makefile.include @@ -635,6 +635,15 @@ $(APPLICATION_MODULE).module: pkg-build $(BUILDDEPS) "$(MAKE)" -C $(APPDIR) -f $(RIOTMAKE)/application.inc.mk $(APPLICATION_MODULE).module: FORCE +COMPILE_COMMANDS_PATH ?= $(RIOTBASE)/compile_commands.json +COMPILE_COMMANDS_FLAGS ?= --clangd +.PHONY: compile-commands +compile-commands: $(BUILDDEPS) + $(Q)DIRS="$(DIRS)" APPLICATION_BLOBS="$(BLOBS)" \ + "$(MAKE)" -C $(APPDIR) -f $(RIOTMAKE)/application.inc.mk compile-commands + $(Q)$(RIOTTOOLS)/compile_commands/compile_commands.py $(COMPILE_COMMANDS_FLAGS) $(BINDIR) \ + > $(COMPILE_COMMANDS_PATH) + # Other modules are built by application.inc.mk and packages building _SUBMAKE_LIBS = $(filter-out $(APPLICATION_MODULE).module $(APPDEPS), $(BASELIBS) $(ARCHIVES)) $(_SUBMAKE_LIBS): $(APPLICATION_MODULE).module pkg-build diff --git a/dist/tools/compile_commands/README.md b/dist/tools/compile_commands/README.md new file mode 100644 index 0000000000..a48d562f2c --- /dev/null +++ b/dist/tools/compile_commands/README.md @@ -0,0 +1,8 @@ +Generation of `compile_commands.json` +===================================== + +This tool can be used to generate `compile_commands.json` e.g. for code completion and linting in +IDEs. It relies on the build system providing the compilation details in bin dir. This tools is +best invoked indirectly via `make compile-commands` inside the application, which will first +generate the required files in the bin directory and then invoke this tool. For more details, check +the getting started page in the RIOT API documentation. diff --git a/dist/tools/compile_commands/compile_commands.py b/dist/tools/compile_commands/compile_commands.py new file mode 100755 index 0000000000..92137bf93e --- /dev/null +++ b/dist/tools/compile_commands/compile_commands.py @@ -0,0 +1,292 @@ +#!/usr/bin/python3 +""" +Command line utility to generate compile_commands.json for RIOT applications +""" +import argparse +import json +import os +import re +import shlex +import subprocess +import sys + +REGEX_VERSION = re.compile(r"\ngcc version ([^ ]+)") +REGEX_INCLUDES = r"^#include <\.\.\.> search starts here:$((?:\n|\r|.)*?)^End of search list\.$" +REGEX_INCLUDES = re.compile(REGEX_INCLUDES, re.MULTILINE) + + +def detect_includes_and_version_gcc(compiler): + """ + Runs the given compiler with -v -E on an no-op compilation unit and parses the built-in + include search directories and the GCC version from the output + + :param compiler: name / path of the compiler to run + :type compiler: str + + :return: (list_of_include_paths, version) + :rtype: tuple + """ + try: + with subprocess.Popen([compiler, "-v", "-E", "-"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + inputdata = b"typedef int dont_be_pedantic;" + _, stderrdata = proc.communicate(input=inputdata) + except FileNotFoundError: + msg = "Compiler {} not found, not adding system include paths\n".format(compiler) + sys.stderr.write(msg) + return [] + + stderrdata = stderrdata.decode("utf-8") + version = REGEX_VERSION.search(stderrdata).group(1) + includes = [os.path.abspath(p) for p in REGEX_INCLUDES.search(stderrdata).group(1).split()] + + return (includes, version) + + +def detect_libstdcxx_includes(compiler, includes, version): + """ + Tries to detect the g++ libstdc++ built-in include search directories using black magic and + adds them to the list given in includes + + :param compiler: Name or path of the compiler + :type compiler: str + :param includes: List of include directories + :type includes: list of str + :param version: Version of g++ + :type version: str + """ + for path in includes: + cxx_lib = os.path.join(path, "c++", version) + if os.path.exists(cxx_lib): + includes.append(cxx_lib) + triple = os.path.basename(compiler)[0:-4] + cxx_extra = os.path.join(cxx_lib, triple) + if os.path.exists(cxx_extra): + includes.append(cxx_extra) + break + + +def detect_built_in_includes(compiler, args): + """ + Tries to detect the built-in include search directories of the given compiler + + :param compiler: Name or path of the compiler + :type compiler: str + :param args: Command line arguments + + :return: List of built-in include directories + :rtype: list of str + """ + if compiler.endswith('-gcc'): + includes, version = detect_includes_and_version_gcc(compiler) + elif compiler.endswith('-g++'): + includes, version = detect_includes_and_version_gcc(compiler) + if args.add_libstdcxx_includes: + detect_libstdcxx_includes(compiler, includes, version) + elif compiler in ('clang', 'clang++', 'gcc', 'g++'): + # clang / clang++ doesn't have any magic include search dirs built in, so we don't need + # to detect them. + # for host gcc/g++ we don't need to detect magic include dirs either. + includes = [] + else: + msg = "Warning: Cannot detect default include search paths for {}\n".format(compiler) + sys.stderr.write(msg) + includes = [] + return includes + + +class CompilationDetails: # pylint: disable=too-few-public-methods,too-many-instance-attributes + """ + Representation of the compilation details stored by RIOT's build system. + + :param path: Path of the module to read compilation details from + :type path: str + """ + def __init__(self, path): # pylint: disable=too-many-branches + with open(os.path.join(path, "compile_cmds.txt"), "r") as file: + for line in file.read().splitlines(): + if line.startswith("SRC: "): + self.src_c = line.lstrip("SRC: ").split() + elif line.startswith("SRC_NO_LTO: "): + self.src_c_no_lto = line.lstrip("SRC_NO_LTO: ").split() + elif line.startswith("SRCXX: "): + self.src_cxx = line.lstrip("SRCXX: ").split() + elif line.startswith("CURDIR: "): + self.dir = line.lstrip("CURDIR: ").strip() + elif line.startswith("CFLAGS: "): + self.cflags = shlex.split(line.lstrip("CFLAGS: ")) + elif line.startswith("LTOFLAGS: "): + self.ltoflags = shlex.split(line.lstrip("LTOFLAGS: ")) + elif line.startswith("INCLUDES: "): + self.includes = shlex.split(line.lstrip("INCLUDES: ")) + elif line.startswith("CXXFLAGS: "): + self.cxxflags = shlex.split(line.lstrip("CXXFLAGS: ")) + elif line.startswith("CXXINCLUDES: "): + self.cxxincludes = shlex.split(line.lstrip("CXXINCLUDES: ")) + elif line.startswith("CC: "): + self.cc = line.lstrip("CC: ").strip() # pylint: disable=invalid-name + elif line.startswith("CXX: "): + self.cxx = line.lstrip("CXX: ").strip() + elif line.startswith("TARGET_ARCH: "): + self.target_arch = line.lstrip("TARGET_ARCH: ").strip() + elif line.startswith("TARGET_ARCH_LLVM: "): + self.target_arch_llvm = line.lstrip("TARGET_ARCH_LLVM: ").strip() + + +class State: # pylint: disable=too-few-public-methods + """ + Entity to store the current programs state + """ + def __init__(self): + self.def_includes = dict() + self.is_first = True + + +def get_built_in_include_flags(compiler, state, args): + """ + Get built-in include search directories as parameter list. + + :param compiler: Name or path of the compiler to get the include search dirs from + :type compiler: str + :param state: state of the program + :param args: command line arguments + + :return: The -isystem <...> compiler flags for the built-in include search dirs as list + :rtype: list + """ + + result = [] + if compiler not in state.def_includes: + state.def_includes[compiler] = detect_built_in_includes(compiler, args) + + for include in state.def_includes[compiler]: + result.append('-isystem') + result.append(include) + + return result + + +def write_compile_command(state, compiler, src, flags, cdetails, path): + """ + Write the compile command for the given source file with the given parameters to stdout + + :param state: state of the program + :param compiler: the C/C++ compiler used + :type compiler: str + :param src: the file to compiler + :type src: str + :param flags: flags used for compiler invocation + :type flags: list of str + :param cetails: compilation details + :type cdetails: CompilationDetails + :param path: the output path + "type path: str + """ + if state.is_first: + state.is_first = False + else: + sys.stdout.write(",\n") + obj = os.path.splitext(src)[0] + ".o" + arguments = [compiler, '-DRIOT_FILE_RELATIVE="' + os.path.join(cdetails.dir, src) + '"', + '-DRIOT_FILE_NOPATH="' + src + '"'] + arguments += flags + arguments += ['-MQ', obj, '-MD', '-MP', '-c', '-o', obj, src] + entry = { + 'arguments': arguments, + 'directory': cdetails.dir, + 'file': os.path.join(cdetails.dir, src), + 'output': os.path.join(path, obj) + } + sys.stdout.write(json.dumps(entry, indent=2)) + + +def generate_module_compile_commands(path, state, args): + """ + Generate section of compile_commands.json for the module in path and write it to stdout. + + :param path: path of the module's bin folder to emit the compile_commands.json chunk for + :type path: str + :param state: state of the program + :param args: command line arguments + """ + cdetails = CompilationDetails(path) + + for flag in args.filter_out: + try: + cdetails.cflags.remove(flag) + except ValueError: + pass + try: + cdetails.cxxflags.remove(flag) + except ValueError: + pass + + c_extra_includes = [] + cxx_extra_includes = [] + + if args.add_built_in_includes: + c_extra_includes = get_built_in_include_flags(cdetails.cc, state, args) + cxx_extra_includes = get_built_in_include_flags(cdetails.cxx, state, args) + + if args.clangd: + if cdetails.target_arch_llvm: + cdetails.cflags += ['-target', cdetails.target_arch_llvm] + cdetails.cxxflags += ['-target', cdetails.target_arch_llvm] + elif cdetails.target_arch: + cdetails.cflags += ['-target', cdetails.target_arch] + cdetails.cxxflags += ['-target', cdetails.target_arch] + + for src in cdetails.src_c: + compiler = 'clang' if args.clangd else cdetails.cc + flags = cdetails.cflags + cdetails.ltoflags + cdetails.includes + c_extra_includes + write_compile_command(state, compiler, src, flags, cdetails, path) + + for src in cdetails.src_c_no_lto: + compiler = 'clang' if args.clangd else cdetails.cc + flags = cdetails.cflags + cdetails.includes + c_extra_includes + write_compile_command(state, compiler, src, flags, cdetails, path) + + for src in cdetails.src_cxx: + compiler = 'clang++' if args.clangd else cdetails.cxx + flags = cdetails.cxxflags + cdetails.cxxincludes + cdetails.includes + cxx_extra_includes + write_compile_command(state, compiler, src, flags, cdetails, path) + + +def generate_compile_commands(args): + """ + Generate the compile_commands.json content and write them to stdout + + :param args: command line arguments + """ + state = State() + sys.stdout.write("[\n") + + for module in os.scandir(args.path): + if module.is_dir() and os.path.isfile(os.path.join(module.path, 'compile_cmds.txt')): + generate_module_compile_commands(module.path, state, args) + sys.stdout.write("\n]\n") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Generate compile_commands.json for RIOT apps') + parser.add_argument('path', metavar='PATH', type=str, + help='Bin path, usually \'/bin/\'') + parser.add_argument('--add-built-in-includes', default=False, action='store_const', const=True, + help='Explicitly add built in include search directories with -I ' + + 'options') + parser.add_argument('--add-libstdcxx-includes', default=False, action='store_const', const=True, + help='Explicitly add libstdc++ include search directories with -I ' + + 'options') + parser.add_argument('--filter-out', type=str, default=[], action='append', + help='Drop the given flag, if present (repeatable)') + parser.add_argument('--clangd', default=False, action='store_const', const=True, + help='Shorthand for --add-built-in-includes --add-libstdxx-includes ' + + '--filter-out=-Wformat-truncation --filter-out=-Wformat-overflow ' + + '--filter-out=-mno-thumb-interwork') + _args = parser.parse_args() + if _args.clangd: + _args.add_built_in_includes = True + _args.add_libstdcxx_includes = True + _args.filter_out = ['-Wformat-truncation', '-Wformat-overflow', '-mno-thumb-interwork'] + generate_compile_commands(_args) diff --git a/doc/doxygen/src/getting-started.md b/doc/doxygen/src/getting-started.md index 1077412eae..b0c46aaf79 100644 --- a/doc/doxygen/src/getting-started.md +++ b/doc/doxygen/src/getting-started.md @@ -225,6 +225,28 @@ Troubleshooting {#docker-troubleshooting} On some Ubuntu versions a make with `BUILD_IN_DOCKER=1` can't resolve the host name of for example github.com. To fix this add the file `/etc/docker/daemon.json` with the address of your DNS Server. +Generating `compile_commands.json` e.g. for code completion in IDEs +=================================================================== + +A `compile_commands.json` for the selected board can be generated by running inside the application +folder the following: + +```console +$ make compile-commands +``` + +This target will honor the variables controlling the build process such as `BOARD`, `TOOLCHAIN`, +`DEVELHELP`, etc. just like the usual build process. This works without actual compilation. By +default, the `compile_commands.json` is placed in the RIOT base directory. This behavior can be +overwritten using the `COMPILE_COMMANDS_PATH` variable by specifying the full absolute path +(including file name) of the `compile_commands.json` instead. + +***Note:*** By default, the built-in include search directories of GCC will be explicitly added +and flags incompatible with `clangd` will be dropped. This will allow using `clangd` as language +server out of the box. If this is not desired, run `export COMPILE_COMMANDS_FLAGS=""` to turn +modification of the compile commands off. For a list of available flags, run +`./dist/tools/compile_commands/compile_commands.py --help` in the RIOT base directory. + Using the native port with networking =====================================