RIOT/dist/tools/ci/can_fast_ci_run.py

#!/usr/bin/env python3
"""
Command line utility to check if only a subset of board / application combinations
need to be build in the CI
"""
import argparse
import io
import json
import os
import re
import subprocess
import sys
from functools import partial

REGEX_GIT_DIFF_RENAME_COMPLEX = re.compile(r"^[0-9-]+\s+[0-9-]+\s+(.*)\{(.*) => (.*)\}(.*)$")
REGEX_GIT_DIFF_RENAME_SIMPLE = re.compile(r"^[0-9-]+\s+[0-9-]+\s+(.*) => (.*)$")
REGEX_GIT_DIFF_SINGLE_FILE = re.compile(r"^[0-9-]+\s+[0-9-]+\s+(.*)$")

# Beware: Order matters. The first matching rule will be applied
OTHER_CLASSIFIERS = [
    [re.compile(r"^(Makefile.*|sys\/Makefile.*|drivers\/Makefile.*|makefiles\/.*)$"), "build-system"],
    [re.compile(r"^(drivers\/include\/.*|sys\/include\/.*)$"), "public-headers"],
    [re.compile(r"^(Kconfig|kconfigs\/.*|pkg\/Kconfig|sys\/Kconfig|drivers\/Kconfig)$"), "kconfig"],
    [re.compile(r"^(.*\.cff|doc\/.*|.*\.md|.*\.txt)$"), "doc"],
    [re.compile(r"^(CODEOWNERS|.mailmap|.gitignore|.github\/.*)$"), "git"],
    [re.compile(r"^(\.murdock|dist\/ls\/.*|\.drone.yml)$"), "ci-murdock"],
    [re.compile(r"^(\.bandit|\.drone.yml)$"), "ci-other"],
    [re.compile(r"^(dist\/.*|Vagrantfile)$"), "tools"],
]

REGEX_MODULE = re.compile(r"^(boards\/common|core|cpu|drivers|sys)\/")
REGEX_PKG = re.compile(r"^pkg\/")
REGEX_BOARD = re.compile(r"^boards\/")
REGEX_APP = re.compile(r"^(bootloaders|examples|fuzzing|tests)\/")

EXCEPTION_MODULES = {"boards/common/nrf52"}

print_err = partial(print, file=sys.stderr)


def print_change_set_section(name, contents):
    """
    Print the given change set section human reable
    """
    if not contents:
        return
    print_err(name)
    print_err("=" * len(name))
    print_err("")
    for category in sorted(contents):
        print_err(category)
        print_err("-" * len(category))
        print_err("")
        for file in sorted(contents[category]):
            print_err("- {}".format(file))
        print_err("")


class ChangeSet:
    """
    Representation of the modules affected by a change set
    """
    def __init__(self, riotbase=os.getcwd()):
        self.apps = {}
        self.boards = {}
        self.modules = {}
        self.other = {}
        self.pkgs = {}
        self._riotbase = os.path.normpath(riotbase)

    def __add_module(self, dest, file):
        module = os.path.dirname(file)
        while module != "":
            makefile = os.path.join(self._riotbase, module, "Makefile")
            if os.path.isfile(makefile) or module in EXCEPTION_MODULES:

                # map all tests/unittests/* to just tests/unittests
                # workaround for #18987
                if module.startswith("tests/unittests/"):
                    module = "tests/unittests"

                if module in dest:
                    dest[module].append(file)
                else:
                    dest[module] = [file]
                return
            module = os.path.dirname(module)
        raise Exception("Module containing file \"{}\" not found".format(file))

    def add_file(self, file):
        """
        Add the given file to the change set
        """
        # normalize path
        file = os.path.normpath(file)
        if file.startswith('./'):
            file = file[2:]
        # turn path into path relative to riotbase, if needed
        if file.startswith(self._riotbase):
            file = file[len(self._riotbase):]

        for regex, name in OTHER_CLASSIFIERS:
            if regex.match(file):
                if name in self.other:
                    self.other[name].append(file)
                else:
                    self.other[name] = [file]
                return

        if REGEX_MODULE.match(file):
            self.__add_module(self.modules, file)
        elif REGEX_PKG.match(file):
            self.__add_module(self.pkgs, file)
        elif REGEX_BOARD.match(file):
            self.__add_module(self.boards, file)
        elif REGEX_APP.match(file):
            self.__add_module(self.apps, file)
        else:
            raise Exception("File \"{}\" doesn't match any known category".format(file))

    def print_files_and_classifications(self):
        """
        Print all files and their classification in human readable format
        """
        print_change_set_section("Other", self.other)
        print_change_set_section("Modules", self.modules)
        print_change_set_section("Packages", self.pkgs)
        print_change_set_section("Boards", self.boards)
        print_change_set_section("Apps", self.apps)


def classify_changes(riotbase=None, upstream_branch="master"):
    """
    Runs the given compiler with -v -E on an no-op compilation unit and parses the built-in
    include search directories and the GCC version from the output

    :param args: parse command line arguments
    :type args: dict
    :param pr_branch: name of the PR branch
    :type pr_branch: str
    :param upstream_branch: name of the main upstream branch the PR should be merged into
    :type upstream_branch: str

    :return: True if fast rebuilt is possible, False otherwise
    :rtype: bool
    """
    change_set = ChangeSet(riotbase)

    with subprocess.Popen(["git", "diff", "--numstat", "HEAD..{}".format(upstream_branch)],
                          stdout=subprocess.PIPE) as proc:
        for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"):
            match = REGEX_GIT_DIFF_RENAME_COMPLEX.match(line)
            if match:
                prefix = match.group(1)
                suffix = match.group(4)
                file_before = prefix + match.group(2) + suffix
                file_after = prefix + match.group(3) + suffix
                change_set.add_file(file_before)
                change_set.add_file(file_after)
                continue

            match = REGEX_GIT_DIFF_RENAME_SIMPLE.match(line)
            if match:
                file_before = match.group(1)
                file_after = match.group(2)
                change_set.add_file(file_before)
                change_set.add_file(file_after)
                continue

            match = REGEX_GIT_DIFF_SINGLE_FILE.match(line)
            if match:
                file = match.group(1)

                if only_comment_change(file, upstream_branch) is False:
                    change_set.add_file(file)

                continue

            raise Exception("Failed to parse \"{}\"".format(line))

    return change_set


def get_hash_without_comments(file, ref):
    result = subprocess.run(
        f"git show {ref}:{file} | gcc -fpreprocessed -dD -E -P - | md5sum",
        shell=True,
        capture_output=True,
        check=True,
    )
    return result.stdout


def only_comment_change(file, upstream_branch):
    try:
        if file[-2:] in {".c", ".h"}:
            hash_a = get_hash_without_comments(file, "HEAD")
            hash_b = get_hash_without_comments(file, upstream_branch)
            return hash_a == hash_b
    except Exception:
        pass

    return False


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Check if a fast CI run is possible and which "
                                     + "boards / applications to build")
    parser.add_argument("--explain", action="store_true",
                        help="Explain the reasoning of the decision")
    parser.add_argument("--debug", default=False, action="store_const", const=True,
                        help="Show detailed list of classifications")
    parser.add_argument("--riotbase", default=os.getcwd(),
                        help="Use given paths as RIOT's base path (instead of pwd)")
    parser.add_argument("--upstreambranch", default="master",
                        help="The branch / commit containing the upstream state")

    parser.add_argument("--changed-boards", action="store_true",
                        help="list all boards for all apps should be rebuilt")

    parser.add_argument("--changed-apps", action="store_true",
                        help="list all apps that should be built for all boards")

    parser.add_argument("--json", action="store_true",
                        help="json dump changed boards / apps")

    args = parser.parse_args()
    try:
        change_set = classify_changes(riotbase=args.riotbase, upstream_branch=args.upstreambranch)
    except Exception as e:
        print_err("Couldn't classify changes: {}".format(e))
        sys.exit(1)

    if args.debug:
        change_set.print_files_and_classifications()

    if "kconfig" in change_set.other or "build-system" in change_set.other:
        if args.explain:
            print_err("General build system / KConfig changes require a full CI run")
        sys.exit(1)

    if "ci-murdock" in change_set.other:
        if args.explain:
            print_err("Murdock related changes require a full CI run")

        # only exit/error if MURDOCK_TEST_CHANGE_FILTER is not set.
        # useful for testing.
        if not os.environ.get("MURDOCK_TEST_CHANGE_FILTER"):
            sys.exit(1)

    if "public-headers" in change_set.other:
        if args.explain:
            print_err("Changes in public headers require a full CI run")
        sys.exit(1)

    if len(change_set.modules) > 0:
        if args.explain:
            print_err("Currently changing modules require a full CI run")
        sys.exit(1)

    if len(change_set.pkgs) > 0:
        if args.explain:
            print_err("Currently changing packages require a full CI run")
        sys.exit(1)

    if args.json or args.changed_boards or args.changed_apps:
        result = {
            "apps": sorted(change_set.apps.keys()),
            "boards": sorted(change_set.boards.keys())
        }

    if args.json:
        print(json.dumps(result, indent=2))

    if args.changed_boards:
        changed_boards = " ".join(result.get("boards", []))
        print(f"BOARDS_CHANGED=\"{changed_boards}\"")

    if args.changed_apps:
        changed_apps = " ".join(result.get("apps", []))
        print(f"APPS_CHANGED=\"{changed_apps}\"")