RIOT/.murdock

#!/bin/sh

: ${TEST_BOARDS_AVAILABLE:="esp32-wroom-32 samr21-xpro"}

# temporarily disabling llvm builds until https://github.com/RIOT-OS/RIOT/pull/15595
# is in
#: ${TEST_BOARDS_LLVM_COMPILE:="iotlab-m3 native nrf52dk mulle nucleo-f401re samr21-xpro slstk3402a"}
: ${TEST_BOARDS_LLVM_COMPILE:=""}

: ${TEST_KCONFIG_samr21_xpro:="examples/hello-world tests/periph_*
tests/test_tools tests/congure_* tests/xtimer_* tests/ztimer_*
tests/driver_ad7746 tests/driver_adcxx1c tests/driver_ads101x tests/driver_adt101x
tests/driver_adt7310 tests/driver_adxl345 tests/driver_aip31068 tests/driver_apa102
tests/driver_apds99xx tests/driver_apds99xx_full tests/driver_at tests/driver_at24cxxx
tests/driver_at24mac tests/driver_at25xxx tests/driver_at30tse75x tests/driver_ata8520e
tests/driver_b* tests/driver_ccs811 tests/driver_ccs811_full tests/driver_dcf77
tests/driver_dfplayer tests/driver_dht tests/driver_ds18 tests/driver_ds75lx
tests/driver_ds1307 tests/driver_ds3231 tests/driver_ds3234 tests/driver_dsp0401
tests/driver_dynamixel tests/driver_edbg_eui tests/driver_f* tests/driver_g*
tests/driver_h* tests/driver_i* tests/driver_j* tests/driver_l*
tests/driver_mag3110 tests/driver_mhz19 tests/driver_mma7660
tests/driver_motor_driver tests/driver_mpl3115a2 tests/driver_mpu9x50
tests/driver_mq3 tests/driver_my9221 tests/driver_nvram_spi tests/mtd_flashpage
tests/mtd_mapper tests/driver_o* tests/driver_p* tests/driver_q*
tests/driver_r* tests/driver_s* tests/driver_t* tests/driver_u*
tests/driver_v*"}
: ${TEST_KCONFIG_native:="examples/hello-world
tests/cb_mux*
tests/congure_*
tests/driver_ws281x
tests/eepreg
tests/periph_*
tests/pkg_c25519
tests/pkg_cayenne-lpp
tests/pkg_cifra
tests/pkg_cn-cbor
tests/pkg_emlearn
tests/pkg_hacl
tests/pkg_heatshrink
tests/pkg_jsmn
tests/pkg_libb2
tests/pkg_libcose
tests/pkg_libfixmath
tests/pkg_libhydrogen
tests/pkg_lora-serialization
tests/pkg_lvgl
tests/pkg_lvgl_touch
tests/pkg_micro-ecc
tests/pkg_minmea
tests/pkg_monocypher
tests/pkg_nanocbor
tests/pkg_nanopb
tests/pkg_qDSA
tests/pkg_qcbor
tests/pkg_relic
tests/pkg_tiny-asn1
tests/pkg_tinycbor
tests/pkg_tinycrypt
tests/pkg_tweetnacl
tests/pkg_umorse
tests/pkg_yxml
tests/posix_sleep
tests/prng_*
tests/shell
tests/struct_tm_utility
tests/sys_crypto
tests/test_tools
tests/xtimer_*
tests/ztimer_*
"}

: ${TEST_WITH_CONFIG_SUPPORTED:="examples/suit_update tests/driver_at86rf2xx_aes"}

export RIOT_CI_BUILD=1
export CC_NOCOLOR=1
export STATIC_TESTS=0
export CFLAGS_DBG=""
export DLCACHE_DIR=${DLCACHE_DIR:-~/.dlcache}
export ENABLE_TEST_CACHE=${ENABLE_TEST_CACHE:-1}

# This is a work around for a bug in CCACHE which interacts very badly with
# some features of RIOT and of murdock. The result is that ccache is
# ineffective (i.e. objects are never reused, resulting in extreme cache miss
# rate) and murdock becomes slow.
#
# - CCACHE thinks that -gz by itself enables debugging, which is not true.
#   see https://github.com/ccache/ccache/issues/464
#   - When debug info is included, CCACHE hashes the file paths, as these
#     influence the debug information (the name of compile units and/or their
#     "comp_dir" attribute)
# - Riot does not set -fdebug-prefix-map. This is not that easy as it may not
#   be supported in every toolchain (some are quite old).
# - Murdock builds PRs in different directories each time.
#
# It is only the combination of these three factors which causes this bug.
export OPTIONAL_CFLAGS_BLACKLIST="-gz"

NIGHTLY=${NIGHTLY:-0}
RUN_TESTS=${RUN_TESTS:-${NIGHTLY}}

DWQ_ENV="-E BOARDS -E APPS -E NIGHTLY -E RUN_TESTS -E ENABLE_TEST_CACHE
         -E TEST_HASH -E CI_PULL_LABELS"

get_kconfig_test_apps() {
    case "$1" in
        "samr21-xpro") echo "${TEST_KCONFIG_samr21_xpro}" ;;
    esac
    case "$1" in
        "native") echo "${TEST_KCONFIG_native}" ;;
    esac
}

check_label() {
    local label="${1}"
    [ -z "${CI_PULL_LABELS}" ] && return 1
    echo "${CI_PULL_LABELS}" | grep -q "${label}"
    return $?
}

[ "$RUN_TESTS" != "1" ] && {
    check_label "CI: run tests" && RUN_TESTS=1
}

[ "$ENABLE_TEST_CACHE" = "1" ] && {
    check_label "CI: disable test cache" && export ENABLE_TEST_CACHE=0
}

error() {
    echo "$@"
    exit 1
}

# true if "$2" starts with "$1", false otherwise
startswith() {
  case "${2}" in
    ${1}*) true ;;
    *) false ;;
  esac
}

# if MURDOCK_HOOK is set, this function will execute it and pass on all it's
# parameters. should the hook script exit with negative exit code, hook() makes
# this script exit with error, too.
# hook() will be called from different locations of this script.
# currently, the only caller is "run_test", which calls "hook run_test_pre".
# More hooks will be added as needed.
hook() {
    if [ -n "${MURDOCK_HOOK}" ]; then
        echo "- executing hook $1"
        "${MURDOCK_HOOK}" "$@" || {
            error "$0: hook \"${MURDOCK_HOOK} $@\" failed!"
        }
        echo "- hook $1 finished"
    fi
}

# true if word "$1" is in list of words "$2", false otherwise
# uses grep -w, thus only alphanum and "_" count as word bounderies
# (word "def" matches "abc-def")
is_in_list() {
    [ $# -ne 2 ] && return 1

    local needle="$1"
    local haystack="$2"

    echo "$haystack" | grep -q -w "$needle"
}

# grep that doesn't return error on empty input
_grep() {
    grep "$@"
    true
}

_greplist() {
    if [ $# -eq 0 ]; then
        echo cat
    else
        echo -n "_grep -E ($1"
        shift
        for i in $*; do
            echo -n "|$i"
        done
        echo ")"
    fi
}

# get list of all app directories
get_apps() {
    make -f makefiles/app_dirs.inc.mk info-applications \
        | $(_greplist $APPS) | sort
}

# take app dir as parameter, print all boards that are supported
# Only print for boards in $BOARDS.
get_supported_boards() {
    local appdir=$1
    local boards="$(make --no-print-directory -C$appdir info-boards-supported 2>/dev/null || echo broken)"

    if [ "$boards" = broken ]; then
        echo "makefile_broken"
        return
    fi

    for board in $boards
    do
        echo $board
    done | $(_greplist $BOARDS)
}

get_supported_toolchains() {
    local appdir=$1
    local board=$2
    local toolchains="gnu"

    if is_in_list "${board}" "${TEST_BOARDS_LLVM_COMPILE}"; then
        toolchains="$(make -s --no-print-directory -C${appdir} BOARD=${board} \
                      info-toolchains-supported 2> /dev/null | grep -o -e "llvm" -e "gnu")"
    fi
    echo "${toolchains}"
}

# given an app dir as parameter, print "$appdir $board:$toolchain" for each
# supported board and toolchain. Only print for boards in $BOARDS.
# if extra args are given, they will be prepended to each output line.
get_app_board_toolchain_pairs() {
    local appdir=$1
    local boards="$(get_supported_boards $appdir)"

    # collect extra arguments into prefix variable
    shift
    local prefix="$*"

    if [ "$boards" = makefile_broken ]; then
        echo "$appdir makefile_broken"
        return
    fi

    for board in ${boards}
    do
        for toolchain in $(get_supported_toolchains $appdir $board)
        do
            echo $prefix $appdir $board:$toolchain
        done
    done | $(_greplist $BOARDS)
}

# use dwqc to create full "appdir board toolchain" compile job list
get_compile_jobs() {
    check_label "CI: skip compile test" && return
    get_apps | \
        dwqc ${DWQ_ENV} -s \
        ${DWQ_JOBID:+--subjob} \
        "$0 get_app_board_toolchain_pairs \${1} $0 compile"
}

print_worker() {
    [ -n "$DWQ_WORKER" ] && \
        echo "-- running on worker ${DWQ_WORKER} thread ${DWQ_WORKER_THREAD}, build number $DWQ_WORKER_BUILDNUM."
}

test_hash_calc() {
    local bindir=$1

    # Why two times cut?
    # "test-input-hash.sha1" contains a list of lines containing
    # "<hash> <filename>" on each line.
    # We need to filter out the filename, as it contains the full path name,
    # which differs depending on the build machine.
    #
    # After piping through sha1sum, we get "<hash> -". " -" has to go so we save the
    # hassle of escaping the resulting hash.

    cat ${bindir}/test-input-hash.sha1 | cut -f1 -d' ' | sha1sum | cut -f1 -d' '
}

test_cache_get() {
    test "${ENABLE_TEST_CACHE}" = "1" || return 1
    test -n "$(redis-cli get $1)" > /dev/null
}

test_cache_put() {
    redis-cli set "$1" ok
}

# compile one app for one board with one toolchain. delete intermediates.
compile() {
    local appdir=$1
    local board=$(echo $2 | cut -f 1 -d':')
    local toolchain=$(echo $2 | cut -f 2 -d':')

    [ "$board" = "makefile_broken" ] && {
        echo "$0: There seems to be a problem in \"$appdir\" while getting supported boards!"
        echo "$0: testing \"make -C$appdir info-boards-supported\"..."
        make -C$appdir info-boards-supported && echo "$0: success. no idea what's wrong." || echo "$0: failed!"
        exit 1
    }

    # set build directory. CI ensures only one build at a time in $(pwd).
    export BINDIR="$(pwd)/build"
    export PKGDIRBASE="${BINDIR}/pkg"

    # Pre-build cleanup
    rm -rf ${BINDIR}

    print_worker

    # sanity checks
    [ $# -ne 2 ] && error "$0: compile: invalid parameters (expected \$appdir \$board:\$toolchain)"
    [ ! -d "$appdir" ] && error "$0: compile: error: application directory \"$appdir\" doesn't exist"

    # We compile a first time with Kconfig based dependency
    # resolution for regression purposes. $TEST_KCONFIG contains a
    # list of board-application tuples that are currently modeled to
    # run with Kconfig

    should_check_kconfig_hash=0

    for app in $(get_kconfig_test_apps "${board}")
    do
        if [ "${appdir}" = "${app}" ]; then
            should_check_kconfig_hash=1
            BOARD=${board} make -C${appdir} clean
            CCACHE_BASEDIR="$(pwd)" BOARD=${board} TOOLCHAIN=${toolchain} RIOT_CI_BUILD=1 TEST_KCONFIG=1 \
                            make -C${appdir} all test-input-hash -j${JOBS:-4}
            RES=$?
            if [ $RES -eq 0 ]; then
                kconfig_test_hash=$(test_hash_calc "${BINDIR}")
            else
                kconfig_test_hash=0
                echo "An error occurred while compiling using Kconfig";
            fi
            break
        fi
    done

    # compile without Kconfig
    CCACHE_BASEDIR="$(pwd)" BOARD=$board TOOLCHAIN=$toolchain RIOT_CI_BUILD=1 \
        make -C${appdir} clean all test-input-hash -j${JOBS:-4}
    RES=$?

    test_hash=$(test_hash_calc "$BINDIR")

    if [ ${should_check_kconfig_hash} != 0 ]; then
        if [ ${kconfig_test_hash} != ${test_hash} ]; then
            echo "Hashes of binaries with and without Kconfig mismatch for ${appdir}";
            echo "Please check that all used modules are modelled in Kconfig and enabled";
            RES=1
        fi
    fi

    # run tests
    if [ $RES -eq 0 ]; then
        if [ $RUN_TESTS -eq 1 -o "$board" = "native" ]; then
            if [ -f "${BINDIR}/.test" ]; then
                if [ "$board" = "native" ]; then
                    BOARD=$board make -C${appdir} test
                    RES=$?
                elif is_in_list "$board" "$TEST_BOARDS_AVAILABLE"; then
                    echo "-- test_hash=$test_hash"
                    if test_cache_get $test_hash; then
                        echo "-- skipping test due to positive cache hit"
                    else
                        BOARD=$board TOOLCHAIN=$toolchain TEST_HASH=$test_hash \
                            make -C${appdir} test-murdock
                        RES=$?
                    fi
                fi
            fi
        fi
    fi

    if [ -d ${BINDIR} ]
    then
        echo "-- build directory size: $(du -sh ${BINDIR} | cut -f1)"

        # cleanup
        rm -rf ${BINDIR}
    fi

    return $RES
}

test_job() {
    local appdir=$1
    local board=$(echo $2 | cut -f 1 -d':')
    local toolchain=$(echo $2 | cut -f 2 -d':')

    # interpret any extra arguments as file names.
    # They will be sent along with the job to the test worker
    # and stored in the application's binary folder.
    shift 2
    local files=""
    for filename in "$@"; do
        # check if the file is within $(BINDIR)
        if startswith "${BINDIR}" "${filename}"; then
          # get relative (to $(BINDIR) path
          local relpath="$(realpath --relative-to ${BINDIR} ${filename})"
        else
          error "$0: error: extra test files not within \${BINDIR}!"
        fi

        # set remote file path.
        # currently, the test workers build in the default build path.
        local remote_bindir="${appdir}/bin/${board}"
        files="${files} --file ${filename}:${remote_bindir}/${relpath}"
    done

    dwqc \
        ${DWQ_ENV} \
        ${DWQ_JOBID:+--subjob} \
        --queue ${TEST_QUEUE:-$board} \
        --maxfail 1 \
        $files \
        "./.murdock run_test $appdir $board:$toolchain"
}

run_test() {
    local appdir=$1
    local board=$(echo $2 | cut -f 1 -d':')
    local toolchain=$(echo $2 | cut -f 2 -d':')
    print_worker
    echo "-- executing tests for $appdir on $board (compiled with $toolchain toolchain):"
    hook run_test_pre

    # do flashing and building of termdeps simultaneously
    BOARD=$board TOOLCHAIN=${toolchain} make -C$appdir flash-only termdeps -j2

    # now run the actual test
    if is_in_list "${appdir}" "${TEST_WITH_CONFIG_SUPPORTED}"; then
        BOARD=${board} TOOLCHAIN=${toolchain} make -C${appdir} test-with-config
    else
        BOARD=$board TOOLCHAIN=${toolchain} make -C$appdir test
    fi
    RES=$?

    if [ $RES -eq 0 -a -n "$TEST_HASH" ]; then
        echo -n "-- saving test result to cache: "
        test_cache_put $TEST_HASH
    fi

    return $RES
}

# execute static tests
static_tests() {
    print_worker
    ./dist/tools/ci/static_tests.sh
}

get_non_compile_jobs() {
    [ "$STATIC_TESTS" = "1" ] && \
        echo "$0 static_tests"
}

get_jobs() {
    get_non_compile_jobs
    get_compile_jobs
}

$*