diff --git a/sys/Makefile b/sys/Makefile index aaa3528747..fc5c0633c2 100644 --- a/sys/Makefile +++ b/sys/Makefile @@ -173,6 +173,9 @@ endif ifneq (,$(filter test_utils_interactive_sync,$(USEMODULE))) DIRS += test_utils/interactive_sync endif +ifneq (,$(filter test_utils_result_output,$(USEMODULE))) + DIRS += test_utils/result_output +endif ifneq (,$(filter udp,$(USEMODULE))) DIRS += net/transport_layer/udp endif diff --git a/sys/Makefile.dep b/sys/Makefile.dep index 019d117ef1..0641aafa7b 100644 --- a/sys/Makefile.dep +++ b/sys/Makefile.dep @@ -1084,6 +1084,10 @@ ifneq (,$(filter suit_%,$(USEMODULE))) USEMODULE += suit endif +ifneq (,$(filter test_utils_result_output_%,$(USEMODULE))) + USEMODULE += test_utils_result_output +endif + # include ztimer dependencies ifneq (,$(filter ztimer% %ztimer,$(USEMODULE))) include $(RIOTBASE)/sys/ztimer/Makefile.dep diff --git a/sys/Makefile.include b/sys/Makefile.include index 058ed847e2..69207bfc82 100644 --- a/sys/Makefile.include +++ b/sys/Makefile.include @@ -129,6 +129,10 @@ ifneq (,$(filter zptr,$(USEMODULE))) include $(RIOTBASE)/sys/zptr/Makefile.include endif +ifneq (,$(filter test_utils_result_output,$(USEMODULE))) + include $(RIOTBASE)/sys/test_utils/result_output/Makefile.include +endif + # Convert xtimer into a pseudo module if its API is already implemented by # ztimer's compatibility wrapper ifneq (,$(filter ztimer_xtimer_compat,$(USEMODULE))) diff --git a/sys/include/test_utils/result_output.h b/sys/include/test_utils/result_output.h new file mode 100644 index 0000000000..da59e06b15 --- /dev/null +++ b/sys/include/test_utils/result_output.h @@ -0,0 +1,357 @@ +/* + * Copyright (C) 2021 HAW Hamburg + * + * This file is subject to the terms and conditions of the GNU Lesser + * General Public License v2.1. See the file LICENSE in the top level + * directory for more details. + */ + +/** + * @defgroup test_utils_result_output Test result output + * @ingroup sys + * @brief Utility function for abstraction of test result output format + * + * The TURO module provides an abstraction layer allowing salient data to be + * provided for tests independent of format or medium. The intention is to have + * a test that expects some data, for example, reading some registers, + * output the results in a know way, for example json. This should help + * keeping the test results stable and not lock anyone into a perticular + * format. If JSON is too heavy all tests using this can be swapped out for + * something lighter, for example CBOR. Then the tests should not have to be + * adapted. There can also be python layers that coordinate the output results, + * ideally done with riotctrl. + * + * Only one implementation should be selected, for example, + * test_utils_result_output_json. + * + * Some of the design decisions include: + * - ability to flush immediately to prevent the need for large buffers + * - selectable output format based on `USEMODULE`: + * + `test_utils_result_output_check`: @ref test_utils_result_output_check + * + `test_utils_result_output_json`: @ref test_utils_result_output_json + * + `test_utils_result_output_txt`: @ref test_utils_result_output_txt + * - exit status similar to a linux exit status. + * - readable raw output used in the CI to assist with reproducing errors + * - structure doesn't need to be enforced in every implementation to save + * bytes, see test_utils_result_output_check for structure assertions + * + * Some limitations are: + * - Only one type of result output implementation can be used at a time + * - Errors may be caused by the specific result output implementation making + * it difficult to debug + * + * @{ + * @file + * @brief Provides abstraction and convention for output of test results + * + * @author Kevin Weiss + */ + +#ifndef TEST_UTILS_RESULT_OUTPUT_H +#define TEST_UTILS_RESULT_OUTPUT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#include "result_output_types.h" + +/** @defgroup turo_API Test Utils Result Output Implementation API + * @brief The turo API that must have an implementation. + * @{ + */ + +/** + * @brief Type for a TURO object + * + * @note API implementors: `struct turo` needs to be defined by + * implementation-specific `result_output_types.h`. + */ +typedef struct turo turo_t; + +/** + * @brief Provides initial values for the turo context. + * + * @param[out] ctx The implementation specific turo context. + */ +void turo_init(turo_t *ctx); + +/** + * @brief Outputs a container open. + * + * Must be used when starting formatted result output. + * + * @param[in] ctx The implementation specific turo context. + */ +void turo_container_open(turo_t *ctx); + +/** + * @brief Outputs a signed 32 bit integer. + * + * @pre `turo_container_open` called + * + * @param[in] ctx The implementation specific turo context. + * @param[in] val The value to output. + */ +void turo_s32(turo_t *ctx, int32_t val); + +/** + * @brief Outputs an unsigned 32 bit integer. + * + * @pre `turo_container_open` called + * + * @param[in] ctx The implementation specific turo context. + * @param[in] val The value to output. + */ +void turo_u32(turo_t *ctx, uint32_t val); + +/** + * @brief Outputs a signed 64 bit integer. + * + * @pre `turo_container_open` called + * + * @param[in] ctx The implementation specific turo context. + * @param[in] val The value to output. + */ +void turo_s64(turo_t *ctx, int64_t val); + +/** + * @brief Outputs a formatted result unsigned 64 bit integer. + * + * @pre `turo_container_open` called + * + * @param[in] ctx The implementation specific turo context. + * @param[in] val The value to output. + */ +void turo_u64(turo_t *ctx, uint64_t val); + +/** + * @brief Outputs a formatted float result of varied precision. + * + * @pre `turo_container_open` called + * + * @param[in] ctx The implementation specific turo context. + * @param[in] val The value to output. + */ +void turo_float(turo_t *ctx, float val); + +/** + * @brief Outputs a formatted string string. + * + * @pre `turo_container_open` called + * + * @param[in] ctx The implementation specific turo context. + * @param[in] str The string to output. + */ +void turo_string(turo_t *ctx, const char *str); + +/** + * @brief Outputs a formatted boolean result. + * + * @pre `turo_container_open` called + * + * @param[in] ctx The implementation specific turo context. + * @param[in] val The value to output. + */ +void turo_bool(turo_t *ctx, bool val); + +/** + * @brief Outputs a formatted open of a dictionary result. + * + * A `turo_dict_close` must match. + * + * @pre `turo_container_open` called + * + * @param[in] ctx The implementation specific turo context. + */ +void turo_dict_open(turo_t *ctx); + +/** + * @brief Outputs a formatted open of a dictionary result. + * + * A turo value must follow. + * + * @pre `turo_container_open` called + * + * @param[in] ctx The implementation specific turo context. + * @param[in] key The key of the dictionary. + */ +void turo_dict_key(turo_t *ctx, const char *key); + +/** + * @brief Outputs a formatted close of a dictionary result. + * + * @pre `turo_container_open` called + * @pre `turo_dict_open` called + * + * @param[in] ctx The implementation specific turo context. + */ +void turo_dict_close(turo_t *ctx); + +/** + * @brief Outputs a formatted open of an array result. + * + * A `turo_array_close` must match. + * + * @pre `turo_container_open` called + * + * @param[in] ctx The implementation specific turo context. + */ +void turo_array_open(turo_t *ctx); + +/** + * @brief Outputs a formatted close of an array result. + * + * @pre `turo_container_open` called + * @pre `turo_array_open` called + * + * @param[in] ctx The implementation specific turo context. + */ +void turo_array_close(turo_t *ctx); + +/** + * @brief Outputs a formatted close of a container result. + * + * @pre `turo_container_open` called + * + * @param[in] ctx The implementation specific turo context. + * @param[in] exit_status Exit status code for the result, 0 is success. + */ +void turo_container_close(turo_t *ctx, int exit_status); +/** @} */ + +/** @defgroup turo_helpers Test Utils Result Output Helpers + * @brief Common functions and helpers that all implementations can use. + * @{ + */ + +/** + * @brief Outputs a formatted uint8 array result. + * + * @pre `turo_container_open` called + * + * @param[in] ctx The implementation specific turo context. + * @param[in] vals A buffer of data to output. + * @param[in] size The amount of elements to output. + */ +void turo_array_u8(turo_t *ctx, uint8_t *vals, size_t size); + +/** + * @brief Outputs a int32 array result. + * + * @pre `turo_container_open` called + * + * @param[in] ctx The implementation specific turo context. + * @param[in] vals A buffer of data to output. + * @param[in] size The amount of elements to output. + */ +void turo_array_s32(turo_t *ctx, int32_t *vals, size_t size); + +/** + * @brief Outputs a dict with string data. + * + * @pre `turo_container_open` called + * + * @param[in] ctx The implementation specific turo context. + * @param[in] key A dictionary key. + * @param[in] val A string value of the dictionary + */ +void turo_dict_string(turo_t *ctx, const char *key, const char *val); + +/** + * @brief Outputs a dict with integer data. + * + * @pre `turo_container_open` called + * + * @param[in] ctx The implementation specific turo context. + * @param[in] key A dictionary key. + * @param[in] val The integer value of the dictionary. + */ +void turo_dict_s32(turo_t *ctx, const char *key, int32_t val); + +/** + * @brief Outputs a full successful int32 turo result. + * + * This includes all opening and closing of turo elements. + * + * @pre turo ctx initialized + * + * @param[in] ctx The implementation specific turo context. + * @param[in] val The value to output. + */ +void turo_simple_s32(turo_t *ctx, int32_t val); + +/** + * @brief Outputs a full successful uint8 array turo result. + * + * This includes all opening and closing of turo elements. + * + * @pre turo ctx initialized + * + * @param[in] ctx The implementation specific turo context. + * @param[in] vals The buffer of the integers. + * @param[in] size Number of elements in the array. + */ +void turo_simple_array_u8(turo_t *ctx, uint8_t *vals, size_t size); + +/** + * @brief Outputs a full successful int32 array turo result. + * + * This includes all opening and closing of turo elements. + * + * @pre turo ctx initialized + * + * @param[in] ctx The implementation specific turo context. + * @param[in] vals The buffer of the integers. + * @param[in] size Number of elements in the array. + */ +void turo_simple_array_s32(turo_t *ctx, int32_t *vals, size_t size); + +/** + * @brief Outputs a full successful dict with string turo result. + * + * This includes all opening and closing of turo elements. + * + * @pre turo ctx initialized + * + * @param[in] ctx The implementation specific turo context. + * @param[in] key The dictionary key. + * @param[in] val The string value. + */ +void turo_simple_dict_string(turo_t *ctx, const char *key, const char *val); + +/** + * @brief Outputs a full successful dict with an integer turo result. + * + * This includes all opening and closing of turo elements. + * + * @pre turo ctx initialized + * + * @param[in] ctx The implementation specific turo context. + * @param[in] key The dictionary key. + * @param[in] val The integer value. + */ +void turo_simple_dict_s32(turo_t *ctx, const char *key, int32_t val); + +/** + * @brief Outputs a full turo result with exit code. + * + * This includes all opening and closing of turo elements. + * + * @pre turo ctx initialized + * + * @param[in] ctx The implementation specific turo context. + * @param[in] exit_status The exit status to output. + */ +void turo_simple_exit_status(turo_t *ctx, int exit_status); +/** @} */ + +#ifdef __cplusplus +} +#endif +#endif /* TEST_UTILS_RESULT_OUTPUT_H */ +/** @} */ diff --git a/sys/test_utils/Makefile.dep b/sys/test_utils/Makefile.dep index 4bafa543b8..a3a1635266 100644 --- a/sys/test_utils/Makefile.dep +++ b/sys/test_utils/Makefile.dep @@ -1,3 +1,6 @@ ifneq (,$(filter test_utils_interactive_sync,$(USEMODULE))) USEMODULE += stdin endif +ifneq (,$(filter test_utils_result_output_%,$(USEMODULE))) + USEMODULE += fmt +endif diff --git a/sys/test_utils/result_output/Makefile b/sys/test_utils/result_output/Makefile new file mode 100644 index 0000000000..d95e7fca9a --- /dev/null +++ b/sys/test_utils/result_output/Makefile @@ -0,0 +1,15 @@ +MODULE = test_utils_result_output + +ifneq (,$(filter test_utils_result_output_json,$(USEMODULE))) + DIRS += json +endif + +ifneq (,$(filter test_utils_result_output_txt,$(USEMODULE))) + DIRS += txt +endif + +ifneq (,$(filter test_utils_result_output_check,$(USEMODULE))) + DIRS += check +endif + +include $(RIOTBASE)/Makefile.base diff --git a/sys/test_utils/result_output/Makefile.include b/sys/test_utils/result_output/Makefile.include new file mode 100644 index 0000000000..5b9ebd9a7a --- /dev/null +++ b/sys/test_utils/result_output/Makefile.include @@ -0,0 +1,9 @@ +ifneq (,$(filter test_utils_result_output_json,$(USEMODULE))) + INCLUDES += -I$(RIOTBASE)/sys/test_utils/result_output/json +endif +ifneq (,$(filter test_utils_result_output_txt,$(USEMODULE))) + INCLUDES += -I$(RIOTBASE)/sys/test_utils/result_output/txt +endif +ifneq (,$(filter test_utils_result_output_check,$(USEMODULE))) + INCLUDES += -I$(RIOTBASE)/sys/test_utils/result_output/check +endif diff --git a/sys/test_utils/result_output/result_output.c b/sys/test_utils/result_output/result_output.c new file mode 100644 index 0000000000..3aeb6727da --- /dev/null +++ b/sys/test_utils/result_output/result_output.c @@ -0,0 +1,113 @@ +/* + * Copyright (C) 2021 HAW Hamburg + * + * This file is subject to the terms and conditions of the GNU Lesser + * General Public License v2.1. See the file LICENSE in the top level + * directory for more details. + */ + +/** + * @ingroup sys + * @{ + * + * @file + * @brief Result output common implementation + * + * + * @author Kevin Weiss + * @} + */ + +#include +#include + +#include "test_utils/result_output.h" + +void __attribute__((weak)) turo_init(turo_t *ctx) +{ + (void)ctx; +} + +void turo_array_u8(turo_t *ctx, uint8_t *vals, size_t size) +{ + turo_array_open(ctx); + while (size > 0) { + turo_u32(ctx, (uint32_t)*vals); + vals++; + size--; + } + turo_array_close(ctx); +} + +void turo_array_s32(turo_t *ctx, int32_t *vals, size_t size) +{ + turo_array_open(ctx); + while (size-- > 0) { + turo_s32(ctx, *vals); + vals++; + } + turo_array_close(ctx); +} + +void turo_dict_string(turo_t *ctx, const char *key, const char *val) +{ + turo_dict_open(ctx); + turo_dict_key(ctx, key); + turo_string(ctx, val); + turo_dict_close(ctx); +} + +void turo_dict_s32(turo_t *ctx, const char *key, int32_t val) +{ + turo_dict_open(ctx); + turo_dict_key(ctx, key); + turo_s32(ctx, val); + turo_dict_close(ctx); +} + +void turo_simple_s32(turo_t *ctx, int32_t val) +{ + turo_container_open(ctx); + turo_s32(ctx, val); + turo_container_close(ctx, 0); +} + +void turo_simple_array_s32(turo_t *ctx, int32_t *vals, size_t size) +{ + turo_container_open(ctx); + turo_array_s32(ctx, vals, size); + turo_container_close(ctx, 0); +} + +void turo_simple_array_u8(turo_t *ctx, uint8_t *vals, size_t size) +{ + turo_container_open(ctx); + turo_array_u8(ctx, vals, size); + turo_container_close(ctx, 0); +} + +void turo_simple_dict_string(turo_t *ctx, const char *key, const char *val) +{ + turo_container_open(ctx); + turo_dict_open(ctx); + turo_dict_key(ctx, key); + turo_string(ctx, val); + turo_dict_close(ctx); + turo_container_close(ctx, 0); +} + +void turo_simple_dict_s32(turo_t *ctx, const char *key, int32_t val) +{ + turo_container_open(ctx); + turo_dict_open(ctx); + turo_dict_key(ctx, key); + turo_s32(ctx, val); + turo_dict_close(ctx); + turo_container_close(ctx, 0); +} + +void turo_simple_exit_status(turo_t *ctx, int exit_status) +{ + turo_container_open(ctx); + turo_container_close(ctx, exit_status); +}