diff --git a/sys/Makefile.dep b/sys/Makefile.dep index 9839575e4b..d1af5218fd 100644 --- a/sys/Makefile.dep +++ b/sys/Makefile.dep @@ -33,6 +33,10 @@ ifneq (,$(filter auto_init_sock_dns,$(USEMODULE))) endif endif +ifneq (,$(filter coding,$(USEMODULE))) + USEMODULE += bitfield +endif + ifneq (,$(filter congure_%,$(USEMODULE))) USEMODULE += congure endif diff --git a/sys/coding/Makefile b/sys/coding/Makefile new file mode 100644 index 0000000000..48422e909a --- /dev/null +++ b/sys/coding/Makefile @@ -0,0 +1 @@ +include $(RIOTBASE)/Makefile.base diff --git a/sys/coding/doc.txt b/sys/coding/doc.txt new file mode 100644 index 0000000000..fb573a9939 --- /dev/null +++ b/sys/coding/doc.txt @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2021 Benjamin Valentin + * + * This file is subject to the terms and conditions of the GNU Lesser + * General Public License v2.1. See the file LICENSE in the top level + * directory for more details. + */ + +/** + * @defgroup sys_coding Error correction codes + * @ingroup sys + * @brief Error correction function libraries + * + * This module provides functions to generate redundancy data for error + * correction. + */ diff --git a/sys/coding/xor.c b/sys/coding/xor.c new file mode 100644 index 0000000000..2b30b02e36 --- /dev/null +++ b/sys/coding/xor.c @@ -0,0 +1,203 @@ +/* + * Copyright (C) 2021 Benjamin Valentin + * + * This file is subject to the terms and conditions of the GNU Lesser + * General Public License v2.1. See the file LICENSE in the top level + * directory for more details. + */ + +/** + * @ingroup sys_coding_xor + * @brief XOR coding algorithm + * + * @{ + * + * @file + * @brief XOR coding implementation + * + * @author Benjamin Valentin + */ + +#include +#include "bitfield.h" +#include "coding/xor.h" + +#define ENABLE_DEBUG 0 +#include "debug.h" + +static void _gen_parity(const void *data, size_t len, uint8_t *out) +{ + const uint8_t *in = data; + + memset(out, 0, CODING_XOR_PARITY_LEN(len)); + for (unsigned i = 0; i < len; ++i) { + out[i / CONFIG_CODING_XOR_CHECK_BYTES] ^= in[i]; + } +} + +static inline size_t _transpose_idx(size_t i, size_t width, size_t height) +{ + size_t x = i % width; + size_t y = i / width; + + return x * height + y; +} + +static inline void _swap(uint8_t *a, uint8_t *b) +{ + uint8_t tmp = *a; + *a = *b; + *b = tmp; +} + +/* https://www.geeksforgeeks.org/inplace-m-x-n-size-matrix-transpose/ */ +static void _transpose(uint8_t *data, size_t len, size_t height) +{ + BITFIELD(visited, len); + memset(visited, 0, sizeof(visited)); + + /* A[0] and A[size-1] won't move */ + size_t i = 1; + size_t last = len -1; + bf_set(visited, 0); + bf_set(visited, last); + + while (i < last) { + size_t cycle_begin = i; + uint8_t tmp = data[i]; + do { + /* Input matrix [r x c] + * Output matrix + * i_new = (i*r)%(N-1) + */ + size_t next = (i * height) % last; + _swap(&data[next], &tmp); + bf_set(visited, i); + i = next; + } while (i != cycle_begin); + + /* Get Next Move (what about querying random location?) */ + i = bf_find_first_unset(visited, len); + } +} + +static inline void _mix(void *data, size_t len) +{ + _transpose(data, len, len / CONFIG_CODING_XOR_CHECK_BYTES); +} + +static inline void _unmix(void *data, size_t len) +{ + _transpose(data, len, CONFIG_CODING_XOR_CHECK_BYTES); +} + +static bool _recover_byte(const uint8_t *in, size_t width, uint8_t height, + const uint8_t *parity, size_t idx, uint8_t *bitfield, + size_t block_size, uint8_t *out) +{ + uint8_t res = parity[idx / CONFIG_CODING_XOR_CHECK_BYTES]; + size_t start = idx - idx % CONFIG_CODING_XOR_CHECK_BYTES; + + for (unsigned i = start; i < start + CONFIG_CODING_XOR_CHECK_BYTES; ++i) { + /* skip the lost byte */ + if (i == idx) { + continue; + } + + /* get index of neighbor byte in transposed matrix */ + size_t idx_in = _transpose_idx(i, height, width); + if (!bf_isset(bitfield, idx_in / block_size)) { + DEBUG("missing chunk %u\n", idx_in / block_size); + return false; + } + res ^= in[idx_in]; + } + + *out = res; + return true; +} + +static bool _recover_blocks(void *data, size_t len, const uint8_t *parity, + uint8_t *bitfield, size_t block_size) +{ + uint8_t *in = data; + + const uint8_t height = CONFIG_CODING_XOR_CHECK_BYTES; + const size_t width = len / height; + const uint8_t num_data_blocks = len / block_size; + bool success = true; + + for (size_t i = 0; i < len; i += block_size) { + /* block is present, nothing to do */ + if (bf_isset(bitfield, i / block_size)) { + continue; + } + + DEBUG("try to recover chunk %u / %u\n", i / block_size, num_data_blocks); + for (size_t j = i; j < i + block_size; ++j) { + + /* get original byte position */ + size_t idx = _transpose_idx(j, width, height); + + /* we can only recover the byte if we have the matching parity block */ + size_t parity_block = idx / (CONFIG_CODING_XOR_CHECK_BYTES * block_size); + if (!bf_isset(bitfield, num_data_blocks + parity_block)) { + DEBUG("missing parity block %u\n", parity_block); + success = false; + goto next_block; + } + + /* try to recover lost byte from parity */ + if (!_recover_byte(in, width, height, parity, idx, + bitfield, block_size, &in[j])) { + success = false; + goto next_block; + } + } + bf_set(bitfield, i / block_size); + +next_block: + /* try to recover another block */ ; + } + + return success; +} + +void coding_xor_generate(void *data, size_t len, uint8_t *parity) +{ + _gen_parity(data, len, parity); + _mix(data, len); +} + +bool coding_xor_recover(void *data, size_t len, uint8_t *parity, + uint8_t *bitfield, size_t block_size, bool recover_parity) +{ + size_t num_data_chunks = len / block_size; + size_t num_parity_chunks = CODING_XOR_PARITY_LEN(len) / block_size; + + if (!_recover_blocks(data, len, parity, bitfield, block_size)) { + return false; + } + + _unmix(data, len); + + if (!recover_parity) { + return true; + } + + /* recover lost parity blocks */ + for (size_t i = 0; i < num_parity_chunks; ++i) { + if (bf_isset(bitfield, num_data_chunks + i)) { + continue; + } + + DEBUG("regenerate parity block %u\n", i); + size_t data_len = block_size * CONFIG_CODING_XOR_CHECK_BYTES; + _gen_parity((uint8_t *)data + i * data_len, + data_len, parity + i * block_size); + } + + return true; +} + +/** @} */ diff --git a/sys/include/coding/xor.h b/sys/include/coding/xor.h new file mode 100644 index 0000000000..37ca60d556 --- /dev/null +++ b/sys/include/coding/xor.h @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2021 Benjamin Valentin + * + * This file is subject to the terms and conditions of the GNU Lesser + * General Public License v2.1. See the file LICENSE in the top level + * directory for more details. + */ + +/** + * @defgroup sys_coding_xor XOR code + * @ingroup sys_coding + * @brief Simple XOR based coding algorithms + * + * @experimental This is a very basic implementation, it can only recover + * 1 lost block in 3 and only has a 33% chance of recovering + * two consecutive lost blocks. + * API / Algorithm might change if that means we can do better. + * + * @{ + * + * @file + * @brief XOR coding definitions + * + * @author Benjamin Valentin + */ + +#ifndef CODING_XOR_H +#define CODING_XOR_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Number of payload bytes per parity byte + */ +#ifndef CONFIG_CODING_XOR_CHECK_BYTES +#define CONFIG_CODING_XOR_CHECK_BYTES 3U +#endif + +/** + * @brief Get the size of the needed parity buffer for a given payload size + * + * @param in Payload length + */ +#define CODING_XOR_PARITY_LEN(in) (((in) + CONFIG_CODING_XOR_CHECK_BYTES - 1) \ + / CONFIG_CODING_XOR_CHECK_BYTES) + +/** + * @brief Generate parity and mix data buffer + * + * This generates parity data to recover one in @ref CONFIG_CODING_XOR_CHECK_BYTES + * bytes. + * The data buffer is then mixed to distribute bytes amongst transfer blocks, so + * that the chance for consecutive bytes to be in the same block is lowered. + * + * @param[in,out] data The data buffer to be processed + * @param[in] len Size of the data buffer + * @param[out] parity Buffer to hold parity data. + * Must be at least `CODING_XOR_PARITY_LEN(len)` bytes + */ +void coding_xor_generate(void *data, size_t len, uint8_t *parity); + +/** + * @brief Restore and unmix buffer + * + * + * @param[in, out] data The data buffer to be restored + * @param[in] len Size of the data buffer + * @param[in,out] parity Buffer with parity data. + * Must be at least `CODING_XOR_PARITY_LEN(len)` bytes + * @param[in,out] blocks Bitfieled to indicate which blocks were received. + * This indicates the presence of both data and parity + * blocks. Parity blocks are appended after the last + * data block. + * If a block was restored it's bit will be set. + * @param[in] block_size Size of a data/payload block + * @param[in] recover_parity If true, missing parity blocks will be re-generated + * from data blocks. + * + * @return True if all data blocks were recovered + */ +bool coding_xor_recover(void *data, size_t len, uint8_t *parity, + uint8_t *blocks, size_t block_size, bool recover_parity); +#ifdef __cplusplus +} +#endif + +#endif /* CODING_XOR_H */ +/** @} */