Merge pull request #17178 from aabadie/pr/pkg/lz4

pkg/lz4: add support for LZ4 compression/decompression

pkg/Kconfig

@@ -29,6 +29,7 @@ rsource "libfixmath/Kconfig"
 rsource "libhydrogen/Kconfig"
 rsource "lora-serialization/Kconfig"
 rsource "lvgl/Kconfig"
+rsource "lz4/Kconfig"
 rsource "micro-ecc/Kconfig"
 rsource "microcoap/Kconfig"
 rsource "minmea/Kconfig"

pkg/lz4/Kconfig

@@ -0,0 +1,9 @@
+# Copyright (c) 2021 Inria
+#
+# This file is subject to the terms and conditions of the GNU Lesser
+# General Public License v2.1. See the file LICENSE in the top level
+# directory for more details.
+
+config PACKAGE_LZ4
+    bool "LZ4 fast compression library"
+    depends on TEST_KCONFIG

pkg/lz4/Makefile

@@ -0,0 +1,9 @@
+PKG_NAME=lz4
+PKG_URL=https://github.com/lz4/lz4
+PKG_VERSION=d44371841a2f1728a3f36839fd4b7e872d0927d3  # v1.9.3
+PKG_LICENSE=BSD
+
+include $(RIOTBASE)/pkg/pkg.mk
+
+all:
+	$(QQ)"$(MAKE)" -f $(CURDIR)/$(PKG_NAME).mk -C $(PKG_SOURCE_DIR)/lib

pkg/lz4/Makefile.dep

@@ -0,0 +1,2 @@
+# lz4 cannot be built on 8bit and 16bit architectures
+FEATURES_BLACKLIST += arch_8bit arch_16bit

pkg/lz4/Makefile.include

@@ -0,0 +1 @@
+INCLUDES += -I$(PKGDIRBASE)/lz4/lib

pkg/lz4/doc.txt

@@ -0,0 +1,19 @@
+/**
+ * @defgroup pkg_lz4   LZ4 compression library
+ * @ingroup  pkg
+ * @brief    LZ4 is a fast compression library
+ *
+ * # Note about memory
+ *
+ * By default, the LZ4 implementation allocates memory on the stack. This
+ * way compression/decompression functions are fast but this comes at the cost
+ * of RAM usage.
+ * It is important to allocated enough stack memory (at least 16384 bytes) to
+ * any thread using LZ4 primitives.
+ *
+ * # License
+ *
+ * Licensed under BSD 2-clause.
+ *
+ * @see      https://github.com/lz4/lz4
+ */

pkg/lz4/lz4.mk

@@ -0,0 +1,8 @@
+MODULE = lz4
+
+SRC := \
+    lz4.c \
+    lz4hc.c \
+	#
+
+include $(RIOTBASE)/Makefile.base

tests/pkg_lz4/Makefile

@@ -0,0 +1,14 @@
+include ../Makefile.tests_common
+
+USEPKG += lz4
+
+# lz4 requires a custom thread stack size
+CFLAGS += -DTHREAD_STACKSIZE_DEFAULT=16384
+
+# stk3200 and nucleo-l011k4 boards don't allow a custom default thread stacksize
+BOARD_BLACKLIST := \
+    nucleo-l011k4 \
+    stk3200 \
+    #
+
+include $(RIOTBASE)/Makefile.include

tests/pkg_lz4/Makefile.ci

@@ -0,0 +1,49 @@
+BOARD_INSUFFICIENT_MEMORY := \
+    airfy-beacon \
+    arduino-mkr1000 \
+    arduino-mkrfox1200 \
+    arduino-mkrwan1300 \
+    arduino-mkrzero \
+    arduino-nano-33-iot \
+    bastwan \
+    bluepill-stm32f030c8 \
+    calliope-mini \
+    feather-m0 \
+    feather-m0-lora \
+    feather-m0-wifi \
+    hifive1 \
+    hifive1b \
+    i-nucleo-lrwan1 \
+    im880b \
+    microbit \
+    nrf51dongle \
+    nrf6310 \
+    nucleo-f030r8 \
+    nucleo-f031k6 \
+    nucleo-f042k6 \
+    nucleo-f070rb \
+    nucleo-f072rb \
+    nucleo-f303k8 \
+    nucleo-f334r8 \
+    nucleo-l031k6 \
+    nucleo-l053r8 \
+    samd10-xmini \
+    saml10-xpro \
+    saml11-xpro \
+    seeeduino_xiao \
+    sensebox_samd21 \
+    serpente \
+    slstk3400a \
+    sodaq-autonomo \
+    sodaq-explorer \
+    sodaq-one \
+    sodaq-sara-aff \
+    sodaq-sara-sff \
+    stm32f030f4-demo \
+    stm32f0discovery \
+    stm32g0316-disco \
+    stm32l0538-disco \
+    wemos-zero \
+    yarm \
+    yunjia-nrf51822 \
+    #

tests/pkg_lz4/app.config.test

@@ -0,0 +1 @@
+CONFIG_PACKAGE_LZ4=y

tests/pkg_lz4/main.c

@@ -0,0 +1,95 @@
+/*
+ * Copyright (C) 2021 Inria
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @ingroup     tests
+ * @{
+ *
+ * @file
+ * @brief       lz4 package test application
+ *
+ * This application is adapted to RIOT from
+ * https://github.com/lz4/lz4/blob/dev/examples/simple_buffer.c
+ *
+ * @author      Alexandre Abadie <alexandre.abadie@inria.fr>
+ *
+ * @}
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include "lz4.h"
+
+#define BUFFER_SIZE 1024
+
+static char compressed_data[BUFFER_SIZE];
+static char regen_buffer[BUFFER_SIZE];
+
+static const char* const src =  "Lorem ipsum dolor sit amet, consectetur "
+                                "adipiscing elit, sed do eiusmod tempor "
+                                "incididunt ut labore et dolore magna aliqua. "
+                                "Nisl vel pretium lectus quam id leo. Volutpat "
+                                "sed cras ornare arcu dui vivamus arcu felis "
+                                "bibendum. Velit scelerisque in dictum non "
+                                "consectetur a erat nam. Pretium viverra "
+                                "suspendisse potenti nullam ac tortor vitae "
+                                "purus faucibus. Tristique et egestas quis "
+                                "ipsum suspendisse. At imperdiet dui accumsan "
+                                "sit amet nulla facilisi. Pulvinar neque "
+                                "laoreet suspendisse interdum consectetur "
+                                "libero. Vulputate sapien nec sagittis aliquam "
+                                "malesuada bibendum arcu vitae. Sed adipiscing "
+                                "diam donec adipiscing tristique risus nec. "
+                                "Venenatis tellus in metus vulputate eu "
+                                "scelerisque. Id faucibus nisl tincidunt eget "
+                                "nullam non nisi est. Integer feugiat "
+                                "scelerisque varus morbi enim. Est sit amet "
+                                "facilisis magna etiam. Venenatis cras sed "
+                                "felis eget velit aliquet sagittis.";
+
+int main(void)
+{
+    /* Compression */
+    const unsigned src_size = strlen(src) + 1;
+
+    const int compressed_data_size = LZ4_compress_default(src, compressed_data,
+                                                          src_size, BUFFER_SIZE);
+    compressed_data[compressed_data_size] = 0;
+    /* Check return_value to determine what happened. */
+    if (compressed_data_size <= 0) {
+        puts("Compression failed.");
+        return -1;
+    }
+
+    unsigned compression_ratio = ((unsigned)compressed_data_size * 10000) / src_size;
+    printf("Data compressed with success (ratio: %u.%u)\n",
+           compression_ratio / 10000, (compression_ratio / 100) % 100);
+
+    /* Decompression */
+    const int decompressed_size = LZ4_decompress_safe(compressed_data, regen_buffer,
+                                                      compressed_data_size, src_size);
+    if (decompressed_size < 0) {
+        puts("Decompression failed.");
+        return -1;
+    }
+    puts("Data decompressed with success!");
+
+    if ((unsigned)decompressed_size != src_size) {
+        puts("Decompressed data is different from original");
+        return -1;
+    }
+
+    /* Validation */
+    if (memcmp(src, regen_buffer, src_size) != 0) {
+        puts("Validation failed");
+        return -1;
+    }
+
+    printf("Validation done, decompressed string:\n%s\n", regen_buffer);
+    return 0;
+}

tests/pkg_lz4/tests/01-run.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2021 Inria
+#
+# This file is subject to the terms and conditions of the GNU Lesser
+# General Public License v2.1. See the file LICENSE in the top level
+# directory for more details.
+
+import sys
+from testrunner import run
+
+
+def testfunc(child):
+    child.expect(r"Data compressed with success \(ratio: (\d+.\d+)\)\r\n")
+    ratio = float(child.match.group(1))
+    assert ratio < 1, "No compression (ratio: {})".format(ratio)
+    child.expect_exact("Data decompressed with success!")
+    child.expect_exact("Validation done, decompressed string:")
+    child.expect_exact(
+        "Lorem ipsum dolor sit amet, consectetur "
+        "adipiscing elit, sed do eiusmod tempor "
+        "incididunt ut labore et dolore magna aliqua. "
+        "Nisl vel pretium lectus quam id leo. Volutpat "
+        "sed cras ornare arcu dui vivamus arcu felis "
+        "bibendum. Velit scelerisque in dictum non "
+        "consectetur a erat nam. Pretium viverra "
+        "suspendisse potenti nullam ac tortor vitae "
+        "purus faucibus. Tristique et egestas quis "
+        "ipsum suspendisse. At imperdiet dui accumsan "
+        "sit amet nulla facilisi. Pulvinar neque "
+        "laoreet suspendisse interdum consectetur "
+        "libero. Vulputate sapien nec sagittis aliquam "
+        "malesuada bibendum arcu vitae. Sed adipiscing "
+        "diam donec adipiscing tristique risus nec. "
+        "Venenatis tellus in metus vulputate eu "
+        "scelerisque. Id faucibus nisl tincidunt eget "
+        "nullam non nisi est. Integer feugiat "
+        "scelerisque varus morbi enim. Est sit amet "
+        "facilisis magna etiam. Venenatis cras sed "
+        "felis eget velit aliquet sagittis."
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(run(testfunc))