Merge pull request #17179 from aabadie/pr/pkg/uzlib

pkg/uzlib: add support for zlib compression/decompression

.murdock

@@ -53,6 +53,7 @@ tests/driver_u*
 tests/driver_v*
 tests/periph_*
 tests/pkg_elk
+tests/pkg_uzlib
 tests/prng_*
 tests/xtimer_*
 tests/ztimer_*

pkg/Kconfig

@@ -55,6 +55,7 @@ rsource "umorse/Kconfig"
 rsource "utensor/Kconfig"
 rsource "uwb-core/Kconfig"
 rsource "uwb-dw1000/Kconfig"
+rsource "uzlib/Kconfig"
 rsource "wakaama/Kconfig"
 rsource "yxml/Kconfig"
 

pkg/uzlib/Kconfig

@@ -0,0 +1,9 @@
+# Copyright (c) 2021 Inria
+#
+# This file is subject to the terms and conditions of the GNU Lesser
+# General Public License v2.1. See the file LICENSE in the top level
+# directory for more details.
+
+config PACKAGE_UZLIB
+    bool "uzlib - Deflate/Zlib-compatible LZ77 compression/decompression library"
+    depends on TEST_KCONFIG

pkg/uzlib/Makefile

@@ -0,0 +1,11 @@
+PKG_NAME=uzlib
+PKG_URL=https://github.com/pfalcon/uzlib
+PKG_VERSION=27e4f4c15ba30c2cfc89575159e8efb50f95037e  # v2.9.5
+PKG_LICENSE=zlib
+
+include $(RIOTBASE)/pkg/pkg.mk
+
+CFLAGS += -Wno-sign-compare
+
+all:
+	$(QQ)"$(MAKE)" -C $(PKG_SOURCE_DIR)/src -f $(CURDIR)/$(PKG_NAME).mk

pkg/uzlib/Makefile.dep

@@ -0,0 +1,2 @@
+# uzlib contains code incompatible with 8bit and 16bit architectures
+FEATURES_BLACKLIST += arch_8bit arch_16bit

pkg/uzlib/Makefile.include

@@ -0,0 +1 @@
+INCLUDES += -I$(PKGDIRBASE)/uzlib/src

pkg/uzlib/doc.txt

@@ -0,0 +1,11 @@
+/**
+ * @defgroup pkg_uzlib   uzlib
+ * @ingroup  pkg
+ * @brief    Deflate/Zlib-compatible LZ77 compression/decompression library
+ *
+ * # License
+ *
+ * Licensed under zlib.
+ *
+ * @see      https://github.com/pfalcon/uzlib
+ */

pkg/uzlib/uzlib.mk

@@ -0,0 +1,3 @@
+MODULE = uzlib
+
+include $(RIOTBASE)/Makefile.base

tests/pkg_uzlib/Makefile

@@ -0,0 +1,9 @@
+include ../Makefile.tests_common
+
+# required packages
+USEPKG += uzlib
+
+# This application requires a large stacksize for the main thread
+CFLAGS += -DTHREAD_STACKSIZE_MAIN=2048
+
+include $(RIOTBASE)/Makefile.include

tests/pkg_uzlib/Makefile.ci

@@ -0,0 +1,7 @@
+BOARD_INSUFFICIENT_MEMORY := \
+    nucleo-f031k6 \
+    nucleo-l011k4 \
+    samd10-xmini \
+    stk3200 \
+    stm32f030f4-demo \
+    #

tests/pkg_uzlib/app.config.test

@@ -0,0 +1 @@
+CONFIG_PACKAGE_UZLIB=y

tests/pkg_uzlib/main.c

@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2021 Inria
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @ingroup     tests
+ * @{
+ *
+ * @file
+ * @brief       Test application for the uzlib package
+ *
+ * This application is adapted to RIOT from the uzlib examples:
+ * tgzip: https://github.com/pfalcon/uzlib/blob/master/examples/tgzip/tgzip.c
+ * tgunzip: https://github.com/pfalcon/uzlib/blob/master/examples/tgunzip/tgunzip.c
+ *
+ * @author      Alexandre Abadie <alexandre.abadie@inria.fr>
+ *
+ * @}
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <inttypes.h>
+
+#include "byteorder.h"
+#include "uzlib.h"
+
+/* The hash table size is computed from the data type (8 bit) multiplied by the
+ * number of possible values (1 << hash_bits, with hash_bits set to 6 in
+ * this application) */
+#define HASH_TABLE_SIZE     (8 * 64)
+#define CHUNK_SIZE          (1)
+static uzlib_hash_entry_t hash_table[HASH_TABLE_SIZE];
+
+static const char *message =    "Lorem ipsum dolor sit amet, consectetur "
+                                "adipiscing elit, sed do eiusmod tempor "
+                                "incididunt ut labore et dolore magna aliqua. "
+                                "Nisl vel pretium lectus quam id leo. Volutpat "
+                                "sed cras ornare arcu dui vivamus arcu felis "
+                                "bibendum. Velit scelerisque in dictum non "
+                                "consectetur a erat nam. Pretium viverra "
+                                "suspendisse potenti nullam ac tortor vitae "
+                                "purus faucibus. Tristique et egestas quis "
+                                "ipsum suspendisse. At imperdiet dui accumsan "
+                                "sit amet nulla facilisi. Pulvinar neque "
+                                "laoreet suspendisse interdum consectetur "
+                                "libero. Vulputate sapien nec sagittis aliquam "
+                                "malesuada bibendum arcu vitae. Sed adipiscing "
+                                "diam donec adipiscing tristique risus nec. "
+                                "Venenatis tellus in metus vulputate eu "
+                                "scelerisque. Id faucibus nisl tincidunt eget "
+                                "nullam non nisi est. Integer feugiat "
+                                "scelerisque varus morbi enim. Est sit amet "
+                                "facilisis magna etiam. Venenatis cras sed "
+                                "felis eget velit aliquet sagittis.";
+static unsigned char decompressed_output[1024];
+
+int main(void)
+{
+    /* Compress the message */
+    struct uzlib_comp comp = {0};
+    comp.dict_size = 32768;
+    comp.hash_bits = 6;
+    comp.hash_table = hash_table;
+
+    unsigned message_len = strlen(message);
+
+    zlib_start_block(&comp.out);
+    uzlib_compress(&comp, (uint8_t *)message, message_len);
+    zlib_finish_block(&comp.out);
+
+    unsigned compression_ratio = (comp.out.outlen * 10000) / message_len;
+    printf("Compressing message (len: %u): \"%s\"\n", message_len, message);
+    printf("Message compressed to %u raw bytes (ratio: %u.%u)\n",
+           comp.out.outlen,
+           compression_ratio / 10000, (compression_ratio / 100) % 100);
+
+    /* Decompress */
+    uzlib_init();
+    unsigned int dlen;
+
+    /* Get decompressed length */
+    dlen = unaligned_get_u32(&comp.out.outbuf[comp.out.outlen - 4]);
+
+    struct uzlib_uncomp d = {
+        /* all 3 fields below must be initialized by user */
+        .source = comp.out.outbuf,
+        .source_limit = comp.out.outbuf + comp.out.outlen - 1,
+        .dest_start = d.dest = decompressed_output,
+    };
+    uzlib_uncompress_init(&d, NULL, 0);
+    d.source_read_cb = NULL;
+
+    while (dlen) {
+        unsigned int chunk_len = dlen < CHUNK_SIZE ? dlen : CHUNK_SIZE;
+        d.dest_limit = d.dest + chunk_len;
+        if (uzlib_uncompress_chksum(&d) != TINF_OK) {
+            break;
+        }
+        dlen -= chunk_len;
+    }
+
+    uint16_t dsize = d.dest - decompressed_output;
+    printf("Decompressed message (len: %i): \"%s\"\n",
+           dsize, decompressed_output);
+
+    return 0;
+}

tests/pkg_uzlib/tests/01-run.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2021 Inria
+#
+# This file is subject to the terms and conditions of the GNU Lesser
+# General Public License v2.1. See the file LICENSE in the top level
+# directory for more details.
+
+import sys
+from testrunner import run
+
+
+def testfunc(child):
+    child.expect(r"Compressing message \(len: (\d+)\): \"([A-Za-z\., ]+)\"\r\n")
+    msg_len = int(child.match.group(1))
+    msg = str(child.match.group(2))
+    assert msg_len == len(msg), "Wrong msg_len: {} (expected: {})".format(len(msg), msg_len)
+    child.expect(r"Message compressed to \d+ raw bytes \(ratio: (\d+.\d+)\)\r\n")
+    ratio = float(child.match.group(1))
+    assert ratio <= 1, "Ratio is greater than 1 ({})".format(ratio)
+    child.expect(r"Decompressed message \(len: (\d+)\): \"([A-Za-z\., ]+)\"\r\n")
+    result_msg_len = int(child.match.group(1))
+    result_msg = str(child.match.group(2))
+    assert result_msg_len == msg_len, "Messages length don't match"
+    assert msg == result_msg, "Messages don't match"
+
+
+if __name__ == "__main__":
+    sys.exit(run(testfunc))