From 8cc689adf82699247fe308a025aedd29b1b5eaf2 Mon Sep 17 00:00:00 2001 From: Benjamin Valentin Date: Tue, 1 Feb 2022 22:33:35 +0100 Subject: [PATCH] pkg: add FlashDB --- Makefile.dep | 4 + pkg/Kconfig | 1 + pkg/flashdb/Kconfig | 46 ++++++++ pkg/flashdb/Makefile | 20 ++++ pkg/flashdb/Makefile.dep | 31 +++++ pkg/flashdb/Makefile.fal | 3 + pkg/flashdb/Makefile.flashdb | 3 + pkg/flashdb/Makefile.include | 12 ++ pkg/flashdb/doc.txt | 71 ++++++++++++ pkg/flashdb/include/fal_cfg.h | 107 ++++++++++++++++++ pkg/flashdb/include/fdb_cfg.h | 29 +++++ pkg/flashdb/mtd/Makefile | 3 + pkg/flashdb/mtd/fal_mtd_port.c | 85 ++++++++++++++ .../0001-remove-example-fdb_cfg.h.patch | Bin 0 -> 1843 bytes .../0002-remove-assert-definition.patch | Bin 0 -> 1383 bytes 15 files changed, 415 insertions(+) create mode 100644 pkg/flashdb/Kconfig create mode 100644 pkg/flashdb/Makefile create mode 100644 pkg/flashdb/Makefile.dep create mode 100644 pkg/flashdb/Makefile.fal create mode 100644 pkg/flashdb/Makefile.flashdb create mode 100644 pkg/flashdb/Makefile.include create mode 100644 pkg/flashdb/doc.txt create mode 100644 pkg/flashdb/include/fal_cfg.h create mode 100644 pkg/flashdb/include/fdb_cfg.h create mode 100644 pkg/flashdb/mtd/Makefile create mode 100644 pkg/flashdb/mtd/fal_mtd_port.c create mode 100644 pkg/flashdb/patches/0001-remove-example-fdb_cfg.h.patch create mode 100644 pkg/flashdb/patches/0002-remove-assert-definition.patch diff --git a/Makefile.dep b/Makefile.dep index 8c6b96497d..31578083c3 100644 --- a/Makefile.dep +++ b/Makefile.dep @@ -84,6 +84,10 @@ ifneq (,$(filter tinydtls_sock_dtls, $(USEMODULE))) USEPKG += tinydtls endif +ifneq (,$(filter flashdb_%,$(USEMODULE))) + USEPKG += flashdb +endif + ifneq (,$(filter tinyusb_%, $(USEMODULE))) USEPKG += tinyusb endif diff --git a/pkg/Kconfig b/pkg/Kconfig index 9f2a2ebe8b..e4ab62722d 100644 --- a/pkg/Kconfig +++ b/pkg/Kconfig @@ -29,6 +29,7 @@ rsource "etl/Kconfig" rsource "fff/Kconfig" rsource "fido2_tests/Kconfig" rsource "flatbuffers/Kconfig" +rsource "flashdb/Kconfig" rsource "gecko_sdk/Kconfig" rsource "gemmlowp/Kconfig" rsource "hacl/Kconfig" diff --git a/pkg/flashdb/Kconfig b/pkg/flashdb/Kconfig new file mode 100644 index 0000000000..8d66a50be0 --- /dev/null +++ b/pkg/flashdb/Kconfig @@ -0,0 +1,46 @@ +# Copyright (c) 2022 ML!PA Consulting GmbH +# +# This file is subject to the terms and conditions of the GNU Lesser +# General Public License v2.1. See the file LICENSE in the top level +# directory for more details. +# + +menuconfig KCONFIG_USEPKG_FLASHDB + bool "Configure FlashDB" + help + Configure FlashDB using Kconfig + +if KCONFIG_USEPKG_FLASHDB + +config MODULE_FLASHDB_MTD + bool "Use MTD backend" + select MODULE_MTD + select MODULE_FLASHDB_FAL + +config MODULE_FLASHDB_VFS + bool "Use VFS backend" + select MODULE_VFS + select MODULE_FLASHDB_FAL + +config MODULE_FLASHDB_FAL + bool "FlashDB flash abstraction layer" + depends on MODULE_FLASHDB_MTD + default y + +config MODULE_FLASHDB_TSDB + bool "FlashDB Time Series Database" + +config MODULE_FLASHDB_KVDB + bool "FlashDB Key-Value Database" + +config MODULE_FLASHDB_KVDB_AUTO_UPDATE + bool "Enable Key-Value automatic upgrade function" + depends on MODULE_FLASHDB_KVDB + help + When this function is enabled, fdb_kvdb.ver_num stores the version of the current + database. If the version changes, it will automatically trigger an upgrade action + and update the new default KV collection to the current database. + +config MODULE_FLASHDB_MTD + +endif # PACKAGE_FLASHDB diff --git a/pkg/flashdb/Makefile b/pkg/flashdb/Makefile new file mode 100644 index 0000000000..6c9eeb1cc3 --- /dev/null +++ b/pkg/flashdb/Makefile @@ -0,0 +1,20 @@ +PKG_NAME=flashdb +PKG_URL=https://github.com/armink/FlashDB.git +# 1.1.2 +PKG_VERSION=7062902a3e7b6b8a7e8f5886ae242271a0164e05 +PKG_LICENSE=Apache-2.0 + +include $(RIOTBASE)/pkg/pkg.mk + +CFLAGS += -Wno-pedantic +CFLAGS += -Wno-cast-align +CFLAGS += -Wno-unused +CFLAGS += -Wno-format + +.PHONY: flashdb_fal + +all: $(filter flashdb_fal, $(USEMODULE)) + $(QQ)"$(MAKE)" -C $(PKG_SOURCE_DIR)/src -f $(CURDIR)/Makefile.flashdb + +flashdb_fal: + $(QQ)"$(MAKE)" -C $(PKG_SOURCE_DIR)/port/fal/src -f $(CURDIR)/Makefile.fal diff --git a/pkg/flashdb/Makefile.dep b/pkg/flashdb/Makefile.dep new file mode 100644 index 0000000000..7538c7dee4 --- /dev/null +++ b/pkg/flashdb/Makefile.dep @@ -0,0 +1,31 @@ +# FlashDB is only supported by 32 bit architectures +FEATURES_REQUIRED += arch_32bit + +ifneq (,$(filter flashdb_tsdb,$(USEMODULE))) + CFLAGS += -DFDB_USING_TSDB +endif + +ifneq (,$(filter flashdb_kvdb_auto_update,$(USEMODULE))) + CFLAGS += -DFDB_KV_AUTO_UPDATE + USEMODULE += flashdb_kvdb +endif + +ifneq (,$(filter flashdb_kvdb,$(USEMODULE))) + CFLAGS += -DFDB_USING_KVDB +endif + +ifneq (,$(filter flashdb_vfs,$(USEMODULE))) + USEMODULE += vfs +endif + +ifneq (,$(filter flashdb_mtd,$(USEMODULE))) + CFLAGS += -DFDB_USING_FAL_MODE + USEMODULE += flashdb_fal + USEMODULE += mtd +else + ifeq (native, $(BOARD)) + CFLAGS += -DFDB_USING_FILE_POSIX_MODE + else + CFLAGS += -DFDB_USING_FILE_LIBC_MODE + endif +endif diff --git a/pkg/flashdb/Makefile.fal b/pkg/flashdb/Makefile.fal new file mode 100644 index 0000000000..e8ab2def61 --- /dev/null +++ b/pkg/flashdb/Makefile.fal @@ -0,0 +1,3 @@ +MODULE := flashdb_fal + +include $(RIOTBASE)/Makefile.base diff --git a/pkg/flashdb/Makefile.flashdb b/pkg/flashdb/Makefile.flashdb new file mode 100644 index 0000000000..e19e2ed3f0 --- /dev/null +++ b/pkg/flashdb/Makefile.flashdb @@ -0,0 +1,3 @@ +MODULE := flashdb + +include $(RIOTBASE)/Makefile.base diff --git a/pkg/flashdb/Makefile.include b/pkg/flashdb/Makefile.include new file mode 100644 index 0000000000..4f97374da3 --- /dev/null +++ b/pkg/flashdb/Makefile.include @@ -0,0 +1,12 @@ +INCLUDES += -I$(PKGDIRBASE)/flashdb/inc +INCLUDES += -I$(PKGDIRBASE)/flashdb/port/fal/inc +INCLUDES += -I$(RIOTBASE)/pkg/flashdb/include + +PSEUDOMODULES += flashdb_tsdb +PSEUDOMODULES += flashdb_kvdb +PSEUDOMODULES += flashdb_kvdb_auto_update +PSEUDOMODULES += flashdb_vfs + +ifneq (,$(filter flashdb_mtd,$(USEMODULE))) + DIRS += $(RIOTBASE)/pkg/flashdb/mtd +endif diff --git a/pkg/flashdb/doc.txt b/pkg/flashdb/doc.txt new file mode 100644 index 0000000000..3d2d4f871c --- /dev/null +++ b/pkg/flashdb/doc.txt @@ -0,0 +1,71 @@ +/** + * @defgroup pkg_flashdb FlashDB: Database for Flash + * @ingroup pkg + * @brief An ultra-lightweight embedded database + * @see http://armink.gitee.io/flashdb/ + * + * This package provides a Key-Value database and a Time series database + * that can be used with a MTD or a file system backend. + * + * It is recommended to use the use the MTD backend for performance reasons, + * but if you have a filesystem anyway or want to store the database as a file + * there is also a VFS backend that can be used instead. + * + * MTD Backend + * =========== + * To enable the MTD backend, select the `flashdb_mtd` module. + * + * FlashDB can only use a single MTD device. To assign a MTD device for FlashDB + * to use, call + * + * ```C + * fdb_mtd_init(MTD_0); + * ``` + * + * FlashDB allows for multiple partitions on the device. They have to configured + * at compile time: + * + * ``` + * CFLAGS += -DFAL_PART0_LABEL=\"part0\" + * CFLAGS += -DFAL_PART0_LENGTH=8*1024 + * + * CFLAGS += -DFAL_PART1_LABEL=\"part1\" + * CFLAGS += -DFAL_PART1_LENGTH=4*1024 + * ``` + * + * The partition names have to be used in `fdb_kvdb_init()` / `fdb_tsdb_init()`. + * + * VFS Backend + * =========== + * To enable the VFS Backend, select the `flashdb_vfs` module. + * + * Here instead of partitions, FlashDB uses directories to store the data. + * The directory names are used instead of the partition names in `fdb_kvdb_init()` / + * `fdb_tsdb_init()` and you have to enable file mode by calling + * + * ```C + * bool file_mode = true; + * fdb_kvdb_control(&kvdb, FDB_KVDB_CTRL_SET_FILE_MODE, &file_mode); + * ``` + * + * for the Key-Value database and + * + * ```C + * bool file_mode = true; + * fdb_tsdb_control(&tsdb, FDB_TSDB_CTRL_SET_FILE_MODE, &file_mode); + * ``` + * + * for the time-series database. + * + * Key-Value database + * ================== + * To enable the Key-Value database, select the `flashdb_kvdb` module. + * + * For use of the FlashDB API, reffer to the [FlashDB documentation](http://armink.gitee.io/flashdb/#/sample-kvdb-basic). + * + * Time series database + * ==================== + * To enable the Key-Value database, select the `flashdb_tsdb` module. + * + * For use of the FlashDB API, reffer to the [FlashDB documentation](http://armink.gitee.io/flashdb/#/sample-tsdb-basic). + */ diff --git a/pkg/flashdb/include/fal_cfg.h b/pkg/flashdb/include/fal_cfg.h new file mode 100644 index 0000000000..557b28658e --- /dev/null +++ b/pkg/flashdb/include/fal_cfg.h @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2022 ML!PA Consulting GmbH + * + * This file is subject to the terms and conditions of the GNU Lesser + * General Public License v2.1. See the file LICENSE in the top level + * directory for more details. + */ + +/** + * @ingroup pkg_flashdb + * @{ + * + * @file + * @brief Flash Abstraction Layer partition configuration + * + * FlashDB can only use a single MTD device, but allows for multiple + * named partitions on the MTD device. + * + * This file pre-defines up to 4 partitions, if you need more edit + * this file or provide your own `FAL_PART_TABLE`. + * + * @author Benjamin Valentin + */ + +#ifndef FAL_CFG_H +#define FAL_CFG_H + +#include "board.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Partition table is defined at compile time (not read from flash) + */ +#define FAL_PART_HAS_TABLE_CFG + +/** + * @brief FAL <-> MTD adapter + */ +extern struct fal_flash_dev mtd_flash0; + +/** + * @brief flash device table + */ +#define FAL_FLASH_DEV_TABLE \ +{ \ + &mtd_flash0, \ +} + +/** + * @brief Partition 0 + */ +#ifdef FAL_PART0_LABEL +#define FAL_ROW_PART0 { FAL_PART_MAGIC_WORD, FAL_PART0_LABEL, "fal_mtd", \ + 0, FAL_PART0_LENGTH, 0 }, +#else +#define FAL_ROW_PART0 +#endif + +/** + * @brief Partition 1 + */ +#ifdef FAL_PART1_LABEL +#define FAL_ROW_PART1 { FAL_PART_MAGIC_WORD, FAL_PART1_LABEL, "fal_mtd", \ + FAL_PART0_LENGTH, FAL_PART1_LENGTH, 0 }, +#else +#define FAL_ROW_PART1 +#endif + +/** + * @brief Partition 2 + */ +#ifdef FAL_PART2_LABEL +#define FAL_ROW_PART2 { FAL_PART_MAGIC_WORD, FAL_PART2_LABEL, "fal_mtd", + FAL_PART1_LENGTH, FAL_PART2_LENGTH, 0 }, +#else +#define FAL_ROW_PART2 +#endif + +/** + * @brief Partition 3 + */ +#ifdef FAL_PART3_LABEL +#define FAL_ROW_PART3 { FAL_PART_MAGIC_WORD, FAL_PART2_LABEL, "fal_mtd", + FAL_PART2_LENGTH, FAL_PART3_LENGTH, 0 }, +#else +#define FAL_ROW_PART3 +#endif + +/** + * @brief Partition Table + */ +#define FAL_PART_TABLE \ +{ \ + FAL_ROW_PART0 \ + FAL_ROW_PART1 \ + FAL_ROW_PART2 \ + FAL_ROW_PART3 \ +} + +#ifdef __cplusplus +} +#endif +#endif /* FAL_CFG_H */ +/** @} */ diff --git a/pkg/flashdb/include/fdb_cfg.h b/pkg/flashdb/include/fdb_cfg.h new file mode 100644 index 0000000000..188a027d1a --- /dev/null +++ b/pkg/flashdb/include/fdb_cfg.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2020, Armink, + * + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * @file + * @brief configuration file + */ + +#ifndef FDB_CFG_H +#define FDB_CFG_H + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief disable all debug output unless explicitly requested + */ +#ifndef FDB_DEBUG_ENABLE +#define FDB_PRINT(...) +#endif + +#ifdef __cplusplus +} +#endif +#endif /* FDB_CFG_H */ diff --git a/pkg/flashdb/mtd/Makefile b/pkg/flashdb/mtd/Makefile new file mode 100644 index 0000000000..b3a16bda33 --- /dev/null +++ b/pkg/flashdb/mtd/Makefile @@ -0,0 +1,3 @@ +MODULE := flashdb_mtd + +include $(RIOTBASE)/Makefile.base diff --git a/pkg/flashdb/mtd/fal_mtd_port.c b/pkg/flashdb/mtd/fal_mtd_port.c new file mode 100644 index 0000000000..0d50448fca --- /dev/null +++ b/pkg/flashdb/mtd/fal_mtd_port.c @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2022 ML!PA Consulting GmbH + * + * This file is subject to the terms and conditions of the GNU Lesser + * General Public License v2.1. See the file LICENSE in the top level + * directory for more details. + */ + +/** + * @ingroup pkg_flashdb + * @{ + * + * @file + * @brief MTD <-> FAL adapter + * + * @author Benjamin Valentin + * @] + */ + +#include "mtd.h" + +#include +#include +#include + +#define MIN(a, b) ((a) > (b) ? (b) : (a)) + +static mtd_dev_t *_mtd; + +static int _read(long offset, uint8_t *buf, size_t size) +{ + int res = mtd_read_page(_mtd, buf, + offset / _mtd->page_size, + offset % _mtd->page_size, + size); + return res ? res : (int)size; +} + +static int _write(long offset, const uint8_t *buf, size_t size) +{ + int res = mtd_write_page_raw(_mtd, buf, + offset / _mtd->page_size, + offset % _mtd->page_size, size); + return res ? res : (int)size; +} + +static int _erase(long offset, size_t size) +{ + int res = mtd_erase(_mtd, offset, size); + return res ? res : (int)size; +} + +static int _init(void) +{ + unsigned sector_size = _mtd->page_size * _mtd->pages_per_sector; + + mtd_flash0.len = _mtd->sector_count * sector_size; + mtd_flash0.blk_size = sector_size; + /* MTD ensures write_gran = 1 is possible */ + mtd_flash0.write_gran = 1; + + return 1; +} + +struct fal_flash_dev mtd_flash0 = { + .name = "fal_mtd", + .ops = { + .init = _init, + .read = _read, + .write = _write, + .erase = _erase, + }, +}; + +void fdb_mtd_init(mtd_dev_t *mtd) +{ + unsigned sector_size; + + mtd_init(mtd); + _mtd = mtd; + + /* long offset limits addressable MTD range */ + sector_size = _mtd->page_size * _mtd->pages_per_sector; + _mtd->sector_count = MIN(_mtd->sector_count, LONG_MAX / sector_size); +} diff --git a/pkg/flashdb/patches/0001-remove-example-fdb_cfg.h.patch b/pkg/flashdb/patches/0001-remove-example-fdb_cfg.h.patch new file mode 100644 index 0000000000000000000000000000000000000000..573b3cfd8ff20d7d6b11dcead5f25d55cba9052d GIT binary patch literal 1843 zcmb7E+iu%95PjEI4A2)lu`ZUJHZh8Y<@i#gPGZD%w!mT`P!cJ#ktlGW5n`(@cp5GJq&!f^!lD>t(*8OP3j^1whA}5pMD2Pi}Hm6{YHv?PJyLyl(5X% zS(a_v7I2ZcZ+q|uTzP=||MRi1AjT0r*%6V^w5MQFn$wz>qUzXP%d99AWl90-d_NeR zQd?Ha8AI-j*FfCUvf|UVrJ-qRIFBGLE>ijp2G{A0PnhF)=24xVV_KJ&m*A%K#T8AS zTcZ)!{T^0*>GeDqjV#-}uxz-1+j4)9JloaKNxDjLuLq&TsXh1LBhfF;_c|YC1>w1w z6Q=3Z^=k6PUh;&BirR}*jhJyNhY;>bvZJ=|cq-1dY{h1BMoLDVJ&Gl#43bhXo;8vv zDRskdIqx{bb{Ryo$vC>5&7)6|WxvB6F6il|!iT2fB7@IQlQA$#>PAw{{UbqvY}bqX zd89&AO^C1H&p#kk*wiI7`xFg{fPjk}U#l9Y%t({h@VcV{l155w+EBhwS*Z?0Q^ZsP zuYP#Nfl)<`FP~r1*ulUEgLG|3bzI)(rzqTR9-{4P5^koZdMeO2M!~kj%^ExGY<5g`~X$6-FFBr-f+VtZ-hijGX!W?b-r$alA<8)7FT^yn| zEsCDPD14l+qsSm=VGVUk<0ivP8HZn(+veQulj(S?st?CY<8^Hdz$%WeAl0!?!&~3YKCDJJs+|_q`sf0OzGuH4C-txQ$ikPR0 zi&9WJa{W=@!E!y7KQYdEvNqqb=IXjP&i$sBKviPqcv36Caie=7mxX!9GJ<sFD8cyzC5L@Zeveu52xolN_H`kxp8=+m5&);3&nE3bcKx`>s zJTqFN|6$r_P>i29>!+C&qbAa+y1u!*x{Hk+W3M(bWp$S=2&Q^92WzlbI}hbf54$>` z9+D)!zdF6Ue0`N9=D|i!$M|uoJKXSoY-|qv^{g1b>YDcV)vZXE6BK;K3l!A^`~9}m zy0+R*^0qC8wCAat90hG#;K_@i`->MfSE8P_vkYTZ$=r0!1EPK@e-tmeUhcwImzhw< Ir;ba=H@AmV^Z)<= literal 0 HcmV?d00001