RIOT/dist/tools/mkconstfs/mkconstfs2.py

#!/usr/bin/env python3

import os
import sys
import argparse
import pathlib
import posixpath
import io
import itertools
import mmap
import shutil
from binascii import hexlify

C_HEADER = """/* This file was automatically generated by mkconstfs2.
 * !!!! DO NOT EDIT !!!!!
 */

#include <stdint.h>
#include "fs/constfs.h"

"""

FILE_TEMPLATE = """ {{
  .path = "{target_name}",
  .data = {buff_name},
  .size = sizeof({buff_name})
 }},
"""

C_FOOTER = """
static const constfs_t _fs_data = {{
    .files = _files,
    .nfiles = sizeof(_files) / sizeof(_files[0]),
}};

vfs_mount_t {constfs_name} = {{
    .fs = &constfs_file_system,
    .mount_point = "{mount_pount}",
    .private_data = (void *)&_fs_data,
}};
"""

FILES_DECL = """
static const constfs_file_t _files[] = {
"""

BLOB_DECL = """
/** {fname} **/
static const uint8_t {varname}[] = {{
"""


def _relpath_p(path, start):
    return posixpath.relpath(pathlib.Path(os.path.abspath(path)).as_posix(),
                             pathlib.Path(os.path.abspath(start)).as_posix())


def mkconstfs(files, root_path, mount_point, constfs_name):
    """Generate a C file containing a constant file system

    Return
    ------

    chunks: Iterator yielding fragments of the of the output file.
    """

    filemap = {f: (_mkident(i), _relpath_p(f, root_path))
               for i, f in enumerate(files)}

    yield C_HEADER
    yield from itertools.chain.from_iterable(
                print_file_data(local_f, *f_data) for local_f, f_data in filemap.items())

    yield FILES_DECL

    yield from (FILE_TEMPLATE.format(target_name=_addroot(relp),
                                     buff_name=ident)
                for ident, relp in sorted(filemap.values()))

    yield "};\n"

    yield C_FOOTER.format(constfs_name=constfs_name, mount_pount=mount_point)


def _addroot(fname):
    return "/" + fname if not fname.startswith("/") else fname


def _mkident(k):
    return "_file{:02X}".format(k)


def print_file_data(local_fname, varname, target_fname=""):
    """Convert a file into a static C array:

    Parameters
    ----------

    local_fname: real Path (where the file is on this machine's fs)
    target_fname: name that the file will have in the constfs.
    output_file: File-like object where the array will be written.

    Return
    ------

    chunks: Iterator yielding fragments of the of the output text.
    """

    yield BLOB_DECL.format(fname=target_fname, varname=varname)

    def byte2s(b):
        return "0x{},".format(hexlify(b).decode('utf-8'))

    def chunk(iterable, blocksize):
        """Break a single iterable into chunks of maximum size 'blocksize'"""
        return (x for _, x in itertools.groupby(enumerate(iterable),
                                                lambda x: x[0]//blocksize))

    with open(local_fname, 'rb') as f, mmap.mmap(f.fileno(), 0,
                                                 access=mmap.ACCESS_READ
                                                 ) as bfile:
        yield from map(lambda x: x[1],
                       itertools.chain.from_iterable(
                            map(lambda l: itertools.chain(l, [(0, "\n")]),
                                chunk(map(byte2s, bfile), 16)
                                )
                        )
                       )

    yield "};\n"


def main():
    parser = argparse.ArgumentParser(
            description="Embed files into a constant file system")

    parser.add_argument("-m", '--mount', metavar="mountpoint",
                        help="Where to mount the resulting fs", default="/")

    parser.add_argument("-o", '--output', metavar="output_file",
                        help="Write the output to a file instead of stdout. "
                             "The file is only written if the command is successful "
                             "(i.e. there is no partial output")

    parser.add_argument("-r", '--root', metavar="root_base_path",
                        type=pathlib.Path,
                        help="Paths on the constf will be generated for the real "
                        "path of the files by considering this path to be the root "
                        "By default the current directory (.) is used",
                        default=pathlib.Path())

    parser.add_argument("name", help="Name for the vfs_mount_t structure")

    parser.add_argument("files", nargs="+", type=pathlib.Path,
                        help="Files to be included.")

    ns = parser.parse_args()

    f_chunks = mkconstfs(ns.files, ns.root, ns.mount, ns.name)

    if ns.output:
        tmp_out = io.StringIO()
    else:
        tmp_out = sys.stdout

    tmp_out.writelines(f_chunks)

    if ns.output:
        with open(ns.output, "w+") as f:
            tmp_out.seek(0)
            shutil.copyfileobj(tmp_out, f)

    return 0


if __name__ == "__main__":
    exit(main())