File format #2

New Issue

2024-07-29T10:16:39Z

veclavtalica commented

2024-07-29 10:16:39 +00:00

No description provided.

veclavtalica added this to the (deleted) project 2024-07-29 10:16:39 +00:00

veclavtalica added the feature label 2024-07-31 19:16:30 +00:00

veclavtalica commented

2024-08-01 01:46:53 +00:00

API I more or less settled on:


#define TABELA_CURRENT_VERSION  (1)
#define TABELA_STACK_LIMIT      (128)
#define TABELA_SLICE_LIMIT      (1 << 16)

typedef struct tabela t_tabela;


enum tabela_type {
    TABELA_TYPE_NONE,
    TABELA_TYPE_INT32,
    TABELA_TYPE_FLOAT,
    TABELA_TYPE_STRING, /* null terminated, sized */
    TABELA_TYPE_BINARY, /* could be anything, even another tabela */
    TABELA_TYPE_TABLE,
};


typedef struct tabela_variant {
    enum tabela_type type;
    union {
        int32_t i32;
        float f32;
        struct {
            char    *data;
            uint32_t size;
        } str;
        struct {
            uint8_t *data;
            uint32_t size;
        } bin;
        struct {
            uint32_t n_elements;
        } table;
    } value;
} t_tabela_variant;


/* NULL is valid for creating empty tabela */
t_tabela* tabela_open(const char *path);

/* if `make_view` is true, then given memory pointer should remain valid until call to `tabela_deinit()` */
t_tabela* tabela_from_memory(const uint8_t *memory, size_t size, bool make_view);

/* TODO: restricted api variant with physfs ? */
bool tabela_save(const t_tabela *tabela, const char *path);

void tabela_close(t_tabela *tabela);

/* appends primitives to an table on top of the stack */
void tabela_push_int32_at_index(t_tabela *tabela, uint32_t index, int32_t value);
void tabela_push_int32_at_key(t_tabela *tabela, const char *key, int32_t value);

void tabela_push_float_at_index(t_tabela *tabela, const uint32_t index, float value);
void tabela_push_float_at_key(t_tabela *tabela, const char *key, float value);

void tabela_push_string_at_index(t_tabela *tabela, uint32_t index, const char *value);
void tabela_push_string_at_key(t_tabela *tabela, const char *key, const char *value);

void tabela_push_binary_at_index(t_tabela *tabela, uint32_t index, const uint8_t *value);
void tabela_push_binary_at_key(t_tabela *tabela, const char *key, const uint8_t *value);

/* pushes a new table on tabela stack, meaning that all future pushes will be against it */
void tabela_push_table_index(t_tabela *tabela, uint32_t index);
void tabela_push_table_at_key(t_tabela *tabela, const char *key);

void tabela_push_none_at_index(t_tabela *tabela, uint32_t index);
void tabela_push_none_at_key(t_tabela *tabela, const char *key);

/* pop the table stack */
void tabela_pop(t_tabela *tabela);

/* get current value on top of the table stack */
struct tabela_variant tabela_current(const t_tabela *tabela);

/* returns none variant if not present */
struct tabela_variant tabela_index_index(const t_tabela *tabela, uint32_t index);
struct tabela_variant tabela_index_key(const t_tabela *tabela, const char *key);

/* push some table inside of current on top of the stack */
void tabela_crawl_index(const t_tabela *tabela, uint32_t index);
void tabela_crawl_key(const t_tabela *tabela, const char *key);

Edit: API only exposes associative array semantics, simplifying the usage. Array optimization is internal.
Edit: Push none as a way to erase elements.
Edit: Pushes are either by index or key, always explicit.

API I more or less settled on: ```c #define TABELA_CURRENT_VERSION (1) #define TABELA_STACK_LIMIT (128) #define TABELA_SLICE_LIMIT (1 << 16) typedef struct tabela t_tabela; enum tabela_type { TABELA_TYPE_NONE, TABELA_TYPE_INT32, TABELA_TYPE_FLOAT, TABELA_TYPE_STRING, /* null terminated, sized */ TABELA_TYPE_BINARY, /* could be anything, even another tabela */ TABELA_TYPE_TABLE, }; typedef struct tabela_variant { enum tabela_type type; union { int32_t i32; float f32; struct { char *data; uint32_t size; } str; struct { uint8_t *data; uint32_t size; } bin; struct { uint32_t n_elements; } table; } value; } t_tabela_variant; /* NULL is valid for creating empty tabela */ t_tabela* tabela_open(const char *path); /* if `make_view` is true, then given memory pointer should remain valid until call to `tabela_deinit()` */ t_tabela* tabela_from_memory(const uint8_t *memory, size_t size, bool make_view); /* TODO: restricted api variant with physfs ? */ bool tabela_save(const t_tabela *tabela, const char *path); void tabela_close(t_tabela *tabela); /* appends primitives to an table on top of the stack */ void tabela_push_int32_at_index(t_tabela *tabela, uint32_t index, int32_t value); void tabela_push_int32_at_key(t_tabela *tabela, const char *key, int32_t value); void tabela_push_float_at_index(t_tabela *tabela, const uint32_t index, float value); void tabela_push_float_at_key(t_tabela *tabela, const char *key, float value); void tabela_push_string_at_index(t_tabela *tabela, uint32_t index, const char *value); void tabela_push_string_at_key(t_tabela *tabela, const char *key, const char *value); void tabela_push_binary_at_index(t_tabela *tabela, uint32_t index, const uint8_t *value); void tabela_push_binary_at_key(t_tabela *tabela, const char *key, const uint8_t *value); /* pushes a new table on tabela stack, meaning that all future pushes will be against it */ void tabela_push_table_index(t_tabela *tabela, uint32_t index); void tabela_push_table_at_key(t_tabela *tabela, const char *key); void tabela_push_none_at_index(t_tabela *tabela, uint32_t index); void tabela_push_none_at_key(t_tabela *tabela, const char *key); /* pop the table stack */ void tabela_pop(t_tabela *tabela); /* get current value on top of the table stack */ struct tabela_variant tabela_current(const t_tabela *tabela); /* returns none variant if not present */ struct tabela_variant tabela_index_index(const t_tabela *tabela, uint32_t index); struct tabela_variant tabela_index_key(const t_tabela *tabela, const char *key); /* push some table inside of current on top of the stack */ void tabela_crawl_index(const t_tabela *tabela, uint32_t index); void tabela_crawl_key(const t_tabela *tabela, const char *key); ``` Edit: API only exposes associative array semantics, simplifying the usage. Array optimization is internal. Edit: Push none as a way to erase elements. Edit: Pushes are either by index or key, always explicit.

veclavtalica commented

2024-08-01 01:48:18 +00:00

/*

    LAYOUT

    | header | | node + | | slice-desc * | | slice * |
    ^          ^
    |          |
    .tbl       |
            root

    - ordering of slice descriptors matches order of slices themselves
    - there must be at least one node (even if of type none) 


    FEATURES

    - associative lua-like table semantics
    - ability to open and inspect files with zero allocation
    - in-place modifications
    - any endianess, swapping in-place
    - string pooling with reference counting
    - hash of strings stored in binary

*/

``` /* LAYOUT | header | | node + | | slice-desc * | | slice * | ^ ^ | | .tbl | root - ordering of slice descriptors matches order of slices themselves - there must be at least one node (even if of type none) FEATURES - associative lua-like table semantics - ability to open and inspect files with zero allocation - in-place modifications - any endianess, swapping in-place - string pooling with reference counting - hash of strings stored in binary */ ```

veclavtalica commented

2024-08-01 20:20:41 +00:00

Thinking about it, it's too complex, with no real reason to have this complexity.

Alternative approach would be having something like JSON pointer spec path-based queries and write operations.
It makes sense, as this file format is designed for other file formats with obvious before-hand layout.

Example of usage, in Python:

# model format

# parsing
if (f.get('/type') != 'model')
  raise "not a model"
vs = f.get('/vertices')
uv = f.get('/uvs')

# storing
f.put('/type', 'model')
f.put('/vertices', vs)
f.put('/uvs', uv)

Thinking about it, it's too complex, with no real reason to have this complexity. Alternative approach would be having something like JSON pointer spec path-based queries and write operations. It makes sense, as this file format is designed for other file formats with obvious before-hand layout. Example of usage, in Python: ```py # model format # parsing if (f.get('/type') != 'model') raise "not a model" vs = f.get('/vertices') uv = f.get('/uvs') # storing f.put('/type', 'model') f.put('/vertices', vs) f.put('/uvs', uv) ```

veclavtalica commented

2024-08-19 10:06:36 +00:00

C API appropriate for idea above:

/* NULL is valid for creating empty tabela */
t_tabela* tabela_open(const char *path, bool write_access);

void tabela_close(t_tabela *tabela);

double tabela_get_number(t_tabela *tabela, const char *at, ...);
void tabela_put_number(t_tabela *tabela, double value, const char *at, ...);

const char *tabela_get_string(t_tabela *tabela, const char *at, ...);
void tabela_put_string(t_tabela *tabela, const char * value, const char *at, ...);

C API appropriate for idea above: ```c /* NULL is valid for creating empty tabela */ t_tabela* tabela_open(const char *path, bool write_access); void tabela_close(t_tabela *tabela); double tabela_get_number(t_tabela *tabela, const char *at, ...); void tabela_put_number(t_tabela *tabela, double value, const char *at, ...); const char *tabela_get_string(t_tabela *tabela, const char *at, ...); void tabela_put_string(t_tabela *tabela, const char * value, const char *at, ...); ```

veclavtalica referenced this issue from a commit

2025-02-20 12:42:52 +00:00

Merge pull request 'twn: add twn to project on `twn init`' (#2) from yagi/townengine:twn-init-symlink-twn into main

Sign in to join this conversation.

1 Participants

Notifications

Due Date

No due date set.

Dependencies

No dependencies set.

Reference: wanp/townengine#2