From db7ef47b16c22a9680630d5ef6e8135e773b978a Mon Sep 17 00:00:00 2001 From: Alex Gaetano Padula Date: Fri, 27 Mar 2026 12:50:34 -0400 Subject: [PATCH] addition of unified memtable config support, object store column family config fields, extended database statistics (unified memtable, object store, local cache, replica mode), iterator key_value combined access method, promote_to_primary replica promotion, and tdb_err_readonly error code; updated ffi struct definitions to align with latest tidesdb db.h; added 6 new tests covering all new features; bumped version to 0.5.7 --- src/tidesdb.lua | 103 +++++++++- tests/test_tidesdb.lua | 190 ++++++++++++++++++ ...5.6-1.rockspec => tidesdb-0.5.7-1.rockspec | 4 +- 3 files changed, 294 insertions(+), 3 deletions(-) rename tidesdb-0.5.6-1.rockspec => tidesdb-0.5.7-1.rockspec (95%) diff --git a/src/tidesdb.lua b/src/tidesdb.lua index e0923dc..2d49ef3 100644 --- a/src/tidesdb.lua +++ b/src/tidesdb.lua @@ -38,6 +38,7 @@ ffi.cdef[[ static const int TDB_ERR_INVALID_DB = -10; static const int TDB_ERR_UNKNOWN = -11; static const int TDB_ERR_LOCKED = -12; + static const int TDB_ERR_READONLY = -13; // Structures static const int TDB_MAX_CF_NAME_LEN = 128; @@ -82,6 +83,9 @@ ffi.cdef[[ int use_btree; tidesdb_commit_hook_fn commit_hook_fn; void *commit_hook_ctx; + size_t object_target_file_size; + int object_lazy_compaction; + int object_prefetch_compaction; } tidesdb_column_family_config_t; typedef struct { @@ -94,6 +98,14 @@ ffi.cdef[[ int log_to_file; size_t log_truncation_at; size_t max_memory_usage; + int unified_memtable; + size_t unified_memtable_write_buffer_size; + int unified_memtable_skip_list_max_level; + float unified_memtable_skip_list_probability; + int unified_memtable_sync_mode; + uint64_t unified_memtable_sync_interval_us; + void* object_store; + void* object_store_config; } tidesdb_config_t; typedef struct { @@ -204,6 +216,22 @@ ffi.cdef[[ int64_t txn_memory_bytes; size_t compaction_queue_size; size_t flush_queue_size; + int unified_memtable_enabled; + int64_t unified_memtable_bytes; + int unified_immutable_count; + int unified_is_flushing; + uint32_t unified_next_cf_index; + uint64_t unified_wal_generation; + int object_store_enabled; + const char* object_store_connector; + size_t local_cache_bytes_used; + size_t local_cache_bytes_max; + int local_cache_num_files; + uint64_t last_uploaded_generation; + size_t upload_queue_depth; + uint64_t total_uploads; + uint64_t total_upload_failures; + int replica_mode; } tidesdb_db_stats_t; int tidesdb_get_db_stats(void* db, tidesdb_db_stats_t* stats); @@ -229,6 +257,12 @@ ffi.cdef[[ int tidesdb_comparator_reverse_memcmp(const uint8_t* key1, size_t key1_size, const uint8_t* key2, size_t key2_size, void* ctx); int tidesdb_comparator_case_insensitive(const uint8_t* key1, size_t key1_size, const uint8_t* key2, size_t key2_size, void* ctx); + // Iterator combined key-value + int tidesdb_iter_key_value(void* iter, uint8_t** key, size_t* key_size, uint8_t** value, size_t* value_size); + + // Replica promotion + int tidesdb_promote_to_primary(void* db); + // Memory management void tidesdb_free(void* ptr); @@ -293,6 +327,7 @@ tidesdb.TDB_ERR_MEMORY_LIMIT = -9 tidesdb.TDB_ERR_INVALID_DB = -10 tidesdb.TDB_ERR_UNKNOWN = -11 tidesdb.TDB_ERR_LOCKED = -12 +tidesdb.TDB_ERR_READONLY = -13 -- Compression algorithms tidesdb.CompressionAlgorithm = { @@ -343,6 +378,7 @@ local error_messages = { [tidesdb.TDB_ERR_INVALID_DB] = "invalid database handle", [tidesdb.TDB_ERR_UNKNOWN] = "unknown error", [tidesdb.TDB_ERR_LOCKED] = "database is locked", + [tidesdb.TDB_ERR_READONLY] = "database is read-only", } -- TidesDBError class @@ -391,6 +427,12 @@ function tidesdb.default_config() log_to_file = false, log_truncation_at = 24 * 1024 * 1024, max_memory_usage = 0, + unified_memtable = false, + unified_memtable_write_buffer_size = 64 * 1024 * 1024, + unified_memtable_skip_list_max_level = 12, + unified_memtable_skip_list_probability = 0.25, + unified_memtable_sync_mode = tidesdb.SyncMode.SYNC_INTERVAL, + unified_memtable_sync_interval_us = 128000, } end @@ -418,6 +460,9 @@ function tidesdb.default_column_family_config() l1_file_count_trigger = c_config.l1_file_count_trigger, l0_queue_stall_threshold = c_config.l0_queue_stall_threshold, use_btree = c_config.use_btree ~= 0, + object_target_file_size = tonumber(c_config.object_target_file_size), + object_lazy_compaction = c_config.object_lazy_compaction ~= 0, + object_prefetch_compaction = c_config.object_prefetch_compaction ~= 0, } end @@ -452,6 +497,9 @@ local function config_to_c_struct(config, cf_name) c_config.l1_file_count_trigger = config.l1_file_count_trigger or 4 c_config.l0_queue_stall_threshold = config.l0_queue_stall_threshold or 20 c_config.use_btree = config.use_btree and 1 or 0 + c_config.object_target_file_size = config.object_target_file_size or 0 + c_config.object_lazy_compaction = config.object_lazy_compaction and 1 or 0 + c_config.object_prefetch_compaction = config.object_prefetch_compaction and 1 or 0 local name = config.comparator_name or "memcmp" local name_len = math.min(#name, 63) @@ -549,6 +597,19 @@ function Iterator:value() return ffi.string(value_ptr[0], value_size[0]) end +function Iterator:key_value() + if self._closed then + error(TidesDBError.new("Iterator is closed")) + end + local key_ptr = ffi.new("uint8_t*[1]") + local key_size = ffi.new("size_t[1]") + local value_ptr = ffi.new("uint8_t*[1]") + local value_size = ffi.new("size_t[1]") + local result = lib.tidesdb_iter_key_value(self._iter, key_ptr, key_size, value_ptr, value_size) + check_result(result, "failed to get key-value") + return ffi.string(key_ptr[0], key_size[0]), ffi.string(value_ptr[0], value_size[0]) +end + function Iterator:close() if not self._closed and self._iter ~= nil then lib.tidesdb_iter_free(self._iter) @@ -889,6 +950,12 @@ function TidesDB.new(config) c_config.log_to_file = config.log_to_file and 1 or 0 c_config.log_truncation_at = config.log_truncation_at or 24 * 1024 * 1024 c_config.max_memory_usage = config.max_memory_usage or 0 + c_config.unified_memtable = config.unified_memtable and 1 or 0 + c_config.unified_memtable_write_buffer_size = config.unified_memtable_write_buffer_size or 64 * 1024 * 1024 + c_config.unified_memtable_skip_list_max_level = config.unified_memtable_skip_list_max_level or 12 + c_config.unified_memtable_skip_list_probability = config.unified_memtable_skip_list_probability or 0.25 + c_config.unified_memtable_sync_mode = config.unified_memtable_sync_mode or tidesdb.SyncMode.SYNC_INTERVAL + c_config.unified_memtable_sync_interval_us = config.unified_memtable_sync_interval_us or 128000 local db_ptr = ffi.new("void*[1]") local result = lib.tidesdb_open(c_config, db_ptr) @@ -910,6 +977,12 @@ function TidesDB.open(path, options) log_to_file = options.log_to_file or false, log_truncation_at = options.log_truncation_at or 24 * 1024 * 1024, max_memory_usage = options.max_memory_usage or 0, + unified_memtable = options.unified_memtable or false, + unified_memtable_write_buffer_size = options.unified_memtable_write_buffer_size, + unified_memtable_skip_list_max_level = options.unified_memtable_skip_list_max_level, + unified_memtable_skip_list_probability = options.unified_memtable_skip_list_probability, + unified_memtable_sync_mode = options.unified_memtable_sync_mode, + unified_memtable_sync_interval_us = options.unified_memtable_sync_interval_us, } return TidesDB.new(config) end @@ -1093,6 +1166,22 @@ function TidesDB:get_db_stats() txn_memory_bytes = tonumber(c_stats.txn_memory_bytes), compaction_queue_size = tonumber(c_stats.compaction_queue_size), flush_queue_size = tonumber(c_stats.flush_queue_size), + unified_memtable_enabled = c_stats.unified_memtable_enabled ~= 0, + unified_memtable_bytes = tonumber(c_stats.unified_memtable_bytes), + unified_immutable_count = c_stats.unified_immutable_count, + unified_is_flushing = c_stats.unified_is_flushing ~= 0, + unified_next_cf_index = tonumber(c_stats.unified_next_cf_index), + unified_wal_generation = tonumber(c_stats.unified_wal_generation), + object_store_enabled = c_stats.object_store_enabled ~= 0, + object_store_connector = c_stats.object_store_connector ~= nil and ffi.string(c_stats.object_store_connector) or nil, + local_cache_bytes_used = tonumber(c_stats.local_cache_bytes_used), + local_cache_bytes_max = tonumber(c_stats.local_cache_bytes_max), + local_cache_num_files = c_stats.local_cache_num_files, + last_uploaded_generation = tonumber(c_stats.last_uploaded_generation), + upload_queue_depth = tonumber(c_stats.upload_queue_depth), + total_uploads = tonumber(c_stats.total_uploads), + total_upload_failures = tonumber(c_stats.total_upload_failures), + replica_mode = c_stats.replica_mode ~= 0, } end @@ -1136,6 +1225,15 @@ function TidesDB:get_comparator(name) return fn_ptr[0], ctx_ptr[0] end +function TidesDB:promote_to_primary() + if self._closed then + error(TidesDBError.new("Database is closed")) + end + + local result = lib.tidesdb_promote_to_primary(self._db) + check_result(result, "failed to promote to primary") +end + tidesdb.TidesDB = TidesDB -- Configuration file operations @@ -1166,6 +1264,9 @@ function tidesdb.load_config_from_ini(ini_file, section_name) l1_file_count_trigger = c_config.l1_file_count_trigger, l0_queue_stall_threshold = c_config.l0_queue_stall_threshold, use_btree = c_config.use_btree ~= 0, + object_target_file_size = tonumber(c_config.object_target_file_size), + object_lazy_compaction = c_config.object_lazy_compaction ~= 0, + object_prefetch_compaction = c_config.object_prefetch_compaction ~= 0, } end @@ -1176,6 +1277,6 @@ function tidesdb.save_config_to_ini(ini_file, section_name, config) end -- Version -tidesdb._VERSION = "0.5.6" +tidesdb._VERSION = "0.5.7" return tidesdb diff --git a/tests/test_tidesdb.lua b/tests/test_tidesdb.lua index eb642b2..1256824 100644 --- a/tests/test_tidesdb.lua +++ b/tests/test_tidesdb.lua @@ -1068,6 +1068,196 @@ function tests.test_get_db_stats() print("PASS: test_get_db_stats") end +function tests.test_iterator_key_value() + local path = "./test_db_iter_kv" + cleanup_db(path) + + local db = tidesdb.TidesDB.open(path) + db:create_column_family("test_cf") + local cf = db:get_column_family("test_cf") + + -- Insert data + local txn = db:begin_txn() + txn:put(cf, "alpha", "one") + txn:put(cf, "beta", "two") + txn:put(cf, "gamma", "three") + txn:commit() + txn:free() + + -- Use key_value() to get both in one call + local read_txn = db:begin_txn() + local iter = read_txn:new_iterator(cf) + iter:seek_to_first() + + local count = 0 + local pairs_found = {} + while iter:valid() do + local k, v = iter:key_value() + pairs_found[k] = v + count = count + 1 + iter:next() + end + + assert_eq(count, 3, "should iterate over 3 entries with key_value") + assert_eq(pairs_found["alpha"], "one", "key_value should return correct pair for alpha") + assert_eq(pairs_found["beta"], "two", "key_value should return correct pair for beta") + assert_eq(pairs_found["gamma"], "three", "key_value should return correct pair for gamma") + + iter:free() + read_txn:free() + + db:drop_column_family("test_cf") + db:close() + cleanup_db(path) + print("PASS: test_iterator_key_value") +end + +function tests.test_unified_memtable_config() + local path = "./test_db_unified_mt" + cleanup_db(path) + + -- Test default config includes unified_memtable fields + local default_cfg = tidesdb.default_config() + assert_true(default_cfg.unified_memtable ~= nil, "unified_memtable should exist in default config") + assert_eq(default_cfg.unified_memtable, false, "unified_memtable should default to false") + assert_true(default_cfg.unified_memtable_write_buffer_size ~= nil, "unified_memtable_write_buffer_size should exist") + assert_true(default_cfg.unified_memtable_skip_list_max_level ~= nil, "unified_memtable_skip_list_max_level should exist") + assert_true(default_cfg.unified_memtable_skip_list_probability ~= nil, "unified_memtable_skip_list_probability should exist") + assert_true(default_cfg.unified_memtable_sync_mode ~= nil, "unified_memtable_sync_mode should exist") + assert_true(default_cfg.unified_memtable_sync_interval_us ~= nil, "unified_memtable_sync_interval_us should exist") + + -- Test opening database with unified_memtable disabled (default) + local db = tidesdb.TidesDB.open(path) + assert_true(db ~= nil, "database should open with unified_memtable disabled") + db:close() + cleanup_db(path) + + -- Test opening database with unified_memtable enabled + local db2 = tidesdb.TidesDB.open(path, { + unified_memtable = true, + unified_memtable_write_buffer_size = 32 * 1024 * 1024, + unified_memtable_skip_list_max_level = 16, + unified_memtable_skip_list_probability = 0.5, + unified_memtable_sync_mode = tidesdb.SyncMode.SYNC_FULL, + }) + assert_true(db2 ~= nil, "database should open with unified_memtable enabled") + + -- Basic operations should still work + db2:create_column_family("test_cf") + local cf = db2:get_column_family("test_cf") + local txn = db2:begin_txn() + txn:put(cf, "key1", "value1") + txn:commit() + txn:free() + + local read_txn = db2:begin_txn() + local v = read_txn:get(cf, "key1") + assert_eq(v, "value1", "should read value with unified_memtable enabled") + read_txn:free() + + db2:drop_column_family("test_cf") + db2:close() + cleanup_db(path) + print("PASS: test_unified_memtable_config") +end + +function tests.test_object_cf_config_fields() + local path = "./test_db_object_cf" + cleanup_db(path) + + -- Test default column family config includes object_* fields + local default_cf = tidesdb.default_column_family_config() + assert_true(default_cf.object_target_file_size ~= nil, "object_target_file_size should exist") + assert_true(default_cf.object_lazy_compaction ~= nil, "object_lazy_compaction should exist") + assert_true(default_cf.object_prefetch_compaction ~= nil, "object_prefetch_compaction should exist") + + -- Test creating CF with custom object_* fields + local db = tidesdb.TidesDB.open(path) + local cf_config = tidesdb.default_column_family_config() + cf_config.object_target_file_size = 16 * 1024 * 1024 + cf_config.object_lazy_compaction = true + cf_config.object_prefetch_compaction = false + db:create_column_family("test_cf", cf_config) + + local cf = db:get_column_family("test_cf") + assert_true(cf ~= nil, "column family should be created with object_* config") + + db:drop_column_family("test_cf") + db:close() + cleanup_db(path) + print("PASS: test_object_cf_config_fields") +end + +function tests.test_db_stats_extended_fields() + local path = "./test_db_stats_extended" + cleanup_db(path) + + local db = tidesdb.TidesDB.open(path, { + log_level = tidesdb.LogLevel.LOG_WARN, + }) + db:create_column_family("test_cf") + local cf = db:get_column_family("test_cf") + + -- Write some data + local txn = db:begin_txn() + txn:put(cf, "key1", "value1") + txn:commit() + txn:free() + + -- Get database-level stats and check new fields + local db_stats = db:get_db_stats() + + -- Unified memtable fields + assert_true(db_stats.unified_memtable_enabled ~= nil, "unified_memtable_enabled should exist") + assert_true(db_stats.unified_memtable_bytes ~= nil, "unified_memtable_bytes should exist") + assert_true(db_stats.unified_immutable_count ~= nil, "unified_immutable_count should exist") + assert_true(db_stats.unified_is_flushing ~= nil, "unified_is_flushing should exist") + assert_true(db_stats.unified_next_cf_index ~= nil, "unified_next_cf_index should exist") + assert_true(db_stats.unified_wal_generation ~= nil, "unified_wal_generation should exist") + + -- Object store fields + assert_true(db_stats.object_store_enabled ~= nil, "object_store_enabled should exist") + assert_true(db_stats.local_cache_bytes_used ~= nil, "local_cache_bytes_used should exist") + assert_true(db_stats.local_cache_bytes_max ~= nil, "local_cache_bytes_max should exist") + assert_true(db_stats.local_cache_num_files ~= nil, "local_cache_num_files should exist") + assert_true(db_stats.last_uploaded_generation ~= nil, "last_uploaded_generation should exist") + assert_true(db_stats.upload_queue_depth ~= nil, "upload_queue_depth should exist") + assert_true(db_stats.total_uploads ~= nil, "total_uploads should exist") + assert_true(db_stats.total_upload_failures ~= nil, "total_upload_failures should exist") + assert_true(db_stats.replica_mode ~= nil, "replica_mode should exist") + + db:drop_column_family("test_cf") + db:close() + cleanup_db(path) + print("PASS: test_db_stats_extended_fields") +end + +function tests.test_promote_to_primary() + local path = "./test_db_promote" + cleanup_db(path) + + -- Open a normal (non-replica) database + local db = tidesdb.TidesDB.open(path) + + -- Calling promote_to_primary on a non-replica should not crash + -- It may return an error or succeed depending on the C implementation + local ok, err = pcall(function() + db:promote_to_primary() + end) + -- We just verify the method exists and is callable + assert_true(ok or err ~= nil, "promote_to_primary should be callable") + + db:close() + cleanup_db(path) + print("PASS: test_promote_to_primary") +end + +function tests.test_error_readonly_constant() + -- Verify TDB_ERR_READONLY constant exists and has correct value + assert_eq(tidesdb.TDB_ERR_READONLY, -13, "TDB_ERR_READONLY should be -13") + print("PASS: test_error_readonly_constant") +end + -- Run all tests local function run_tests() print("Running TidesDB Lua tests...") diff --git a/tidesdb-0.5.6-1.rockspec b/tidesdb-0.5.7-1.rockspec similarity index 95% rename from tidesdb-0.5.6-1.rockspec rename to tidesdb-0.5.7-1.rockspec index 0e2fae1..4599b41 100644 --- a/tidesdb-0.5.6-1.rockspec +++ b/tidesdb-0.5.7-1.rockspec @@ -1,8 +1,8 @@ package = "tidesdb" -version = "0.5.6-1" +version = "0.5.7-1" source = { url = "git://github.com/tidesdb/tidesdb-lua.git", - tag = "v0.5.6" + tag = "v0.5.7" } description = { summary = "Official Lua bindings for TidesDB - A high-performance embedded key-value storage engine",