From 253768a9ae3650edf10c2ae2f3a62eed5c0b5611 Mon Sep 17 00:00:00 2001 From: Vincenzo Palazzo Date: Tue, 24 Mar 2026 21:28:16 +0100 Subject: [PATCH 1/2] sql: store short_channel_id as integer for efficient indexing Short channel IDs were stored as TEXT strings (e.g., "735095x480x1") in SQLite, which prevented efficient use of indexes on SCID columns. Change storage to INTEGER (the u64 encoding), using a custom "SCID" column type so the result-reading code can detect these columns and format them back as "NNNxNNNxNNN" strings for backward-compatible JSON output. Add two new SQL functions: - scid('NNNxNNNxNNN') -> integer: for efficient WHERE clause filtering - fmt_scid(integer) -> 'NNNxNNNxNNN': for formatting in SQL expressions Fixes #8941 --- contrib/msggen/msggen/schema.json | 19 ++++--- doc/schemas/listsqlschemas.json | 9 ++-- doc/schemas/sql-template.json | 10 ++-- plugins/sql.c | 85 +++++++++++++++++++++++++++++-- tests/test_plugin.py | 4 +- 5 files changed, 106 insertions(+), 21 deletions(-) diff --git a/contrib/msggen/msggen/schema.json b/contrib/msggen/msggen/schema.json index 0215a35b841f..e277b40b7ff1 100644 --- a/contrib/msggen/msggen/schema.json +++ b/contrib/msggen/msggen/schema.json @@ -26381,7 +26381,8 @@ "INTEGER", "BLOB", "TEXT", - "REAL" + "REAL", + "SCID" ], "description": [ "The SQL type of the column." @@ -26501,15 +26502,15 @@ }, { "name": "short_channel_id", - "type": "TEXT" + "type": "SCID" }, { "name": "alias_local", - "type": "TEXT" + "type": "SCID" }, { "name": "alias_remote", - "type": "TEXT" + "type": "SCID" }, { "name": "opener", @@ -34093,9 +34094,11 @@ " * JSON: string", " * sqlite3: TEXT", "", - "* *short_channel_id*. A short-channel-id of form 1x2x3.", + "* *short_channel_id*. A short-channel-id of form 1x2x3. Stored as an integer internally for efficient indexing.", " * JSON: string", - " * sqlite3: TEXT" + " * sqlite3: SCID (INTEGER affinity)", + "", + "You can use the `scid()` function to convert a short_channel_id string to its integer representation for queries, e.g. `WHERE in_channel = scid('1x2x3')`. The `fmt_scid()` function converts back to string form." ], "permitted_sqlite3_functions": [ "Writing to the database is not permitted, and limits are placed on various other query parameters.", @@ -34124,7 +34127,9 @@ "* total", "* unixepoch", "* json_object", - "* json_group_array" + "* json_group_array", + "* scid", + "* fmt_scid" ], "tables": [ "Note that tables which have a `created_index` field use that as the primary key (and `rowid` is an alias to this), otherwise an explicit `rowid` integer primary key is generated, whose value changes on each refresh. This field is used for related tables to refer to specific rows in their parent. (sqlite3 usually has this as an implicit column, but we make it explicit as the implicit version is not allowed to be used as a foreign key).", diff --git a/doc/schemas/listsqlschemas.json b/doc/schemas/listsqlschemas.json index 4ab6e1ad5aac..43bdca4e5f97 100644 --- a/doc/schemas/listsqlschemas.json +++ b/doc/schemas/listsqlschemas.json @@ -68,7 +68,8 @@ "INTEGER", "BLOB", "TEXT", - "REAL" + "REAL", + "SCID" ], "description": [ "The SQL type of the column." @@ -188,15 +189,15 @@ }, { "name": "short_channel_id", - "type": "TEXT" + "type": "SCID" }, { "name": "alias_local", - "type": "TEXT" + "type": "SCID" }, { "name": "alias_remote", - "type": "TEXT" + "type": "SCID" }, { "name": "opener", diff --git a/doc/schemas/sql-template.json b/doc/schemas/sql-template.json index 758eac7b1c48..d7b9e17029c8 100644 --- a/doc/schemas/sql-template.json +++ b/doc/schemas/sql-template.json @@ -78,9 +78,11 @@ " * JSON: string", " * sqlite3: TEXT", "", - "* *short_channel_id*. A short-channel-id of form 1x2x3.", + "* *short_channel_id*. A short-channel-id of form 1x2x3. Stored as an integer internally for efficient indexing.", " * JSON: string", - " * sqlite3: TEXT" + " * sqlite3: SCID (INTEGER affinity)", + "", + "You can use the `scid()` function to convert a short_channel_id string to its integer representation for queries, e.g. `WHERE in_channel = scid('1x2x3')`. The `fmt_scid()` function converts back to string form." ], "permitted_sqlite3_functions": [ "Writing to the database is not permitted, and limits are placed on various other query parameters.", @@ -109,7 +111,9 @@ "* total", "* unixepoch", "* json_object", - "* json_group_array" + "* json_group_array", + "* scid", + "* fmt_scid" ], "tables": [ "Note that tables which have a `created_index` field use that as the primary key (and `rowid` is an alias to this), otherwise an explicit `rowid` integer primary key is generated, whose value changes on each refresh. This field is used for related tables to refer to specific rows in their parent. (sqlite3 usually has this as an implicit column, but we make it explicit as the implicit version is not allowed to be used as a foreign key).", diff --git a/plugins/sql.c b/plugins/sql.c index 24c3b28c5a40..474e20e00002 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -71,7 +71,7 @@ static const struct fieldtypemap fieldtypemap[] = { { "boolean", "INTEGER" }, /* FIELD_BOOL */ { "number", "REAL" }, /* FIELD_NUMBER */ { "string", "TEXT" }, /* FIELD_STRING */ - { "short_channel_id", "TEXT" }, /* FIELD_SCID */ + { "short_channel_id", "SCID" }, /* FIELD_SCID */ { "outpoint", "TEXT" }, /* FIELD_OUTPOINT */ }; @@ -227,6 +227,52 @@ static enum fieldtype find_fieldtype(const jsmntok_t *name) name->end - name->start, schemas + name->start); } +/* SQLite custom function: scid('NNNxNNNxNNN') -> u64 integer. + * Allows efficient queries like: WHERE in_channel = scid('735095x480x1') */ +static void sql_scid_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) +{ + struct short_channel_id scid; + const char *str; + + if (argc != 1) { + sqlite3_result_error(ctx, "scid() requires exactly one argument", -1); + return; + } + if (sqlite3_value_type(argv[0]) == SQLITE_NULL) { + sqlite3_result_null(ctx); + return; + } + + str = (const char *)sqlite3_value_text(argv[0]); + if (!str || !short_channel_id_from_str(str, strlen(str), &scid)) { + sqlite3_result_error(ctx, "invalid short_channel_id format, expected NNNxNNNxNNN", -1); + return; + } + + sqlite3_result_int64(ctx, scid.u64); +} + +/* SQLite custom function: fmt_scid(u64) -> 'NNNxNNNxNNN' string. + * Useful for displaying integer SCIDs in text format within SQL expressions. */ +static void sql_fmt_scid_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) +{ + struct short_channel_id scid; + char *str; + + if (argc != 1) { + sqlite3_result_error(ctx, "fmt_scid() requires exactly one argument", -1); + return; + } + if (sqlite3_value_type(argv[0]) == SQLITE_NULL) { + sqlite3_result_null(ctx); + return; + } + + scid.u64 = sqlite3_value_int64(argv[0]); + str = fmt_short_channel_id(tmpctx, scid); + sqlite3_result_text(ctx, str, -1, SQLITE_TRANSIENT); +} + static struct sqlite3 *sqlite_setup(struct plugin *plugin) { int err; @@ -283,6 +329,12 @@ static struct sqlite3 *sqlite_setup(struct plugin *plugin) plugin_err(plugin, "Could not disable sync: %s", errmsg); } + /* Register custom SCID functions for integer<->text conversion */ + sqlite3_create_function(db, "scid", 1, SQLITE_UTF8, NULL, + sql_scid_func, NULL, NULL); + sqlite3_create_function(db, "fmt_scid", 1, SQLITE_UTF8, NULL, + sql_fmt_scid_func, NULL, NULL); + return db; } @@ -406,6 +458,10 @@ static int sqlite_authorize(void *dbq_, int code, return SQLITE_OK; if (streq(b, "json_group_array")) return SQLITE_OK; + if (streq(b, "scid")) + return SQLITE_OK; + if (streq(b, "fmt_scid")) + return SQLITE_OK; } /* See https://www.sqlite.org/c3ref/c_alter_table.html to decode these! */ @@ -447,7 +503,15 @@ static struct command_result *refresh_complete(struct command *cmd, switch (sqlite3_column_type(dbq->stmt, i)) { case SQLITE_INTEGER: { s64 v = sqlite3_column_int64(dbq->stmt, i); - json_add_s64(ret, NULL, v); + const char *decltype = sqlite3_column_decltype(dbq->stmt, i); + if (decltype && streq(decltype, "SCID")) { + struct short_channel_id scid; + scid.u64 = (u64)v; + json_add_string(ret, NULL, + fmt_short_channel_id(tmpctx, scid)); + } else { + json_add_s64(ret, NULL, v); + } break; } case SQLITE_FLOAT: { @@ -781,7 +845,18 @@ static struct command_result *process_json_obj(struct command *cmd, } sqlite3_bind_int64(stmt, (*sqloff)++, valmsat.millisatoshis /* Raw: db */); break; - case FIELD_SCID: + case FIELD_SCID: { + struct short_channel_id scid; + if (!json_to_short_channel_id(buf, coltok, &scid)) { + return command_fail(cmd, LIGHTNINGD, + "column %zu row %zu not a valid short_channel_id: %.*s", + i, row, + json_tok_full_len(coltok), + json_tok_full(buf, coltok)); + } + sqlite3_bind_int64(stmt, (*sqloff)++, scid.u64); + break; + } case FIELD_STRING: case FIELD_OUTPOINT: sqlite3_bind_text(stmt, (*sqloff)++, buf + coltok->start, @@ -976,8 +1051,8 @@ static void delete_channel_from_db(struct command *cmd, err = sqlite3_exec(sql->db, tal_fmt(tmpctx, "DELETE FROM channels" - " WHERE short_channel_id = '%s'", - fmt_short_channel_id(tmpctx, scid)), + " WHERE short_channel_id = %"PRIu64, + scid.u64), NULL, NULL, &errmsg); if (err != SQLITE_OK) plugin_err(cmd->plugin, "Could not delete from channels: %s", diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 9644d4161b62..eb2d11141d80 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -4129,7 +4129,7 @@ def test_sql(node_factory, bitcoind): 'pubkey': 'BLOB', 'secret': 'BLOB', 'number': 'REAL', - 'short_channel_id': 'TEXT'} + 'short_channel_id': 'SCID'} # Check schemas match for table, schema in expected_schemas.items(): @@ -4295,7 +4295,7 @@ def test_sql(node_factory, bitcoind): l1.rpc.pay(l3.rpc.invoice(amount_msat=1000000, label='inv1000', description='description 1000 msat')['bolt11']) # Two channels, l1->l3 *may* have an HTLC in flight. - ret = l1.rpc.sql("SELECT json_object('peer_id', hex(pc.peer_id), 'alias', alias, 'scid', short_channel_id, 'htlcs'," + ret = l1.rpc.sql("SELECT json_object('peer_id', hex(pc.peer_id), 'alias', alias, 'scid', fmt_scid(short_channel_id), 'htlcs'," " (SELECT json_group_array(json_object('id', hex(id), 'amount_msat', amount_msat))" " FROM peerchannels_htlcs ph WHERE ph.row = pc.rowid)) FROM peerchannels pc JOIN nodes n" " ON pc.peer_id = n.nodeid ORDER BY n.alias, pc.peer_id;") From 84562dfed2a50b4c2bb74acac76e26e515d45431 Mon Sep 17 00:00:00 2001 From: Vincenzo Palazzo Date: Wed, 25 Mar 2026 13:49:50 +0100 Subject: [PATCH 2/2] sql: rewrite scid string literals to use scid() for index support When users query with WHERE in_channel='735095x480x1', the string literal is compared directly against the integer SCID column, forcing SQLite to perform a full table scan even when an index exists. Automatically rewrite scid string literals (matching NNNxNNNxNNN format) to use the scid() function before passing the query to SQLite, so '735095x480x1' becomes scid('735095x480x1'). This allows SQLite to use indexes on SCID columns transparently. Queries already using scid() explicitly are detected and left unchanged. Changelog-Fixed: sql plugin now automatically translates short_channel_id string literals to integers for efficient index usage. Fixes: #8941 --- plugins/sql.c | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/plugins/sql.c b/plugins/sql.c index 474e20e00002..aed2a4fd43a6 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -1322,6 +1322,63 @@ static struct command_result *refresh_tables(struct command *cmd, return td->refresh(cmd, dbq->tables[0], dbq); } +/* Check if a string is a valid short_channel_id (NNNxNNNxNNN format) */ +static bool looks_like_scid(const char *str, size_t len) +{ + struct short_channel_id scid; + + return short_channel_id_from_str(str, len, &scid); +} + +/* Rewrite SQL query to wrap scid string literals with scid() function. + * This transforms '735095x480x1' into scid('735095x480x1') so that + * SQLite can use indexes on integer SCID columns. */ +static const char *rewrite_scid_literals(const tal_t *ctx, const char *query) +{ + char *result = tal_strdup(ctx, ""); + const char *p = query; + + while (*p) { + /* Look for single-quoted string literals */ + if (*p == '\'') { + const char *start = p + 1; + const char *end = strchr(start, '\''); + + if (!end) { + /* Unterminated quote, just copy rest */ + tal_append_fmt(&result, "%s", p); + break; + } + + if (looks_like_scid(start, end - start)) { + /* Check if already wrapped in scid() by looking + * back for "scid(" before the quote */ + bool already_wrapped = false; + if (p - query >= 5) { + const char *before = p - 5; + if (strncmp(before, "scid(", 5) == 0) + already_wrapped = true; + } + if (!already_wrapped) { + tal_append_fmt(&result, "scid('%.*s')", + (int)(end - start), start); + p = end + 1; + continue; + } + } + /* Not a scid or already wrapped: copy quote and content */ + tal_append_fmt(&result, "%.*s", + (int)(end - p + 1), p); + p = end + 1; + } else { + tal_append_fmt(&result, "%c", *p); + p++; + } + } + + return result; +} + static struct command_result *json_sql(struct command *cmd, const char *buffer, const jsmntok_t *params) @@ -1336,6 +1393,10 @@ static struct command_result *json_sql(struct command *cmd, NULL)) return command_param_failed(); + /* Rewrite scid string literals to use scid() function so + * SQLite can use indexes on integer SCID columns. */ + query = rewrite_scid_literals(tmpctx, query); + dbq->tables = tal_arr(dbq, struct table_desc *, 0); dbq->authfail = NULL; dbq->cmd = cmd;