From 5434141b3448f2203047dd71a7e3ac95b19525b8 Mon Sep 17 00:00:00 2001 From: Vincenzo Palazzo Date: Tue, 24 Mar 2026 21:28:16 +0100 Subject: [PATCH 1/3] sql: store short_channel_id as integer for efficient indexing Short channel IDs were stored as TEXT strings (e.g., "735095x480x1") in SQLite, which prevented efficient use of indexes on SCID columns. Change storage to INTEGER (the u64 encoding), using a custom "SCID" column type so the result-reading code can detect these columns and format them back as "NNNxNNNxNNN" strings for backward-compatible JSON output. Add two new SQL functions: - scid('NNNxNNNxNNN') -> integer: for efficient WHERE clause filtering - fmt_scid(integer) -> 'NNNxNNNxNNN': for formatting in SQL expressions Fixes #8941 --- contrib/msggen/msggen/schema.json | 19 ++++--- doc/schemas/listsqlschemas.json | 9 ++-- doc/schemas/sql-template.json | 10 ++-- plugins/sql.c | 85 +++++++++++++++++++++++++++++-- tests/test_plugin.py | 4 +- 5 files changed, 106 insertions(+), 21 deletions(-) diff --git a/contrib/msggen/msggen/schema.json b/contrib/msggen/msggen/schema.json index 32111a6396c7..2df583ce142a 100644 --- a/contrib/msggen/msggen/schema.json +++ b/contrib/msggen/msggen/schema.json @@ -29747,7 +29747,8 @@ "INTEGER", "BLOB", "TEXT", - "REAL" + "REAL", + "SCID" ], "description": [ "The SQL type of the column." @@ -29875,15 +29876,15 @@ }, { "name": "short_channel_id", - "type": "TEXT" + "type": "SCID" }, { "name": "alias_local", - "type": "TEXT" + "type": "SCID" }, { "name": "alias_remote", - "type": "TEXT" + "type": "SCID" }, { "name": "opener", @@ -37959,9 +37960,11 @@ " * JSON: string", " * sqlite3: TEXT", "", - "* *short_channel_id*. A short-channel-id of form 1x2x3.", + "* *short_channel_id*. A short-channel-id of form 1x2x3. Stored as an integer internally for efficient indexing.", " * JSON: string", - " * sqlite3: TEXT" + " * sqlite3: SCID (INTEGER affinity)", + "", + "You can use the `scid()` function to convert a short_channel_id string to its integer representation for queries, e.g. `WHERE in_channel = scid('1x2x3')`. The `fmt_scid()` function converts back to string form." ], "permitted_sqlite3_functions": [ "Writing to the database is not permitted, and limits are placed on various other query parameters.", @@ -37990,7 +37993,9 @@ "* total", "* unixepoch", "* json_object", - "* json_group_array" + "* json_group_array", + "* scid", + "* fmt_scid" ], "tables": [ "Note that tables which have a `created_index` field use that as the primary key (and `rowid` is an alias to this), otherwise an explicit `rowid` integer primary key is generated, whose value changes on each refresh. This field is used for related tables to refer to specific rows in their parent. (sqlite3 usually has this as an implicit column, but we make it explicit as the implicit version is not allowed to be used as a foreign key).", diff --git a/doc/schemas/listsqlschemas.json b/doc/schemas/listsqlschemas.json index f312742469b5..70d7b502cec3 100644 --- a/doc/schemas/listsqlschemas.json +++ b/doc/schemas/listsqlschemas.json @@ -68,7 +68,8 @@ "INTEGER", "BLOB", "TEXT", - "REAL" + "REAL", + "SCID" ], "description": [ "The SQL type of the column." @@ -196,15 +197,15 @@ }, { "name": "short_channel_id", - "type": "TEXT" + "type": "SCID" }, { "name": "alias_local", - "type": "TEXT" + "type": "SCID" }, { "name": "alias_remote", - "type": "TEXT" + "type": "SCID" }, { "name": "opener", diff --git a/doc/schemas/sql-template.json b/doc/schemas/sql-template.json index 2f71b36bb068..58c3301137e6 100644 --- a/doc/schemas/sql-template.json +++ b/doc/schemas/sql-template.json @@ -78,9 +78,11 @@ " * JSON: string", " * sqlite3: TEXT", "", - "* *short_channel_id*. A short-channel-id of form 1x2x3.", + "* *short_channel_id*. A short-channel-id of form 1x2x3. Stored as an integer internally for efficient indexing.", " * JSON: string", - " * sqlite3: TEXT" + " * sqlite3: SCID (INTEGER affinity)", + "", + "You can use the `scid()` function to convert a short_channel_id string to its integer representation for queries, e.g. `WHERE in_channel = scid('1x2x3')`. The `fmt_scid()` function converts back to string form." ], "permitted_sqlite3_functions": [ "Writing to the database is not permitted, and limits are placed on various other query parameters.", @@ -109,7 +111,9 @@ "* total", "* unixepoch", "* json_object", - "* json_group_array" + "* json_group_array", + "* scid", + "* fmt_scid" ], "tables": [ "Note that tables which have a `created_index` field use that as the primary key (and `rowid` is an alias to this), otherwise an explicit `rowid` integer primary key is generated, whose value changes on each refresh. This field is used for related tables to refer to specific rows in their parent. (sqlite3 usually has this as an implicit column, but we make it explicit as the implicit version is not allowed to be used as a foreign key).", diff --git a/plugins/sql.c b/plugins/sql.c index f065fc588e45..70ed3fc84ddd 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -70,7 +70,7 @@ static const struct fieldtypemap fieldtypemap[] = { { "boolean", "INTEGER" }, /* FIELD_BOOL */ { "number", "REAL" }, /* FIELD_NUMBER */ { "string", "TEXT" }, /* FIELD_STRING */ - { "short_channel_id", "TEXT" }, /* FIELD_SCID */ + { "short_channel_id", "SCID" }, /* FIELD_SCID */ { "outpoint", "TEXT" }, /* FIELD_OUTPOINT */ }; @@ -245,6 +245,52 @@ static enum fieldtype find_fieldtype(const jsmntok_t *name) name->end - name->start, schemas + name->start); } +/* SQLite custom function: scid('NNNxNNNxNNN') -> u64 integer. + * Allows efficient queries like: WHERE in_channel = scid('735095x480x1') */ +static void sql_scid_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) +{ + struct short_channel_id scid; + const char *str; + + if (argc != 1) { + sqlite3_result_error(ctx, "scid() requires exactly one argument", -1); + return; + } + if (sqlite3_value_type(argv[0]) == SQLITE_NULL) { + sqlite3_result_null(ctx); + return; + } + + str = (const char *)sqlite3_value_text(argv[0]); + if (!str || !short_channel_id_from_str(str, strlen(str), &scid)) { + sqlite3_result_error(ctx, "invalid short_channel_id format, expected NNNxNNNxNNN", -1); + return; + } + + sqlite3_result_int64(ctx, scid.u64); +} + +/* SQLite custom function: fmt_scid(u64) -> 'NNNxNNNxNNN' string. + * Useful for displaying integer SCIDs in text format within SQL expressions. */ +static void sql_fmt_scid_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) +{ + struct short_channel_id scid; + char *str; + + if (argc != 1) { + sqlite3_result_error(ctx, "fmt_scid() requires exactly one argument", -1); + return; + } + if (sqlite3_value_type(argv[0]) == SQLITE_NULL) { + sqlite3_result_null(ctx); + return; + } + + scid.u64 = sqlite3_value_int64(argv[0]); + str = fmt_short_channel_id(tmpctx, scid); + sqlite3_result_text(ctx, str, -1, SQLITE_TRANSIENT); +} + static struct sqlite3 *sqlite_setup(struct plugin *plugin) { int err; @@ -301,6 +347,12 @@ static struct sqlite3 *sqlite_setup(struct plugin *plugin) plugin_err(plugin, "Could not disable sync: %s", errmsg); } + /* Register custom SCID functions for integer<->text conversion */ + sqlite3_create_function(db, "scid", 1, SQLITE_UTF8, NULL, + sql_scid_func, NULL, NULL); + sqlite3_create_function(db, "fmt_scid", 1, SQLITE_UTF8, NULL, + sql_fmt_scid_func, NULL, NULL); + return db; } @@ -424,6 +476,10 @@ static int sqlite_authorize(void *dbq_, int code, return SQLITE_OK; if (streq(b, "json_group_array")) return SQLITE_OK; + if (streq(b, "scid")) + return SQLITE_OK; + if (streq(b, "fmt_scid")) + return SQLITE_OK; } /* See https://www.sqlite.org/c3ref/c_alter_table.html to decode these! */ @@ -465,7 +521,15 @@ static struct command_result *refresh_complete(struct command *cmd, switch (sqlite3_column_type(dbq->stmt, i)) { case SQLITE_INTEGER: { s64 v = sqlite3_column_int64(dbq->stmt, i); - json_add_s64(ret, NULL, v); + const char *decltype = sqlite3_column_decltype(dbq->stmt, i); + if (decltype && streq(decltype, "SCID")) { + struct short_channel_id scid; + scid.u64 = (u64)v; + json_add_string(ret, NULL, + fmt_short_channel_id(tmpctx, scid)); + } else { + json_add_s64(ret, NULL, v); + } break; } case SQLITE_FLOAT: { @@ -790,7 +854,18 @@ static struct command_result *process_json_obj(struct command *cmd, } sqlite3_bind_int64(stmt, (*sqloff)++, valmsat.millisatoshis /* Raw: db */); break; - case FIELD_SCID: + case FIELD_SCID: { + struct short_channel_id scid; + if (!json_to_short_channel_id(buf, coltok, &scid)) { + return command_fail(cmd, LIGHTNINGD, + "column %zu row %zu not a valid short_channel_id: %.*s", + i, row, + json_tok_full_len(coltok), + json_tok_full(buf, coltok)); + } + sqlite3_bind_int64(stmt, (*sqloff)++, scid.u64); + break; + } case FIELD_STRING: case FIELD_OUTPOINT: sqlite3_bind_text(stmt, (*sqloff)++, buf + coltok->start, @@ -1029,8 +1104,8 @@ static void delete_channel_from_db(struct command *cmd, err = sqlite3_exec(sql->db, tal_fmt(tmpctx, "DELETE FROM channels" - " WHERE short_channel_id = '%s'", - fmt_short_channel_id(tmpctx, scid)), + " WHERE short_channel_id = %"PRIu64, + scid.u64), NULL, NULL, &errmsg); if (err != SQLITE_OK) plugin_err(cmd->plugin, "Could not delete from channels: %s", diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 31c253c134d4..ba07d2e0b741 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -4469,7 +4469,7 @@ def test_sql(node_factory, bitcoind): 'pubkey': 'BLOB', 'secret': 'BLOB', 'number': 'REAL', - 'short_channel_id': 'TEXT'} + 'short_channel_id': 'SCID'} # Check schemas match for table, schema in expected_schemas.items(): @@ -4640,7 +4640,7 @@ def test_sql(node_factory, bitcoind): l1.rpc.xpay(l3.rpc.invoice(amount_msat=1000000, label='inv1000', description='description 1000 msat')['bolt11']) # Two channels, l1->l3 *may* have an HTLC in flight. - ret = l1.rpc.sql("SELECT json_object('peer_id', hex(pc.peer_id), 'alias', alias, 'scid', short_channel_id, 'htlcs'," + ret = l1.rpc.sql("SELECT json_object('peer_id', hex(pc.peer_id), 'alias', alias, 'scid', fmt_scid(short_channel_id), 'htlcs'," " (SELECT json_group_array(json_object('id', hex(id), 'amount_msat', amount_msat))" " FROM peerchannels_htlcs ph WHERE ph.row = pc.rowid)) FROM peerchannels pc JOIN nodes n" " ON pc.peer_id = n.nodeid ORDER BY n.alias, pc.peer_id;") From 142280e0b1999176bbeba4e12f32e1d1a9c4e9fd Mon Sep 17 00:00:00 2001 From: Vincenzo Palazzo Date: Wed, 25 Mar 2026 13:49:50 +0100 Subject: [PATCH 2/3] sql: rewrite scid string literals to use scid() for index support When users query with WHERE in_channel='735095x480x1', the string literal is compared directly against the integer SCID column, forcing SQLite to perform a full table scan even when an index exists. Automatically rewrite scid string literals (matching NNNxNNNxNNN format) to use the scid() function before passing the query to SQLite, so '735095x480x1' becomes scid('735095x480x1'). This allows SQLite to use indexes on SCID columns transparently. Queries already using scid() explicitly are detected and left unchanged. Changelog-Fixed: sql plugin now automatically translates short_channel_id string literals to integers for efficient index usage. Fixes: #8941 --- plugins/sql.c | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/plugins/sql.c b/plugins/sql.c index 70ed3fc84ddd..a145bc68ccfc 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -1479,6 +1479,63 @@ static struct command_result *refresh_tables(struct command *cmd, return td->refresh(cmd, dbq->tables[0], dbq); } +/* Check if a string is a valid short_channel_id (NNNxNNNxNNN format) */ +static bool looks_like_scid(const char *str, size_t len) +{ + struct short_channel_id scid; + + return short_channel_id_from_str(str, len, &scid); +} + +/* Rewrite SQL query to wrap scid string literals with scid() function. + * This transforms '735095x480x1' into scid('735095x480x1') so that + * SQLite can use indexes on integer SCID columns. */ +static const char *rewrite_scid_literals(const tal_t *ctx, const char *query) +{ + char *result = tal_strdup(ctx, ""); + const char *p = query; + + while (*p) { + /* Look for single-quoted string literals */ + if (*p == '\'') { + const char *start = p + 1; + const char *end = strchr(start, '\''); + + if (!end) { + /* Unterminated quote, just copy rest */ + tal_append_fmt(&result, "%s", p); + break; + } + + if (looks_like_scid(start, end - start)) { + /* Check if already wrapped in scid() by looking + * back for "scid(" before the quote */ + bool already_wrapped = false; + if (p - query >= 5) { + const char *before = p - 5; + if (strncmp(before, "scid(", 5) == 0) + already_wrapped = true; + } + if (!already_wrapped) { + tal_append_fmt(&result, "scid('%.*s')", + (int)(end - start), start); + p = end + 1; + continue; + } + } + /* Not a scid or already wrapped: copy quote and content */ + tal_append_fmt(&result, "%.*s", + (int)(end - p + 1), p); + p = end + 1; + } else { + tal_append_fmt(&result, "%c", *p); + p++; + } + } + + return result; +} + static struct command_result *json_sql(struct command *cmd, const char *buffer, const jsmntok_t *params) @@ -1493,6 +1550,10 @@ static struct command_result *json_sql(struct command *cmd, NULL)) return command_param_failed(); + /* Rewrite scid string literals to use scid() function so + * SQLite can use indexes on integer SCID columns. */ + query = rewrite_scid_literals(tmpctx, query); + dbq->tables = tal_arr(dbq, struct table_desc *, 0); dbq->authfail = NULL; dbq->cmd = cmd; From c248e4108d0a18d9da51ce105592c09794f85e4f Mon Sep 17 00:00:00 2001 From: Vincenzo Palazzo Date: Mon, 17 Aug 2026 13:26:43 +0200 Subject: [PATCH 3/3] changelog: add Changed entry for #9430 Signed-off-by: Vincenzo Palazzo --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dd85a1e9c59..43fc7e1eed64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [Unreleased] + +### Changed + + + - plugins: the `sql` plugin now stores short_channel_id as integers; use `scid('1x2x3')` in queries instead of string literals (breaking change for string comparisons) ([#9430]) + +[#9430]: https://github.com/ElementsProject/lightning/pull/9430 + ## [26.06.6] - 2026-07-20: "Quantum-Resistant Lightning Channel III" v26.06.3, v26.06.4, and v26.06.5 had issues during publishing with the pypi releases and were deleted.