From 97917dedbc0e5dc149b093f88d020348d7563638 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 22 Jul 2026 08:50:07 +0800 Subject: [PATCH 1/5] fix(test): bootstrap trust superuser identity in cluster harness nodes TestClusterNode listeners run in AuthMode::Trust like production and the pgwire harness, which resolves but never fabricates a durable stored identity. Without bootstrapping it, every cluster node rejected the harness connect with "trust auth: user 'nodedb' does not exist". --- .../src/cluster_harness/node/lifecycle/spawn_full.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs b/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs index 4f7eec7f7..4905162af 100644 --- a/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs +++ b/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs @@ -107,6 +107,11 @@ impl TestClusterNode { &data_dir_path.join("system.redb"), )?, ); + // Mirror production/pgwire-harness bootstrap: both listeners run in + // `AuthMode::Trust`, which resolves — but never fabricates — a durable + // stored identity. Without this every node rejects the harness connect + // with `trust auth: user 'nodedb' does not exist`. + credentials.bootstrap_trust_superuser("nodedb")?; let mut shared = SharedState::new_with_credentials(dispatcher, Arc::clone(&wal), credentials)?; From ea2c67779eb4d10388cf2f1156ae8781c736f7e9 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 22 Jul 2026 08:50:15 +0800 Subject: [PATCH 2/5] fix(ddl): reclaim collection storage inline during single-node tenant purge A PurgeCollection apply only deactivates the catalog row (the crash-durable same-name barrier); actual owner/collection row deletion and storage reclaim happen in the post-apply finalize_purge half. On the clustered path the metadata applier schedules that reclaim on every node, but on the single-node path (log_index == 0) there is no applier, so tenant teardown never reclaimed the collection's storage and left the owner row behind. Drive the same hard-purge reclaim drop.rs runs inline for this case. Since the reclaim runs via block_in_place, the affected test needs a multi-thread runtime, and the catalog-entry unit test now drives both the apply and finalize_purge halves to match the real DROP_COLLECTION lifecycle. --- .../control/catalog_entry/tests/collection.rs | 6 ++++ .../shared/ddl/neutral/user/tenant_purge.rs | 28 +++++++++++++++++++ nodedb/tests/graph_drop_hides_edges.rs | 5 +++- 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/nodedb/src/control/catalog_entry/tests/collection.rs b/nodedb/src/control/catalog_entry/tests/collection.rs index 9b21d565a..467a021fe 100644 --- a/nodedb/src/control/catalog_entry/tests/collection.rs +++ b/nodedb/src/control/catalog_entry/tests/collection.rs @@ -102,6 +102,10 @@ fn purge_collection_is_scoped_to_database() { apply_to(&CatalogEntry::PutCollection(Box::new(default)), catalog); apply_to(&CatalogEntry::PutCollection(Box::new(other)), catalog); + // A `PurgeCollection` apply only deactivates the catalog row (the + // crash-durable same-name barrier); the row/owner/surrogate deletion is the + // `finalize_purge` half that runs once storage reclaim succeeds. Drive both + // to assert the delete is scoped to the target database. apply_to( &CatalogEntry::PurgeCollection { database_id: 9, @@ -110,6 +114,8 @@ fn purge_collection_is_scoped_to_database() { }, catalog, ); + crate::control::catalog_entry::apply::collection::finalize_purge(9, 1, "shared", catalog) + .expect("finalize purge for database 9"); assert!( catalog diff --git a/nodedb/src/control/server/shared/ddl/neutral/user/tenant_purge.rs b/nodedb/src/control/server/shared/ddl/neutral/user/tenant_purge.rs index 2bcb923f3..bfe827cf9 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/user/tenant_purge.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/user/tenant_purge.rs @@ -41,6 +41,34 @@ pub(super) fn purge_owned_for_tenant_teardown( crate::control::catalog_entry::apply::local::apply_locally_if_needed( state, &entry, log_index, ); + // A `PurgeCollection` apply only deactivates the catalog row — the + // durable owner/collection deletion and storage reclaim are the + // post-apply half. On the clustered path the metadata applier schedules + // that reclaim on every node; on the single-node path (`log_index == 0`) + // there is no applier, so drive the same reclaim `drop.rs` runs inline. + // Without this the teardown leaves the owner row and never reclaims the + // collection's storage. Other owner kinds delete fully in their apply. + if kind == OwnerKind::Collection && log_index == 0 { + let purge_lsn = state.wal.next_lsn().as_u64(); + let reclaim = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on( + crate::control::server::shared::ddl::neutral::collection::purge::hard_purge_collection( + state, + owner.database_id, + tenant.as_u64(), + &owner.object_name, + purge_lsn, + false, + ), + ) + }); + reclaim.map_err(|failure| { + ddl_err(format!( + "tenant teardown collection reclaim failed for '{}': {}", + owner.object_name, failure.error + )) + })?; + } if log_index == 0 { state .permissions diff --git a/nodedb/tests/graph_drop_hides_edges.rs b/nodedb/tests/graph_drop_hides_edges.rs index 78782f44a..980c7d932 100644 --- a/nodedb/tests/graph_drop_hides_edges.rs +++ b/nodedb/tests/graph_drop_hides_edges.rs @@ -105,7 +105,10 @@ async fn soft_drop_named_collection_stats_still_deactivated() { /// `DROP ... PURGE` must still fully remove edges and stats — the hard-purge /// path is untouched by this fix and must keep working exactly as before. -#[tokio::test] +/// +/// Multi-threaded runtime: the single-node purge path reclaims storage inline +/// via `block_in_place`, which panics on the current-thread runtime. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn hard_purge_removes_edges_and_stats() { let srv = TestServer::start().await; srv.exec("CREATE COLLECTION g_hard_purge").await.unwrap(); From 700966e055da474fb53eaee08b921510035375ef Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 22 Jul 2026 09:22:48 +0800 Subject: [PATCH 3/5] ci(test): strip symbols from dev/test binaries With --all-features every test binary statically links the full dependency closure, and there are hundreds of them, so an unstripped target/ runs to ~100 GB. Stripping removes better than a third of each binary. Add a debugging profile that keeps full symbols for when unsymbolised panics/backtraces are a problem. --- Cargo.toml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index c6c1bd143..309fac916 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -265,6 +265,13 @@ fluxbench = "0.1" [profile.dev] debug = "line-tables-only" +# Strip symbol tables from dev/test binaries. With --all-features every test +# binary statically links the full dependency closure, and there are hundreds of +# them, so an unstripped target/ runs to ~100 GB. Stripping removes better than a +# third of each binary. This unsymbolises panics/backtraces on ordinary dev +# builds; when you need to debug, build with `--profile debugging`, which keeps +# full symbols (see below). +strip = true [profile.dev.package."*"] debug = false @@ -286,3 +293,5 @@ strip = true [profile.debugging] inherits = "dev" debug = true +# dev strips symbols; override it here so the debugging profile stays symbolised. +strip = false From ddf57c6b6ae7264687191a24b44fa69a7e3e64ca Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 22 Jul 2026 10:03:31 +0800 Subject: [PATCH 4/5] fix(test): retry sequencer election race in crash harness queries The crash harness's simple query helpers issued statements right after connect, but /healthz can report ready before the Calvin sequencer group elects a leader, so early cross-shard writes could race the election and fail transiently. Add a bounded retry around the shared query path instead of failing immediately. --- nodedb/tests/crash_harness/mod.rs | 71 ++++++++++++++++++------------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/nodedb/tests/crash_harness/mod.rs b/nodedb/tests/crash_harness/mod.rs index 8b0519062..a4e98fa53 100644 --- a/nodedb/tests/crash_harness/mod.rs +++ b/nodedb/tests/crash_harness/mod.rs @@ -233,34 +233,54 @@ impl CrashHarness { self.wait_ready(Duration::from_secs(20)); } + /// Open a fresh pgwire connection, run one statement, and return the + /// resulting messages. Panics on connect/exec error. + /// + /// Retries the transient "no sequencer leader elected yet" startup + /// condition: `/healthz` intentionally reports ready before the Calvin + /// sequencer group has elected a leader (the sequencer is deliberately not + /// a data group in the readiness gate), so a cross-shard write issued in + /// the first moments of uptime can race the election and get a clean, + /// retryable error. On a loaded machine that window is wide enough to lose. + /// A real client retries; so does the harness, bounded, before writing. + async fn simple_query_ready(&self, sql: &str) -> Vec { + let deadline = Instant::now() + Duration::from_secs(20); + loop { + let (client, connection) = + tokio_postgres::connect(&self.pgwire_conn_str(), tokio_postgres::NoTls) + .await + .expect("connect for exec"); + let conn_handle = tokio::spawn(async move { + let _ = connection.await; + }); + let result = client.simple_query(sql).await; + drop(client); + let _ = conn_handle.await; + match result { + Ok(messages) => return messages, + Err(e) + if Instant::now() < deadline + && e.as_db_error().is_some_and(|db| { + db.message().contains("no sequencer leader elected yet") + }) => + { + tokio::time::sleep(Duration::from_millis(100)).await; + } + Err(e) => panic!("exec: {e}"), + } + } + } + /// Open a fresh pgwire connection, run one statement, and drop the /// connection. Panics on connect/exec error. pub async fn exec(&self, sql: &str) { - let (client, connection) = - tokio_postgres::connect(&self.pgwire_conn_str(), tokio_postgres::NoTls) - .await - .expect("connect for exec"); - let conn_handle = tokio::spawn(async move { - let _ = connection.await; - }); - client.simple_query(sql).await.expect("exec"); - drop(client); - let _ = conn_handle.await; + let _ = self.simple_query_ready(sql).await; } /// Run a query and return column `col` from every returned row, as text /// (via `simple_query`, so the value survives regardless of its type OID). pub async fn query_col(&self, sql: &str, col: &str) -> Vec { - let (client, connection) = - tokio_postgres::connect(&self.pgwire_conn_str(), tokio_postgres::NoTls) - .await - .expect("connect for query"); - let conn_handle = tokio::spawn(async move { - let _ = connection.await; - }); - let messages = client.simple_query(sql).await.expect("query"); - drop(client); - let _ = conn_handle.await; + let messages = self.simple_query_ready(sql).await; messages .iter() .filter_map(|m| match m { @@ -278,16 +298,7 @@ impl CrashHarness { /// `COUNT(*) AS n` — does not surface a usable column name through the /// pgwire row description, so aggregates must be read by index. pub async fn query_col_idx(&self, sql: &str, idx: usize) -> Vec { - let (client, connection) = - tokio_postgres::connect(&self.pgwire_conn_str(), tokio_postgres::NoTls) - .await - .expect("connect for query"); - let conn_handle = tokio::spawn(async move { - let _ = connection.await; - }); - let messages = client.simple_query(sql).await.expect("query"); - drop(client); - let _ = conn_handle.await; + let messages = self.simple_query_ready(sql).await; messages .iter() .filter_map(|m| match m { From df7f3cf7ddd4629720f09a72ef298ef54f3d3da3 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 22 Jul 2026 12:00:26 +0800 Subject: [PATCH 5/5] fix(test): authenticate native client as trust superuser in join OCC test The cluster harness runs trust auth with nodedb as the sole materialized superuser, but PoolConfig defaults to user=admin, which trust mode rejects. Authenticate as nodedb to match the pgwire path. --- .../tests/native_gather_join_in_txn_occ.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nodedb-cluster-tests/tests/native_gather_join_in_txn_occ.rs b/nodedb-cluster-tests/tests/native_gather_join_in_txn_occ.rs index 3cc2acd23..c556502d4 100644 --- a/nodedb-cluster-tests/tests/native_gather_join_in_txn_occ.rs +++ b/nodedb-cluster-tests/tests/native_gather_join_in_txn_occ.rs @@ -92,6 +92,13 @@ fn pinned_native_client(node: &TestClusterNode) -> NativeClient { NativeClient::new(PoolConfig { addr: format!("127.0.0.1:{}", node.native_port), max_size: 1, + // The cluster harness runs trust auth with `nodedb` as the sole + // materialized superuser; the `PoolConfig` default user is `admin`, + // which trust mode rejects as a non-existent identity. Authenticate as + // the harness superuser, matching the pgwire path (`user=nodedb`). + auth: nodedb_types::protocol::AuthMethod::Trust { + username: "nodedb".into(), + }, ..Default::default() }) }