From f6a5b2f7d8fcdefa6077ece24c8eccd1740a0eeb Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 13:45:11 -0500 Subject: [PATCH 01/61] feat(node): add per-IP write-surface rate brake and apply to write_routes Introduces a dedicated write_rate_limiter bucket (GITLAWB_WRITE_RATE_LIMIT, default 600/hr, 0 disables), separate from the creation brake so a write flood cannot drain the creation budget (KTD-1). Attaches rate_limit_by_ip to write_routes, mirroring creation_routes. Per-DID deliberately not paired (a DID farm never trips it). Tests (route-level, execution-verified against Postgres): a write_routes sink (PUT /star) is 429'd before auth; an exhausted write bucket does not throttle repo creation (separate-bucket property). --- crates/gitlawb-node/src/api/repos.rs | 76 +++++++++++++++++++++++++ crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 22 +++++++ crates/gitlawb-node/src/server.rs | 16 +++++- crates/gitlawb-node/src/state.rs | 10 ++++ crates/gitlawb-node/src/test_support.rs | 1 + 6 files changed, 124 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b9cbc352..7bb3c000 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2625,4 +2625,80 @@ mod tests { "repo creation must be IP-throttled before signature verification" ); } + + #[sqlx::test] + async fn write_route_is_rate_limited_by_ip(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Tiny write bucket, keyed on the socket peer (no trusted proxy). + state.write_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.88:7000".parse().unwrap(); + // Exhaust this peer's single-request write budget up front. + assert!(state.write_rate_limiter.check(&peer.ip().to_string()).await); + + let router = crate::server::build_router(state); + // A write_routes sink (star). The IP brake is outermost, so the 429 + // fires before auth/handler — the path only needs to match. + let mut req = Request::builder() + .method(Method::PUT) + .uri("/api/v1/repos/someowner/somerepo/star") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let status = router.oneshot(req).await.unwrap().status(); + assert_eq!( + status, + StatusCode::TOO_MANY_REQUESTS, + "a write_routes sink must be IP-throttled before signature verification" + ); + } + + // KTD-1: the write bucket is separate from the creation bucket, so a write + // flood must not consume the creation budget (and vice versa). + #[sqlx::test] + async fn write_flood_does_not_drain_creation_budget(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Exhaust the write bucket for this peer; leave the creation bucket ample. + state.write_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.create_ip_rate_limiter = + crate::rate_limit::RateLimiter::new(1000, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.99:7000".parse().unwrap(); + assert!(state.write_rate_limiter.check(&peer.ip().to_string()).await); + + let router = crate::server::build_router(state); + // Creation from the same peer must NOT be 429 — its bucket is untouched. + // (It fails later on missing signature; the point is it is not throttled.) + let mut req = Request::builder() + .method(Method::POST) + .uri("/api/v1/repos") + .header("content-type", "application/json") + .body(Body::from(r#"{"name":"legit","is_public":true}"#)) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let status = router.oneshot(req).await.unwrap().status(); + assert_ne!( + status, + StatusCode::TOO_MANY_REQUESTS, + "an exhausted write bucket must not throttle repo creation (separate buckets)" + ); + } } diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 720fb3ae..a9baa314 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -515,6 +515,7 @@ mod tests { repo_store: crate::git::repo_store::RepoStore::for_testing(PathBuf::from("/tmp"), pool), rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), + write_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index aa0483db..4fd6e922 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -307,6 +307,27 @@ async fn main() -> Result<()> { tracing::warn!("GITLAWB_CREATE_RATE_LIMIT=0 — per-IP creation rate limiting disabled"); } + // Per-IP brake for the authenticated non-creation write surface (issue/PR + // comments, labels, stars, merges, protect, replicas, visibility, tasks, + // bounties, profile, GraphQL mutations). Its own bucket, separate from the + // creation brake, so a write flood cannot drain the creation budget and vice + // versa (same rationale as the sync_trigger / peer_write split). Sized above + // any legitimate per-IP write rate — real agent automation makes many small + // writes per hour. GITLAWB_WRITE_RATE_LIMIT overrides; 0 disables. Bounded + // key set — the key is a client-influenced IP. + let write_limit = std::env::var("GITLAWB_WRITE_RATE_LIMIT") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(600); + let write_rate_limiter = rate_limit::RateLimiter::new_bounded( + write_limit, + std::time::Duration::from_secs(3600), + 200_000, + ); + if write_limit == 0 { + tracing::warn!("GITLAWB_WRITE_RATE_LIMIT=0 — per-IP write rate limiting disabled"); + } + // Push-path flood brake: max git-receive-pack requests per client IP per // hour (counts both the info/refs advertisement and the push POST). Sized // for heavy agent automation while still stopping flood traffic (the June @@ -373,6 +394,7 @@ async fn main() -> Result<()> { repo_store, rate_limiter, create_ip_rate_limiter, + write_rate_limiter, push_rate_limiter, push_limiter_trust, sync_trigger_rate_limiter, diff --git a/crates/gitlawb-node/src/server.rs b/crates/gitlawb-node/src/server.rs index f4c0d3e3..5df9ca0d 100644 --- a/crates/gitlawb-node/src/server.rs +++ b/crates/gitlawb-node/src/server.rs @@ -111,7 +111,17 @@ pub fn build_router(state: AppState) -> Router { .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) .layer(axum::Extension(create_ip_limiter)); - // ── Write routes — require HTTP Signature (no rate limit) ───────────── + // ── Write routes — require HTTP Signature + per-IP write brake ──────── + // Same per-IP flood defense as the creation routes, on its own `write` + // bucket (see `AppState::write_rate_limiter`). Per-DID is deliberately not + // paired (a DID farm never trips it; busy legitimate agents would false- + // positive). The bundled visibility GET is a read that also draws from the + // write bucket — harmless, since the bucket is sized far above any legit + // per-IP read rate. + let write_ip_limiter = rate_limit::IpRateLimiter { + limiter: state.write_rate_limiter.clone(), + trust: state.push_limiter_trust, + }; let write_routes = add_auth_layers( Router::new() .route( @@ -181,7 +191,9 @@ pub fn build_router(state: AppState) -> Router { axum::routing::delete(agents::deregister_agent), ), state.clone(), - ); + ) + .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) + .layer(axum::Extension(write_ip_limiter.clone())); // Body limit is raised to GITLAWB_MAX_PACK_BYTES (default 2 GB) for git // routes only — all other API routes keep axum's default 2 MB cap. diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 6a84b3c2..d38541f9 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -61,6 +61,16 @@ pub struct AppState { /// resolved client IP is what actually stops a single-source flood (same /// rationale as `push_rate_limiter`). Keyed by `push_limiter_trust`. pub create_ip_rate_limiter: RateLimiter, + /// Per-client-IP rate limiter for the authenticated write surface that is + /// not repo/agent creation: issue/PR comments, labels, stars, merges, + /// protect/unprotect, replicas, visibility, tasks, bounties, profile, and + /// the GraphQL `MutationRoot`. Separate bucket from `create_ip_rate_limiter` + /// (its own budget, per the `sync_trigger`/`peer_write` precedent) so a + /// write flood cannot drain the creation budget and vice versa. Per-DID is + /// deliberately NOT paired here: a DID farm never trips it, and busy + /// legitimate agents would false-positive. `GITLAWB_WRITE_RATE_LIMIT` + /// overrides the default, 0 disables. Keyed by `push_limiter_trust`. + pub write_rate_limiter: RateLimiter, /// Per-client-IP rate limiter for git-receive-pack. Per-DID limits cannot /// brake a push flood from a DID farm (one throwaway DID per repo), so the /// push path throttles on the resolved client IP instead. diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 4fe52f64..687b5f90 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -77,6 +77,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { repo_store: crate::git::repo_store::RepoStore::for_testing(PathBuf::from("/tmp"), pool), rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), + write_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), From 7d680dc9c27fc10223da006e71a00fec12ecfcd1 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 14:11:38 -0500 Subject: [PATCH 02/61] feat(node): extend the per-IP write brake to the remaining write sinks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Attaches rate_limit_by_ip (write bucket) to task_write_routes, issue_write_routes, bounty_write_routes, profile_write_routes, and the /graphql POST/GET surface (KTD-5 — an HTTP-layer brake counts every /graphql request; /graphql/ws subscriptions stay unbraked). write_ip_limiter is now built once and shared. Every non-git, non-creation authenticated write group is braked. Tests (execution-verified against Postgres): /graphql POST is 429'd, and a representative REST write sink (issue comment) is 429'd, by the write brake. --- crates/gitlawb-node/src/api/repos.rs | 69 ++++++++++++++++++++++++++++ crates/gitlawb-node/src/server.rs | 41 ++++++++++++----- 2 files changed, 99 insertions(+), 11 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 7bb3c000..a9cfcc7e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2701,4 +2701,73 @@ mod tests { "an exhausted write bucket must not throttle repo creation (separate buckets)" ); } + + // KTD-5: /graphql POST (the MutationRoot surface) draws from the write bucket. + #[sqlx::test] + async fn graphql_post_is_rate_limited_by_ip(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.write_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.111:7000".parse().unwrap(); + assert!(state.write_rate_limiter.check(&peer.ip().to_string()).await); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/graphql") + .header("content-type", "application/json") + .body(Body::from(r#"{"query":"{ __typename }"}"#)) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let status = router.oneshot(req).await.unwrap().status(); + assert_eq!( + status, + StatusCode::TOO_MANY_REQUESTS, + "/graphql must be IP-throttled by the write brake" + ); + } + + // Representative REST write group (issue comment) — same attachment as the + // task/bounty/profile groups. + #[sqlx::test] + async fn issue_comment_is_rate_limited_by_ip(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.write_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.122:7000".parse().unwrap(); + assert!(state.write_rate_limiter.check(&peer.ip().to_string()).await); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/api/v1/repos/someowner/somerepo/issues/1/comments") + .header("content-type", "application/json") + .body(Body::from(r#"{"body":"flood"}"#)) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let status = router.oneshot(req).await.unwrap().status(); + assert_eq!( + status, + StatusCode::TOO_MANY_REQUESTS, + "issue-write routes must be IP-throttled by the write brake" + ); + } } diff --git a/crates/gitlawb-node/src/server.rs b/crates/gitlawb-node/src/server.rs index 5df9ca0d..0c24d953 100644 --- a/crates/gitlawb-node/src/server.rs +++ b/crates/gitlawb-node/src/server.rs @@ -58,15 +58,30 @@ fn add_auth_layers(router: Router, state: AppState) -> Router Router { // ── GraphQL routes ───────────────────────────────────────────────────── let schema = state.graphql_schema.as_ref().clone(); + + // Per-IP write brake, shared across every authenticated non-creation write + // group (see `AppState::write_rate_limiter`). Built once here so each group + // clones the same bucket; attached outermost on each group so a flood is + // rejected before auth runs. Separate bucket from the creation brake. + let write_ip_limiter = rate_limit::IpRateLimiter { + limiter: state.write_rate_limiter.clone(), + trust: state.push_limiter_trust, + }; + let graphql_routes = Router::new() .route("/graphql", get(graphql_playground).post(graphql_handler)) // Attach the verified DID to /graphql when a signature is present. The // layer covers only routes added before it, so /graphql/ws (added after, - // read-only subscriptions) stays open. + // read-only subscriptions) stays open. The per-IP write brake covers the + // same /graphql (KTD-5: an HTTP-layer brake cannot tell a query POST from + // a mutation POST, so it counts every /graphql request — acceptable, both + // cost work); /graphql/ws subscriptions stay unbraked. .layer(middleware::from_fn(auth::optional_signature)) + .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) + .layer(axum::Extension(write_ip_limiter.clone())) .route_service("/graphql/ws", GraphQLSubscription::new(schema)); - // ── Task routes (write — require HTTP Signature) ─────────────────────── + // ── Task routes (write — require HTTP Signature + per-IP write brake) ── let task_write_routes = add_auth_layers( Router::new() .route("/api/v1/tasks", post(tasks::create_task)) @@ -74,7 +89,9 @@ pub fn build_router(state: AppState) -> Router { .route("/api/v1/tasks/{id}/complete", post(tasks::complete_task)) .route("/api/v1/tasks/{id}/fail", post(tasks::fail_task)), state.clone(), - ); + ) + .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) + .layer(axum::Extension(write_ip_limiter.clone())); // ── Task routes (read — open) ────────────────────────────────────────── let task_read_routes = Router::new() @@ -118,10 +135,6 @@ pub fn build_router(state: AppState) -> Router { // positive). The bundled visibility GET is a read that also draws from the // write bucket — harmless, since the bucket is sized far above any legit // per-IP read rate. - let write_ip_limiter = rate_limit::IpRateLimiter { - limiter: state.write_rate_limiter.clone(), - trust: state.push_limiter_trust, - }; let write_routes = add_auth_layers( Router::new() .route( @@ -259,7 +272,9 @@ pub fn build_router(state: AppState) -> Router { post(bounties::dispute_bounty), ), state.clone(), - ); + ) + .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) + .layer(axum::Extension(write_ip_limiter.clone())); // ── Bounty routes (read — open) ────────────────────────────────────── let bounty_read_routes = Router::new() @@ -280,9 +295,11 @@ pub fn build_router(state: AppState) -> Router { let profile_write_routes = add_auth_layers( Router::new().route("/api/v1/profile", axum::routing::put(profiles::set_profile)), state.clone(), - ); + ) + .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) + .layer(axum::Extension(write_ip_limiter.clone())); - // ── Issue routes (write — require HTTP Signature, no rate limit) ───── + // ── Issue routes (write — require HTTP Signature + per-IP write brake) ─ let issue_write_routes = add_auth_layers( Router::new() .route( @@ -294,7 +311,9 @@ pub fn build_router(state: AppState) -> Router { post(issues::create_issue_comment), ), state.clone(), - ); + ) + .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) + .layer(axum::Extension(write_ip_limiter.clone())); // ── Peer discovery routes ───────────────────────────────────────────── // Peer writes accept signatures when present and can require them after a From 665b4ffcb58fe73a4e70ec02b59c77b0b2d9c3fd Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 14:14:15 -0500 Subject: [PATCH 03/61] test(node): adversarial TrustedProxy verification through the write brake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Middleware-level tests complementing the client_key unit coverage: a spoofer varying the client-controlled leftmost X-Forwarded-For hop cannot rotate its bucket key (keyed on the trusted rightmost hop); distinct trusted client hops get distinct buckets (no cross-starvation); and a request with no trusted header in a proxy mode falls back to the socket-peer key — the documented collapse, asserted by name so any change is deliberate. Operator-warning on missing-header deferred deliberately: a per-request warning is a log-noise risk and the security-load-bearing fallback (peer key, never skip) is what these tests lock down. --- crates/gitlawb-node/src/rate_limit.rs | 100 ++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index d40e2691..4c41575c 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -605,4 +605,104 @@ mod tests { "an exhausted IP must be braked with 429 before auth runs, not leak to 401" ); } + + // ── Adversarial TrustedProxy verification through the middleware (U5) ── + // These complement the client_key unit tests above by driving hostile + // headers through rate_limit_by_ip on a real router: the property under test + // is that the *bucket key* cannot be rotated by a spoofer. + + async fn post_with( + router: &axum::Router, + peer: SocketAddr, + hdrs: &[(&str, &str)], + ) -> StatusCode { + use tower::ServiceExt; + let mut b = axum::http::Request::builder() + .method(axum::http::Method::POST) + .uri("/api/v1/repos"); + for (k, v) in hdrs { + b = b.header(*k, *v); + } + let mut req = b.body(axum::body::Body::empty()).unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + router.clone().oneshot(req).await.unwrap().status() + } + + #[tokio::test] + async fn xff_spoofed_leftmost_hop_cannot_rotate_bucket() { + // XForwardedFor mode, budget 1. A spoofer varies the client-controlled + // leftmost hop every request but the trusted proxy always appends the + // same rightmost hop. All requests must key to the rightmost hop, so the + // second is braked despite a fresh leftmost value. + let router = ip_limited_over_auth_router(IpRateLimiter { + limiter: RateLimiter::new(1, Duration::from_secs(60)), + trust: TrustedProxy::XForwardedFor, + }); + let p1: SocketAddr = "10.0.0.1:1".parse().unwrap(); + let p2: SocketAddr = "10.0.0.2:1".parse().unwrap(); + // First request passes the brake, reaches (failing) auth. + assert_eq!( + post_with(&router, p1, &[("x-forwarded-for", "9.9.9.1, 203.0.113.5")]).await, + StatusCode::UNAUTHORIZED + ); + // Different leftmost + different socket peer, SAME rightmost trusted hop: + // braked, because the key is the rightmost hop, not the spoofed value. + assert_eq!( + post_with(&router, p2, &[("x-forwarded-for", "9.9.9.2, 203.0.113.5")]).await, + StatusCode::TOO_MANY_REQUESTS, + "a spoofer varying the leftmost XFF hop must not rotate its bucket key" + ); + } + + #[tokio::test] + async fn xff_distinct_real_clients_get_distinct_buckets() { + // Distinct trusted (rightmost) hops are distinct clients: neither throttles + // the other, so a busy legitimate client cannot starve a different one. + let router = ip_limited_over_auth_router(IpRateLimiter { + limiter: RateLimiter::new(1, Duration::from_secs(60)), + trust: TrustedProxy::XForwardedFor, + }); + let peer: SocketAddr = "10.0.0.9:1".parse().unwrap(); + assert_eq!( + post_with( + &router, + peer, + &[("x-forwarded-for", "1.1.1.1, 203.0.113.5")] + ) + .await, + StatusCode::UNAUTHORIZED + ); + assert_eq!( + post_with( + &router, + peer, + &[("x-forwarded-for", "1.1.1.1, 203.0.113.6")] + ) + .await, + StatusCode::UNAUTHORIZED, + "a different trusted client hop must get its own bucket" + ); + } + + #[tokio::test] + async fn absent_trusted_header_collapses_to_socket_peer() { + // The documented fallback: in a trusted-proxy mode, a request with no + // forwarded header keys on the socket peer. Behind a real edge every + // request shares the proxy's socket, so they collapse onto one bucket — + // asserted here by name so any change to this behavior is deliberate. + let router = ip_limited_over_auth_router(IpRateLimiter { + limiter: RateLimiter::new(1, Duration::from_secs(60)), + trust: TrustedProxy::XForwardedFor, + }); + let proxy: SocketAddr = "172.16.0.1:1".parse().unwrap(); + assert_eq!( + post_with(&router, proxy, &[]).await, + StatusCode::UNAUTHORIZED + ); + assert_eq!( + post_with(&router, proxy, &[]).await, + StatusCode::TOO_MANY_REQUESTS, + "with no trusted header, requests fall back to the socket peer key (collapse)" + ); + } } From 69a98cf43186ae7813fe704bd47fa89b8dd4f621 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 14:28:45 -0500 Subject: [PATCH 04/61] feat(node): purge-spam admin subcommand for the empty-burst cleanup Adds a purge-spam subcommand (clap #[command(subcommand)] split; no-subcommand path still runs the node unchanged). Candidate selection is signature-based, not per-DID: a repo qualifies only if owned by the named burst DID AND verified empty (zero git refs) per repo. A hard exclusion gate runs BEFORE the empty check, so an empty repo owned by the content-bearing user or the mirror-bot DID is never a candidate. Dry-run is the default; deletion needs an explicit --execute flag and operates per-repo, never 'all repos of DID X'. Tests (RED->GREEN, exclusion gate confirmed load-bearing by targeting an excluded DID directly): empty target listed; target-with-refs spared; empty excluded/ intern repos spared (must-assert); dry-run deletes nothing; execute removes only the empty target on disk. --- crates/gitlawb-node/src/admin.rs | 469 ++++++++++++++++++++++++++++++ crates/gitlawb-node/src/config.rs | 21 +- crates/gitlawb-node/src/db/mod.rs | 29 ++ crates/gitlawb-node/src/main.rs | 25 ++ 4 files changed, 543 insertions(+), 1 deletion(-) create mode 100644 crates/gitlawb-node/src/admin.rs diff --git a/crates/gitlawb-node/src/admin.rs b/crates/gitlawb-node/src/admin.rs new file mode 100644 index 00000000..3e6ba3cb --- /dev/null +++ b/crates/gitlawb-node/src/admin.rs @@ -0,0 +1,469 @@ +//! Admin subcommands invoked out-of-band, not part of the running node. +//! +//! `purge-spam` produces a reviewable dry-run list of empty spam-burst repos and, +//! only behind an explicit `--execute` flag, deletes them one at a time. The +//! selection logic is the load-bearing security part: a repo qualifies ONLY if it +//! is owned by the named burst DID AND is verified empty (zero git refs) PER REPO, +//! and a hard exclusion gate (evaluated BEFORE the empty check) keeps +//! content-bearing and intern/mirror-bot DIDs out no matter what. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use tracing::{info, warn}; + +use crate::db::{Db, RepoRecord}; +use crate::git::store; + +/// The did:key of the spam burst this tool targets. The purge is scoped to +/// exactly this owner; an empty repo owned by anyone else is never a candidate. +pub const SPAM_BURST_TARGET_DID: &str = "did:key:z6Mkopj6mhcMayipekXbTRFMZPM6Bsgy4FQZuN9fannXSLTC"; + +/// DIDs that must never be touched, even when they own an empty repo whose +/// signature otherwise matches the burst. The gate is evaluated BEFORE the empty +/// check so it wins unconditionally: +/// - `z6Mkk4L…` is a content-bearing live user. +/// - `z6MkqRz…` is the intern / mirror-bot DID. +pub const EXCLUDED_DIDS: &[&str] = &[ + "did:key:z6Mkk4LDvfA8VQmdehbJDvxp133sdtXUhR2UkUnMPguX7gnP", + "did:key:z6MkqRzACJ5iCDdkiymAPK3gq18z2iecZHeAuUyW6JnwRfoM", +]; + +/// A repo selected for purge, with the evidence that qualified it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Candidate { + pub id: String, + pub owner_did: String, + pub name: String, + /// Number of git refs found on disk. Always 0 for a selected candidate — kept + /// as explicit evidence in the dry-run output rather than an implicit "empty". + pub ref_count: usize, +} + +/// Whether a DID is on the hard exclusion list. +fn is_excluded(owner_did: &str) -> bool { + EXCLUDED_DIDS.contains(&owner_did) +} + +/// Pure candidate selector — the security core, isolated from disk and DB so the +/// exclusion + empty logic is directly testable. +/// +/// `repos` is the raw row set to consider (the caller supplies the target DID's +/// rows). `ref_count_of` returns the number of git refs for a given repo; the CLI +/// wires the real on-disk ref source, tests inject precomputed counts. +/// +/// A repo qualifies ONLY if, PER REPO: +/// 1. its owner is NOT on the exclusion list (gate evaluated FIRST), AND +/// 2. its owner is exactly the target burst DID, AND +/// 3. it is verified empty — `ref_count_of` returns 0. +/// +/// The exclusion gate is checked before the empty check so that an empty repo +/// owned by an excluded DID is dropped regardless of its ref signature. +pub fn select_spam_candidates( + repos: &[RepoRecord], + target_did: &str, + mut ref_count_of: F, +) -> Vec +where + F: FnMut(&RepoRecord) -> usize, +{ + let mut out = Vec::new(); + for repo in repos { + // Hard exclusion gate FIRST — wins over the empty signature. + if is_excluded(&repo.owner_did) { + continue; + } + // Scope to the named burst only. + if repo.owner_did != target_did { + continue; + } + // Per-repo empty check. + let ref_count = ref_count_of(repo); + if ref_count != 0 { + continue; + } + out.push(Candidate { + id: repo.id.clone(), + owner_did: repo.owner_did.clone(), + name: repo.name.clone(), + ref_count, + }); + } + out +} + +/// Count the git refs of a repo on disk. Zero refs means the repo is empty. +/// +/// Resolves the repo's bare path from `repos_dir` + owner/name (the same layout +/// `git::store::repo_disk_path` writes) and shells to `git for-each-ref` via +/// `store::list_refs`. A repo whose on-disk path is missing or unreadable is +/// treated as having an unknown, non-empty ref count so it is NOT selected — the +/// tool fails closed and never deletes on a read error. +fn on_disk_ref_count(repos_dir: &Path, repo: &RepoRecord) -> usize { + let path = store::repo_disk_path(repos_dir, &repo.owner_did, &repo.name); + match store::list_refs(&path) { + Ok(refs) => refs.len(), + Err(e) => { + // Fail closed: an unreadable repo is not provably empty, so keep it + // out of the candidate set (report it as one ref so it's excluded). + warn!(repo = %repo.id, path = %path.display(), err = %e, + "purge-spam: could not read refs — treating as non-empty (skipped)"); + 1 + } + } +} + +/// Run the `purge-spam` admin subcommand. +/// +/// Enumerates the target burst DID's repos, verifies each is empty on disk, +/// applies the exclusion gate, prints one dry-run row per candidate with owner + +/// ref-count evidence, and — only when `execute` is true — deletes the DB row of +/// each candidate one at a time. Dry-run (the default) deletes nothing. +pub async fn run_purge_spam(db: &Db, repos_dir: &Path, execute: bool) -> Result<()> { + let repos_dir: PathBuf = repos_dir.to_path_buf(); + let rows = db + .list_repos_by_owner_did(SPAM_BURST_TARGET_DID) + .await + .context("listing repos for the spam-burst target DID")?; + + let candidates = select_spam_candidates(&rows, SPAM_BURST_TARGET_DID, |repo| { + on_disk_ref_count(&repos_dir, repo) + }); + + info!( + target = SPAM_BURST_TARGET_DID, + scanned = rows.len(), + candidates = candidates.len(), + execute, + "purge-spam: candidate selection complete" + ); + + if candidates.is_empty() { + println!("purge-spam: no empty spam-burst repos found for {SPAM_BURST_TARGET_DID}"); + return Ok(()); + } + + println!( + "purge-spam: {} candidate(s) for {} ({} mode)", + candidates.len(), + SPAM_BURST_TARGET_DID, + if execute { "EXECUTE" } else { "dry-run" } + ); + for c in &candidates { + println!( + " {} owner={} name={} refs={}", + c.id, c.owner_did, c.name, c.ref_count + ); + } + + if !execute { + println!( + "purge-spam: dry-run — nothing deleted. Re-run with --execute to delete the {} candidate(s).", + candidates.len() + ); + return Ok(()); + } + + // Execute: delete per-repo, never a single blanket "delete all of owner X". + let mut deleted = 0u64; + for c in &candidates { + match db.delete_repo_by_id(&c.id).await { + Ok(n) => { + deleted += n; + info!(repo = %c.id, rows = n, "purge-spam: deleted repo row"); + } + Err(e) => { + warn!(repo = %c.id, err = %e, "purge-spam: failed to delete repo row"); + } + } + } + println!("purge-spam: deleted {deleted} repo row(s)."); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + + const TARGET: &str = SPAM_BURST_TARGET_DID; + const EXCLUDED_CONTENT: &str = EXCLUDED_DIDS[0]; // z6Mkk4L… content-bearing live user + const EXCLUDED_INTERN: &str = EXCLUDED_DIDS[1]; // z6MkqRz… intern/mirror-bot + const UNRELATED: &str = "did:key:z6MkUnrelatedStrangerDidThatIsNotTheBurst"; + + fn repo(id: &str, owner: &str, name: &str) -> RepoRecord { + RepoRecord { + id: id.to_string(), + name: name.to_string(), + owner_did: owner.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: Utc::now(), + updated_at: Utc::now(), + disk_path: format!("/srv/{id}.git"), + forked_from: None, + machine_id: None, + } + } + + /// Ref counts keyed by repo id; anything absent defaults to 0 (empty). + fn refs_by_id<'a>(map: &'a [(&'a str, usize)]) -> impl Fn(&RepoRecord) -> usize + 'a { + move |r: &RepoRecord| { + map.iter() + .find(|(id, _)| *id == r.id) + .map(|(_, n)| *n) + .unwrap_or(0) + } + } + + // Test 1: an empty repo owned by the target DID is a candidate. + #[test] + fn empty_target_repo_is_a_candidate() { + let repos = vec![repo("t-empty", TARGET, "spam1")]; + let got = select_spam_candidates(&repos, TARGET, refs_by_id(&[])); + assert_eq!(got.len(), 1, "empty target repo must be selected"); + assert_eq!(got[0].id, "t-empty"); + assert_eq!(got[0].owner_did, TARGET); + assert_eq!( + got[0].ref_count, 0, + "candidate must carry ref-count evidence" + ); + } + + // Test 2: a target-owned repo WITH refs is absent (per-repo empty check, not + // per-DID). + #[test] + fn target_repo_with_refs_is_absent() { + let repos = vec![ + repo("t-empty", TARGET, "spam1"), + repo("t-nonempty", TARGET, "real"), + ]; + let got = select_spam_candidates(&repos, TARGET, refs_by_id(&[("t-nonempty", 3)])); + let ids: Vec<&str> = got.iter().map(|c| c.id.as_str()).collect(); + assert!(ids.contains(&"t-empty"), "empty target repo still selected"); + assert!( + !ids.contains(&"t-nonempty"), + "a target repo WITH refs must NOT be selected (per-repo, not per-DID)" + ); + } + + // Test 3 (MUST ASSERT): an EMPTY repo owned by the excluded content DID is + // absent — the exclusion gate wins over the empty signature. + // + // Driven with the excluded DID passed AS the target so the exclusion gate is + // the ONLY barrier: with the gate removed this repo would match owner==target + // and be selected, so the test goes RED. This is what makes the gate + // load-bearing rather than shadowed by the target-scope check. + #[test] + fn empty_excluded_content_repo_is_absent() { + let repos = vec![repo("x-content", EXCLUDED_CONTENT, "anything")]; + let got = select_spam_candidates(&repos, EXCLUDED_CONTENT, refs_by_id(&[])); + assert!( + got.is_empty(), + "an empty repo owned by the excluded content DID must be excluded even \ + if that DID were the target, got {got:?}" + ); + // And of course it is also absent when the real burst DID is the target. + let got = select_spam_candidates(&repos, TARGET, refs_by_id(&[])); + assert!(got.is_empty()); + } + + // Test 4 (MUST ASSERT): an EMPTY repo owned by the intern DID is absent. + // Same construction as test 3: the intern DID is passed as the target so the + // exclusion gate is the sole reason it is dropped (RED without the gate). + #[test] + fn empty_intern_repo_is_absent() { + let repos = vec![repo("x-intern", EXCLUDED_INTERN, "mirror")]; + let got = select_spam_candidates(&repos, EXCLUDED_INTERN, refs_by_id(&[])); + assert!( + got.is_empty(), + "an empty repo owned by the intern/mirror-bot DID must be excluded even \ + if that DID were the target, got {got:?}" + ); + let got = select_spam_candidates(&repos, TARGET, refs_by_id(&[])); + assert!(got.is_empty()); + } + + // Test 5: an empty repo owned by an unrelated DID is absent — the tool targets + // the named burst only. + #[test] + fn empty_unrelated_repo_is_absent() { + let repos = vec![repo("u-empty", UNRELATED, "whatever")]; + let got = select_spam_candidates(&repos, TARGET, refs_by_id(&[])); + assert!( + got.is_empty(), + "an empty repo owned by a non-target DID must not be selected, got {got:?}" + ); + } + + // The full matrix in one selector pass: only the empty target repo survives. + #[test] + fn full_matrix_selects_only_empty_target() { + let repos = vec![ + repo("t-empty", TARGET, "spam1"), // selected + repo("t-nonempty", TARGET, "real"), // has refs → out + repo("x-content", EXCLUDED_CONTENT, "a"), // excluded, empty → out + repo("x-intern", EXCLUDED_INTERN, "b"), // excluded, empty → out + repo("u-empty", UNRELATED, "c"), // wrong owner → out + ]; + let got = select_spam_candidates(&repos, TARGET, refs_by_id(&[("t-nonempty", 2)])); + assert_eq!( + got.iter().map(|c| c.id.as_str()).collect::>(), + vec!["t-empty"], + "only the empty target repo may survive the full matrix" + ); + } + + // An excluded DID that is empty is still out even when that excluded DID is + // itself the target — pins that the exclusion gate runs BEFORE the owner/empty + // checks and is the sole barrier here (RED without the gate). + #[test] + fn exclusion_gate_precedes_empty_check() { + let repos = vec![repo("collision", EXCLUDED_CONTENT, "spam1")]; + let got = select_spam_candidates(&repos, EXCLUDED_CONTENT, refs_by_id(&[])); + assert!(got.is_empty(), "exclusion must win even on an empty repo"); + } +} + +#[cfg(test)] +mod db_tests { + use super::*; + use crate::db::{Db, RepoRecord}; + use chrono::Utc; + use sqlx::PgPool; + + async fn db(pool: PgPool) -> Db { + let db = Db::for_testing(pool); + db.run_migrations().await.unwrap(); + db + } + + fn rec(id: &str, owner: &str, name: &str) -> RepoRecord { + RepoRecord { + id: id.to_string(), + name: name.to_string(), + owner_did: owner.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: Utc::now(), + updated_at: Utc::now(), + disk_path: format!("/srv/{id}.git"), + forked_from: None, + machine_id: None, + } + } + + async fn count_rows(db: &Db) -> i64 { + sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM repos") + .fetch_one(db.pool()) + .await + .unwrap() + } + + // Test 6: a dry-run over the full matrix deletes nothing. The empty check is + // driven off a repos_dir with NO repos on disk, so every row reads as empty + // (list_refs on a missing repo returns Err → treated as non-empty and skipped), + // which is fine here: the assertion is that dry-run mutates no rows regardless. + #[sqlx::test] + async fn dry_run_deletes_nothing(pool: PgPool) { + let db = db(pool).await; + for r in [ + rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"), + rec("t-nonempty", SPAM_BURST_TARGET_DID, "real"), + rec("x-content", EXCLUDED_DIDS[0], "a"), + rec("x-intern", EXCLUDED_DIDS[1], "b"), + rec("u-empty", "did:key:z6MkUnrelated", "c"), + ] { + db.create_repo(&r).await.unwrap(); + } + let before = count_rows(&db).await; + assert_eq!(before, 5); + + // Empty repos dir: no repo exists on disk, so nothing is provably empty. + let tmp = tempfile::TempDir::new().unwrap(); + run_purge_spam(&db, tmp.path(), false).await.unwrap(); + + let after = count_rows(&db).await; + assert_eq!(after, before, "dry-run must not delete any repo rows"); + } + + // The DB accessor lists exactly the target DID's rows (exact owner match), and + // delete_repo_by_id removes exactly one row, so the execute path deletes per + // repo. This exercises the DB wiring end-to-end with a real empty repo on disk. + #[sqlx::test] + async fn execute_deletes_only_the_empty_target_repo_on_disk(pool: PgPool) { + let db = db(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + + // One empty target repo (real bare repo, zero refs) and one target repo + // with a ref, plus an excluded-owner empty repo. + let empty = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + let nonempty = rec("t-refs", SPAM_BURST_TARGET_DID, "real"); + let excluded = rec("x-content", EXCLUDED_DIDS[0], "keep"); + for r in [&empty, &nonempty, &excluded] { + db.create_repo(r).await.unwrap(); + } + + // Materialize the two target repos on disk. + let empty_path = store::repo_disk_path(tmp.path(), &empty.owner_did, &empty.name); + store::init_bare(&empty_path).unwrap(); + let refs_path = store::repo_disk_path(tmp.path(), &nonempty.owner_did, &nonempty.name); + store::init_bare(&refs_path).unwrap(); + // Give the non-empty repo an actual ref. + seed_one_ref(&refs_path); + + // Sanity: our on-disk ref reader sees the expected counts. + assert_eq!(store::list_refs(&empty_path).unwrap().len(), 0); + assert!(!store::list_refs(&refs_path).unwrap().is_empty()); + + run_purge_spam(&db, tmp.path(), true).await.unwrap(); + + // Only the empty target repo row is gone. + assert!(db + .get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none()); + assert!(db + .get_repo(SPAM_BURST_TARGET_DID, "real") + .await + .unwrap() + .is_some()); + assert!(db + .get_repo(EXCLUDED_DIDS[0], "keep") + .await + .unwrap() + .is_some()); + } + + /// Create a single commit + ref in a bare repo via a throwaway worktree, so the + /// repo reads as non-empty (≥1 ref). + fn seed_one_ref(bare: &Path) { + use std::process::Command; + let wt = bare.join("_seed"); + let run = |args: &[&str], dir: &Path| { + let ok = Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(); + assert!(ok, "git {args:?} failed"); + }; + run( + &["worktree", "add", "--orphan", "-b", "main", "_seed"], + bare, + ); + std::fs::write(wt.join("f.txt"), b"x").unwrap(); + run(&["config", "user.email", "t@t"], &wt); + run(&["config", "user.name", "t"], &wt); + run(&["add", "."], &wt); + run(&["commit", "-qm", "seed"], &wt); + let _ = Command::new("git") + .args(["worktree", "remove", "--force", "_seed"]) + .current_dir(bare) + .status(); + } +} diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fc2247d9..38f31e6e 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -1,9 +1,28 @@ -use clap::Parser; +use clap::{Parser, Subcommand}; use std::path::PathBuf; +/// Optional admin subcommands. When none is given, the binary runs the node +/// daemon as before — the default (no-subcommand) startup path is unchanged. +#[derive(Subcommand, Debug, Clone)] +pub enum Command { + /// Dry-run (default) or, with --execute, delete empty spam-burst repos owned + /// by the known burst DID. Never touches the hard-excluded DIDs, and verifies + /// each repo is empty (zero git refs) per repo before selecting it. + PurgeSpam { + /// Actually delete the candidates. Omit for a dry-run that prints the + /// candidate list and deletes nothing. + #[arg(long, default_value_t = false)] + execute: bool, + }, +} + #[derive(Parser, Debug, Clone)] #[command(name = "gitlawb-node", about = "gitlawb node daemon", version)] pub struct Config { + /// Admin subcommand to run instead of the node daemon. Absent = run the node. + #[command(subcommand)] + pub command: Option, + /// Directory where bare git repositories are stored #[arg(long, env = "GITLAWB_REPOS_DIR", default_value = "./data/repos")] pub repos_dir: PathBuf, diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 5324a4b9..46b5be71 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1259,6 +1259,35 @@ impl Db { .await?; Ok(()) } + + /// Every repo row owned by exactly `owner_did` (an exact `owner_did` match, no + /// did:key normalization and no dedup) so a caller enumerating one DID's repos + /// sees every physical row. Backs the `purge-spam` admin tool, whose selection + /// then applies its own per-repo empty check and exclusion gate. Ordered by + /// `id` for a stable candidate listing. + pub async fn list_repos_by_owner_did(&self, owner_did: &str) -> Result> { + let rows = sqlx::query( + "SELECT id, name, owner_did, description, is_public, default_branch, + created_at, updated_at, disk_path, forked_from, machine_id + FROM repos WHERE owner_did = $1 ORDER BY id", + ) + .bind(owner_did) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter().map(row_to_repo).collect()) + } + + /// Delete a single repo row by its primary key `id`. Returns the number of + /// rows removed (0 if no such repo). Operates on one repo at a time by design: + /// the `purge-spam` tool deletes a vetted candidate list per-repo, never a + /// blanket "delete all repos of owner X". + pub async fn delete_repo_by_id(&self, id: &str) -> Result { + let result = sqlx::query("DELETE FROM repos WHERE id = $1") + .bind(id) + .execute(&self.pool) + .await?; + Ok(result.rows_affected()) + } } // ── Agents / Trust ──────────────────────────────────────────────────────────── diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 4fd6e922..88984469 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -1,3 +1,4 @@ +mod admin; mod api; mod arweave; mod auth; @@ -70,6 +71,12 @@ async fn main() -> Result<()> { let mut config = Config::parse(); + // Admin subcommands run out-of-band and exit, never starting the node. The + // no-subcommand path below is the unchanged daemon startup. + if let Some(command) = config.command.clone() { + return run_admin_command(command, &config).await; + } + // Merge the embedded seed list of public network nodes into the runtime // bootstrap peers. Operators can opt out via GITLAWB_BOOTSTRAP_DISABLE_SEEDS. bootstrap::merge_seeds(&mut config); @@ -586,6 +593,24 @@ async fn main() -> Result<()> { Ok(()) } +/// Dispatch an admin subcommand. Connects the database directly (no server, no +/// p2p, no metrics) and runs the requested tool to completion, then exits. +async fn run_admin_command(command: config::Command, config: &Config) -> Result<()> { + match command { + config::Command::PurgeSpam { execute } => { + let acquire_timeout = std::time::Duration::from_secs(config.db_acquire_timeout_secs); + let db = Db::connect( + &config.database_url, + config.db_max_connections, + acquire_timeout, + ) + .await + .context("connecting to database for purge-spam")?; + admin::run_purge_spam(&db, &config.repos_dir, execute).await + } + } +} + fn spawn_shutdown_signal(tx: watch::Sender) { tokio::spawn(async move { #[cfg(unix)] From 13e59b4e40d205142b3ec8d03f4dc2c8566f3381 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 14:47:07 -0500 Subject: [PATCH 05/61] fix(review): make purge-spam empty-check reject git-discovery reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1 from code review: on_disk_ref_count shelled 'git for-each-ref' with only the repo path as cwd and no --git-dir, so a path that exists but is not a bare repo let git discover a parent .git (repos_dir may sit inside the operator's checkout) and read a DIFFERENT repo's refs — possibly 0, deleting a real repo. Now require the bare-repo markers (HEAD file + objects/ dir) before trusting any count; anything else fails closed (skipped). Tests: a non-git dir under a git ancestor (zero-ref) now fails closed — verified RED without the guard (read the ancestor's 0 refs); target-DID-never-excluded invariant pinned; write_flood test anchored to a genuinely-drained bucket so its assert_ne can't pass vacuously. --- crates/gitlawb-node/src/admin.rs | 60 ++++++++++++++++++++++++++++ crates/gitlawb-node/src/api/repos.rs | 16 ++++++++ 2 files changed, 76 insertions(+) diff --git a/crates/gitlawb-node/src/admin.rs b/crates/gitlawb-node/src/admin.rs index 3e6ba3cb..202b4a51 100644 --- a/crates/gitlawb-node/src/admin.rs +++ b/crates/gitlawb-node/src/admin.rs @@ -99,8 +99,24 @@ where /// `store::list_refs`. A repo whose on-disk path is missing or unreadable is /// treated as having an unknown, non-empty ref count so it is NOT selected — the /// tool fails closed and never deletes on a read error. +/// +/// Critically, a `0` count must come from THIS exact bare repo and never from +/// git's upward repository discovery. `git for-each-ref` runs with the repo path +/// as its cwd and no explicit `--git-dir`, so if the path exists but is not +/// itself a git dir, git walks parent directories for a `.git` — and `repos_dir` +/// may live inside the operator's own git checkout. That would read a DIFFERENT +/// repo's refs (possibly `0`) and delete a real repo. We defend by requiring the +/// bare-repo markers (`HEAD` file + `objects/` dir) before trusting any count; +/// anything else fails closed (treated non-empty, skipped). fn on_disk_ref_count(repos_dir: &Path, repo: &RepoRecord) -> usize { let path = store::repo_disk_path(repos_dir, &repo.owner_did, &repo.name); + if !path.join("HEAD").is_file() || !path.join("objects").is_dir() { + // Not a bare git repo at the exact expected path. Do NOT trust a ref + // count that git discovery could have read from a parent repository. + warn!(repo = %repo.id, path = %path.display(), + "purge-spam: path is not a bare git repo — treating as non-empty (skipped)"); + return 1; + } match store::list_refs(&path) { Ok(refs) => refs.len(), Err(e) => { @@ -466,4 +482,48 @@ mod db_tests { .current_dir(bare) .status(); } + + /// A `0` ref count must come only from a real bare repo at the exact path, + /// never from git discovery walking up to a parent `.git`. Load-bearing: the + /// tempdir root is itself a git repo (zero refs), so a naive `for-each-ref` + /// run from a child non-git dir would discover it and report 0 — the delete-a- + /// real-repo fail-open. The marker check must make that path fail closed. + #[test] + fn nongit_path_fails_closed_even_under_a_git_ancestor() { + let tmp = tempfile::TempDir::new().unwrap(); + // Make the tempdir root a git repo with zero refs (the discovery trap). + std::process::Command::new("git") + .args(["init", "-q"]) + .current_dir(tmp.path()) + .status() + .unwrap(); + + let repo = rec("t-x", SPAM_BURST_TARGET_DID, "spam1"); + let path = store::repo_disk_path(tmp.path(), &repo.owner_did, &repo.name); + + // (a) Path exists as a plain (non-git) directory under the git ancestor. + // Without the marker guard, git discovery reads the ancestor's 0 refs and + // this repo would be deleted. It must fail closed (>=1, skipped). + std::fs::create_dir_all(&path).unwrap(); + assert_eq!( + on_disk_ref_count(tmp.path(), &repo), + 1, + "a non-git dir under a git ancestor must fail closed, not read the ancestor's refs" + ); + + // (b) A real empty bare repo at the same path reads 0 — a genuine candidate. + std::fs::remove_dir_all(&path).unwrap(); + store::init_bare(&path).unwrap(); + assert_eq!(on_disk_ref_count(tmp.path(), &repo), 0); + } + + /// The exclusion gate and the target scope must never overlap: if the burst + /// target were ever set to an excluded DID, the gate would fail to protect it. + #[test] + fn target_did_is_never_excluded() { + assert!( + !EXCLUDED_DIDS.contains(&SPAM_BURST_TARGET_DID), + "the purge target must never be an excluded (protected) DID" + ); + } } diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index a9cfcc7e..6c1fd92e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2684,6 +2684,22 @@ mod tests { assert!(state.write_rate_limiter.check(&peer.ip().to_string()).await); let router = crate::server::build_router(state); + + // Anchor the test: prove the write bucket is genuinely drained at the + // router (a write sink from this peer 429s) so the creation assertion + // below cannot pass vacuously on some unrelated non-429 status. + let mut wreq = Request::builder() + .method(Method::PUT) + .uri("/api/v1/repos/someowner/somerepo/star") + .body(Body::empty()) + .unwrap(); + wreq.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.clone().oneshot(wreq).await.unwrap().status(), + StatusCode::TOO_MANY_REQUESTS, + "write bucket must be drained for this peer (test precondition)" + ); + // Creation from the same peer must NOT be 429 — its bucket is untouched. // (It fails later on missing signature; the point is it is not throttled.) let mut req = Request::builder() From 458fe50312fbf373baed5d87b534b024c9086261 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 15:03:41 -0500 Subject: [PATCH 06/61] fix(review): normalization-consistent purge, brake helper, CLI + coverage Addresses the remaining code-review findings: - purge-spam DID matching now uses did:key normalization on both sides (list_repos_by_owner_did via OWNER_KEY_CASE_SQL, is_excluded and the target scope via normalize_owner_key), so short/full form is consistent and an excluded identity is protected in either form. Adds a short-form exclusion test. - Extract the per-IP write brake into a chainable WriteBraked helper so a future write group can't silently ship unbraked (kept separate from add_auth_layers by design; documented why). - Mark --repos-dir / --database-url global so admin subcommands accept them after the subcommand (was an 'unexpected argument' parse error); verified. - Document GITLAWB_WRITE_RATE_LIMIT in .env.example incl. the per-IP NAT-aggregate caveat so operators can size it. - Coverage: adoption-floor (under-limit write passes), write-limit=0 disables, task/bounty/profile route-level 429s, /graphql/ws stays unbraked. Full suite 927 pass / 0 fail; fmt + clippy -D warnings clean. --- .env.example | 11 +++ crates/gitlawb-node/src/admin.rs | 43 ++++++++- crates/gitlawb-node/src/api/repos.rs | 136 +++++++++++++++++++++++++++ crates/gitlawb-node/src/config.rs | 16 +++- crates/gitlawb-node/src/db/mod.rs | 26 ++--- crates/gitlawb-node/src/server.rs | 36 ++++--- 6 files changed, 237 insertions(+), 31 deletions(-) diff --git a/.env.example b/.env.example index bbd9a342..5987a527 100644 --- a/.env.example +++ b/.env.example @@ -127,6 +127,17 @@ GITLAWB_PUSH_RATE_LIMIT=600 # the client IP. 0 disables. Default 120. GITLAWB_CREATE_RATE_LIMIT=120 +# ── Write rate limiting (non-creation authenticated writes) ─────────────── +# Max non-creation write requests per client IP per hour: issue/PR comments, +# labels, stars, merges, protect/unprotect, replicas, visibility, tasks, +# bounties, profile, and GraphQL mutations. Its own bucket, separate from the +# creation and push brakes. Uses GITLAWB_TRUSTED_PROXY to resolve the client IP. +# NOTE: this is a per-IP aggregate across ALL those write actions, so behind a +# shared NAT/egress IP (or with GITLAWB_TRUSTED_PROXY unset) many users collapse +# onto one bucket — raise this for automation-heavy or multi-user single-IP +# deployments. 0 disables. Default 600. +GITLAWB_WRITE_RATE_LIMIT=600 + # ── Peer-sync rate limiting (per client IP, uses GITLAWB_TRUSTED_PROXY below) ─ # /api/v1/peers/announce and /api/v1/sync/notify accept unsigned requests from # known peers and run at higher frequency, so a generous bucket. Separate from diff --git a/crates/gitlawb-node/src/admin.rs b/crates/gitlawb-node/src/admin.rs index 202b4a51..c428b765 100644 --- a/crates/gitlawb-node/src/admin.rs +++ b/crates/gitlawb-node/src/admin.rs @@ -40,9 +40,15 @@ pub struct Candidate { pub ref_count: usize, } -/// Whether a DID is on the hard exclusion list. +/// Whether a DID is on the hard exclusion list. Compared under did:key +/// normalization (the same convention the repos table and every ownership check +/// use), so an excluded identity stored in either `did:key:z6…` or bare `z6…` +/// form is protected regardless of the form the exclusion constant is written in. fn is_excluded(owner_did: &str) -> bool { - EXCLUDED_DIDS.contains(&owner_did) + let owner_key = crate::db::normalize_owner_key(owner_did); + EXCLUDED_DIDS + .iter() + .any(|d| crate::db::normalize_owner_key(d) == owner_key) } /// Pure candidate selector — the security core, isolated from disk and DB so the @@ -73,8 +79,11 @@ where if is_excluded(&repo.owner_did) { continue; } - // Scope to the named burst only. - if repo.owner_did != target_did { + // Scope to the named burst only, under did:key normalization so a burst + // row stored in either did:key or bare form is matched consistently. + if crate::db::normalize_owner_key(&repo.owner_did) + != crate::db::normalize_owner_key(target_did) + { continue; } // Per-repo empty check. @@ -526,4 +535,30 @@ mod db_tests { "the purge target must never be an excluded (protected) DID" ); } + + /// Exclusion is normalization-consistent: an excluded identity stored in the + /// bare short form (as mirror upserts write it) is still excluded, even though + /// the exclusion constants are full did:key form — and an empty repo it owns + /// is never selected. + #[test] + fn short_form_excluded_did_is_still_protected() { + let short = crate::db::normalize_owner_key(EXCLUDED_DIDS[1]); // bare z6MkqRz… + assert_ne!( + short, EXCLUDED_DIDS[1], + "fixture must actually be short form" + ); + assert!( + is_excluded(short), + "short-form of an excluded DID must be excluded" + ); + + // An empty repo owned by the short-form excluded DID is spared even though + // its ref signature (0) otherwise matches the burst. + let empty_excluded_short = rec("x-short", short, "spam"); + let cands = select_spam_candidates(&[empty_excluded_short], SPAM_BURST_TARGET_DID, |_| 0); + assert!( + cands.is_empty(), + "an empty repo owned by a short-form excluded DID must never be a candidate" + ); + } } diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 6c1fd92e..c48d2a09 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2786,4 +2786,140 @@ mod tests { "issue-write routes must be IP-throttled by the write brake" ); } + + // Adoption floor: an under-limit write must NOT be throttled. Guards against + // an off-by-one that braked the first request (invisible to the 429 tests, + // which all pre-exhaust the bucket). + #[sqlx::test] + async fn under_limit_write_is_not_throttled(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Ample budget; bucket NOT exhausted. + state.write_rate_limiter = + crate::rate_limit::RateLimiter::new(100, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let peer: SocketAddr = "203.0.113.150:7000".parse().unwrap(); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::PUT) + .uri("/api/v1/repos/someowner/somerepo/star") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_ne!( + router.oneshot(req).await.unwrap().status(), + StatusCode::TOO_MANY_REQUESTS, + "an under-limit write must pass the brake, not be 429'd" + ); + } + + // GITLAWB_WRITE_RATE_LIMIT=0 disables the brake end-to-end: no write is 429'd + // however many arrive from one IP. + #[sqlx::test] + async fn write_rate_limit_zero_disables_the_brake(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.write_rate_limiter = crate::rate_limit::RateLimiter::new(0, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let peer: SocketAddr = "203.0.113.151:7000".parse().unwrap(); + + let router = crate::server::build_router(state); + for _ in 0..5 { + let mut req = Request::builder() + .method(Method::PUT) + .uri("/api/v1/repos/someowner/somerepo/star") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_ne!( + router.clone().oneshot(req).await.unwrap().status(), + StatusCode::TOO_MANY_REQUESTS, + "a 0 write limit must disable the brake" + ); + } + } + + // The task/bounty/profile write groups share the write brake (same + // attachment as write_routes); prove each 429s at the route level. + #[sqlx::test] + async fn task_bounty_profile_writes_are_rate_limited(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.write_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let peer: SocketAddr = "203.0.113.152:7000".parse().unwrap(); + assert!(state.write_rate_limiter.check(&peer.ip().to_string()).await); + + let router = crate::server::build_router(state); + for (method, uri) in [ + (Method::POST, "/api/v1/tasks"), + (Method::POST, "/api/v1/repos/o/r/bounties"), + (Method::PUT, "/api/v1/profile"), + ] { + let mut req = Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .body(Body::from("{}")) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.clone().oneshot(req).await.unwrap().status(), + StatusCode::TOO_MANY_REQUESTS, + "write group {uri} must be IP-throttled by the write brake" + ); + } + } + + // /graphql/ws (subscriptions) is deliberately mounted AFTER the write brake + // layer, so it must stay unbraked even when the write bucket is exhausted. + #[sqlx::test] + async fn graphql_ws_is_not_braked(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.write_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let peer: SocketAddr = "203.0.113.153:7000".parse().unwrap(); + assert!(state.write_rate_limiter.check(&peer.ip().to_string()).await); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/graphql/ws") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + // Not a real ws upgrade, so the subscription service rejects it with some + // non-429 status; the point is the write brake never sees it. + assert_ne!( + router.oneshot(req).await.unwrap().status(), + StatusCode::TOO_MANY_REQUESTS, + "/graphql/ws must not be behind the write brake" + ); + } } diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 38f31e6e..06ff5081 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -23,15 +23,23 @@ pub struct Config { #[command(subcommand)] pub command: Option, - /// Directory where bare git repositories are stored - #[arg(long, env = "GITLAWB_REPOS_DIR", default_value = "./data/repos")] + /// Directory where bare git repositories are stored. `global` so it can + /// follow a subcommand (e.g. `gitlawb-node purge-spam --repos-dir …`). + #[arg( + long, + env = "GITLAWB_REPOS_DIR", + default_value = "./data/repos", + global = true + )] pub repos_dir: PathBuf, - /// PostgreSQL connection URL (Supabase or any Postgres instance) + /// PostgreSQL connection URL (Supabase or any Postgres instance). `global` so + /// admin subcommands accept it in either position. #[arg( long, env = "DATABASE_URL", - default_value = "postgresql://localhost/gitlawb" + default_value = "postgresql://localhost/gitlawb", + global = true )] pub database_url: String, diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 46b5be71..706d670b 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1260,20 +1260,24 @@ impl Db { Ok(()) } - /// Every repo row owned by exactly `owner_did` (an exact `owner_did` match, no - /// did:key normalization and no dedup) so a caller enumerating one DID's repos - /// sees every physical row. Backs the `purge-spam` admin tool, whose selection - /// then applies its own per-repo empty check and exclusion gate. Ordered by - /// `id` for a stable candidate listing. + /// Every repo row whose owner resolves to `owner_did` under did:key + /// normalization, so `did:key:z6…` and bare `z6…` rows of the same identity + /// both match (mirroring how ownership is resolved everywhere else via + /// `OWNER_KEY_CASE_SQL` / the `idx_repos_owner_key_name` index). No dedup, so a + /// caller enumerating one DID's repos sees every physical row. Backs the + /// `purge-spam` admin tool, whose selection then applies its own per-repo empty + /// check and exclusion gate (both normalization-consistent). Ordered by `id`. pub async fn list_repos_by_owner_did(&self, owner_did: &str) -> Result> { - let rows = sqlx::query( + let sql = format!( "SELECT id, name, owner_did, description, is_public, default_branch, created_at, updated_at, disk_path, forked_from, machine_id - FROM repos WHERE owner_did = $1 ORDER BY id", - ) - .bind(owner_did) - .fetch_all(&self.pool) - .await?; + FROM repos WHERE {key} = $1 ORDER BY id", + key = OWNER_KEY_CASE_SQL + ); + let rows = sqlx::query(&sql) + .bind(normalize_owner_key(owner_did)) + .fetch_all(&self.pool) + .await?; Ok(rows.into_iter().map(row_to_repo).collect()) } diff --git a/crates/gitlawb-node/src/server.rs b/crates/gitlawb-node/src/server.rs index 0c24d953..1456b700 100644 --- a/crates/gitlawb-node/src/server.rs +++ b/crates/gitlawb-node/src/server.rs @@ -55,6 +55,24 @@ fn add_auth_layers(router: Router, state: AppState) -> Router Self; +} + +impl WriteBraked for Router { + fn write_braked(self, limiter: &rate_limit::IpRateLimiter) -> Self { + self.layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) + .layer(axum::Extension(limiter.clone())) + } +} + pub fn build_router(state: AppState) -> Router { // ── GraphQL routes ───────────────────────────────────────────────────── let schema = state.graphql_schema.as_ref().clone(); @@ -77,8 +95,7 @@ pub fn build_router(state: AppState) -> Router { // a mutation POST, so it counts every /graphql request — acceptable, both // cost work); /graphql/ws subscriptions stay unbraked. .layer(middleware::from_fn(auth::optional_signature)) - .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) - .layer(axum::Extension(write_ip_limiter.clone())) + .write_braked(&write_ip_limiter) .route_service("/graphql/ws", GraphQLSubscription::new(schema)); // ── Task routes (write — require HTTP Signature + per-IP write brake) ── @@ -90,8 +107,7 @@ pub fn build_router(state: AppState) -> Router { .route("/api/v1/tasks/{id}/fail", post(tasks::fail_task)), state.clone(), ) - .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) - .layer(axum::Extension(write_ip_limiter.clone())); + .write_braked(&write_ip_limiter); // ── Task routes (read — open) ────────────────────────────────────────── let task_read_routes = Router::new() @@ -205,8 +221,7 @@ pub fn build_router(state: AppState) -> Router { ), state.clone(), ) - .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) - .layer(axum::Extension(write_ip_limiter.clone())); + .write_braked(&write_ip_limiter); // Body limit is raised to GITLAWB_MAX_PACK_BYTES (default 2 GB) for git // routes only — all other API routes keep axum's default 2 MB cap. @@ -273,8 +288,7 @@ pub fn build_router(state: AppState) -> Router { ), state.clone(), ) - .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) - .layer(axum::Extension(write_ip_limiter.clone())); + .write_braked(&write_ip_limiter); // ── Bounty routes (read — open) ────────────────────────────────────── let bounty_read_routes = Router::new() @@ -296,8 +310,7 @@ pub fn build_router(state: AppState) -> Router { Router::new().route("/api/v1/profile", axum::routing::put(profiles::set_profile)), state.clone(), ) - .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) - .layer(axum::Extension(write_ip_limiter.clone())); + .write_braked(&write_ip_limiter); // ── Issue routes (write — require HTTP Signature + per-IP write brake) ─ let issue_write_routes = add_auth_layers( @@ -312,8 +325,7 @@ pub fn build_router(state: AppState) -> Router { ), state.clone(), ) - .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) - .layer(axum::Extension(write_ip_limiter.clone())); + .write_braked(&write_ip_limiter); // ── Peer discovery routes ───────────────────────────────────────────── // Peer writes accept signatures when present and can require them after a From 27b4a2a74778b4eee56b20762b42ba25a975691e Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 15:23:19 -0500 Subject: [PATCH 07/61] test(review): close the reasoned-not-run gaps by execution - purge-spam execute now re-verifies emptiness immediately before delete (TOCTOU: a push between selection and delete no longer risks deleting a now-non-empty repo) and removes the on-disk bare repo dir so DB and disk stay consistent; a per-repo delete failure warns and continues rather than aborting the batch. - Tests (RED-verified where they cover new behavior): partition_for_delete skips a repo no longer empty; execute removes the on-disk dir (RED without the removal); list_repos_by_owner_did finds a short-form burst row (RED without the SQL normalization); every braked write group passes under-limit (per-group grant path, not just the star representative). --- crates/gitlawb-node/src/admin.rs | 136 +++++++++++++++++++++++++-- crates/gitlawb-node/src/api/repos.rs | 42 +++++++++ 2 files changed, 171 insertions(+), 7 deletions(-) diff --git a/crates/gitlawb-node/src/admin.rs b/crates/gitlawb-node/src/admin.rs index c428b765..01c03432 100644 --- a/crates/gitlawb-node/src/admin.rs +++ b/crates/gitlawb-node/src/admin.rs @@ -118,11 +118,17 @@ where /// bare-repo markers (`HEAD` file + `objects/` dir) before trusting any count; /// anything else fails closed (treated non-empty, skipped). fn on_disk_ref_count(repos_dir: &Path, repo: &RepoRecord) -> usize { - let path = store::repo_disk_path(repos_dir, &repo.owner_did, &repo.name); + ref_count_on_disk(repos_dir, &repo.owner_did, &repo.name) +} + +/// Core of [`on_disk_ref_count`], keyed on owner+name so the execute path can +/// re-verify emptiness right before deleting (using only a [`Candidate`]). +fn ref_count_on_disk(repos_dir: &Path, owner_did: &str, name: &str) -> usize { + let path = store::repo_disk_path(repos_dir, owner_did, name); if !path.join("HEAD").is_file() || !path.join("objects").is_dir() { // Not a bare git repo at the exact expected path. Do NOT trust a ref // count that git discovery could have read from a parent repository. - warn!(repo = %repo.id, path = %path.display(), + warn!(path = %path.display(), "purge-spam: path is not a bare git repo — treating as non-empty (skipped)"); return 1; } @@ -131,13 +137,36 @@ fn on_disk_ref_count(repos_dir: &Path, repo: &RepoRecord) -> usize { Err(e) => { // Fail closed: an unreadable repo is not provably empty, so keep it // out of the candidate set (report it as one ref so it's excluded). - warn!(repo = %repo.id, path = %path.display(), err = %e, + warn!(path = %path.display(), err = %e, "purge-spam: could not read refs — treating as non-empty (skipped)"); 1 } } } +/// Split selected candidates into (delete, skip) by a fresh emptiness re-check, +/// so a repo that gained a ref between selection and deletion (a TOCTOU push) +/// is never deleted. Pure over the `recheck` closure so the skip branch is +/// directly testable; the CLI wires the real on-disk re-check. +fn partition_for_delete( + candidates: &[Candidate], + mut recheck: F, +) -> (Vec<&Candidate>, Vec<&Candidate>) +where + F: FnMut(&Candidate) -> usize, +{ + let mut to_delete = Vec::new(); + let mut to_skip = Vec::new(); + for c in candidates { + if recheck(c) == 0 { + to_delete.push(c); + } else { + to_skip.push(c); + } + } + (to_delete, to_skip) +} + /// Run the `purge-spam` admin subcommand. /// /// Enumerates the target burst DID's repos, verifies each is empty on disk, @@ -189,20 +218,44 @@ pub async fn run_purge_spam(db: &Db, repos_dir: &Path, execute: bool) -> Result< return Ok(()); } + // Re-verify emptiness immediately before deleting: a push may have landed + // between selection and now (TOCTOU). Anything no longer empty is skipped, + // never deleted. + let (to_delete, to_skip) = partition_for_delete(&candidates, |c| { + ref_count_on_disk(&repos_dir, &c.owner_did, &c.name) + }); + for c in &to_skip { + warn!(repo = %c.id, "purge-spam: repo no longer empty at delete time — skipped (TOCTOU)"); + } + // Execute: delete per-repo, never a single blanket "delete all of owner X". + // A per-repo failure warns and continues rather than aborting the batch. let mut deleted = 0u64; - for c in &candidates { + for c in &to_delete { match db.delete_repo_by_id(&c.id).await { + Ok(0) => { + warn!(repo = %c.id, "purge-spam: repo row already gone — nothing to delete"); + } Ok(n) => { deleted += n; - info!(repo = %c.id, rows = n, "purge-spam: deleted repo row"); + // Remove the now-orphaned on-disk bare repo (empty, so cheap) so + // the DB row and disk stay consistent. + let path = store::repo_disk_path(&repos_dir, &c.owner_did, &c.name); + if let Err(e) = std::fs::remove_dir_all(&path) { + warn!(repo = %c.id, path = %path.display(), err = %e, + "purge-spam: deleted DB row but could not remove on-disk repo dir"); + } + info!(repo = %c.id, rows = n, "purge-spam: deleted repo row + on-disk dir"); } Err(e) => { - warn!(repo = %c.id, err = %e, "purge-spam: failed to delete repo row"); + warn!(repo = %c.id, err = %e, "purge-spam: failed to delete repo row — continuing"); } } } - println!("purge-spam: deleted {deleted} repo row(s)."); + println!( + "purge-spam: deleted {deleted} repo row(s), skipped {} (no longer empty).", + to_skip.len() + ); Ok(()) } @@ -349,6 +402,30 @@ mod tests { let got = select_spam_candidates(&repos, EXCLUDED_CONTENT, refs_by_id(&[])); assert!(got.is_empty(), "exclusion must win even on an empty repo"); } + + // TOCTOU: a candidate that gained a ref between selection and the pre-delete + // re-check must be skipped, not deleted; the rest of the batch still deletes. + #[test] + fn partition_for_delete_skips_repos_no_longer_empty() { + let cand = |id: &str| Candidate { + id: id.into(), + owner_did: "o".into(), + name: id.into(), + ref_count: 0, + }; + let cands = vec![cand("still-empty"), cand("now-nonempty")]; + // Re-check reports the second repo as no longer empty. + let (to_delete, to_skip) = + partition_for_delete(&cands, |c| usize::from(c.id == "now-nonempty")); + assert_eq!( + to_delete.iter().map(|c| c.id.as_str()).collect::>(), + ["still-empty"] + ); + assert_eq!( + to_skip.iter().map(|c| c.id.as_str()).collect::>(), + ["now-nonempty"] + ); + } } #[cfg(test)] @@ -463,6 +540,51 @@ mod db_tests { .is_some()); } + // Execute removes the on-disk bare repo dir too, so the DB row and disk stay + // consistent (no orphaned empty git dir left behind). + #[sqlx::test] + async fn execute_removes_the_on_disk_repo_dir(pool: PgPool) { + let db = db(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let empty = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&empty).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &empty.owner_did, &empty.name); + store::init_bare(&path).unwrap(); + assert!(path.exists(), "precondition: on-disk repo exists"); + + run_purge_spam(&db, tmp.path(), true).await.unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none(), + "DB row deleted" + ); + assert!(!path.exists(), "on-disk bare repo dir must be removed too"); + } + + // The DB query normalizes did:key form (OWNER_KEY_CASE_SQL), so a burst repo + // stored in SHORT (bare) form is still found when querying by the full-form + // target DID — the SQL side of the normalization fix. + #[sqlx::test] + async fn list_repos_by_owner_did_matches_short_form(pool: PgPool) { + let db = db(pool).await; + let short = crate::db::normalize_owner_key(SPAM_BURST_TARGET_DID); + assert_ne!(short, SPAM_BURST_TARGET_DID, "fixture must be short form"); + let repo = rec("short-owned", short, "spam"); + db.create_repo(&repo).await.unwrap(); + + let rows = db + .list_repos_by_owner_did(SPAM_BURST_TARGET_DID) + .await + .unwrap(); + assert!( + rows.iter().any(|r| r.id == "short-owned"), + "a short-form burst row must be found when querying by the full-form target" + ); + } + /// Create a single commit + ref in a bare repo via a throwaway worktree, so the /// repo reads as non-empty (≥1 ref). fn seed_one_ref(bare: &Path) { diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index c48d2a09..3b9d888e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2922,4 +2922,46 @@ mod tests { "/graphql/ws must not be behind the write brake" ); } + + // Adoption floor, per group: with an un-exhausted bucket, a write to EVERY + // braked group passes the brake (reaches auth/handler), not 429. Guards each + // group's grant path, not just the star representative. + #[sqlx::test] + async fn every_write_group_passes_under_limit(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.write_rate_limiter = + crate::rate_limit::RateLimiter::new(100, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let peer: SocketAddr = "203.0.113.160:7000".parse().unwrap(); + + let router = crate::server::build_router(state); + for (method, uri) in [ + (Method::PUT, "/api/v1/repos/o/r/star"), + (Method::POST, "/graphql"), + (Method::POST, "/api/v1/repos/o/r/issues/1/comments"), + (Method::POST, "/api/v1/tasks"), + (Method::POST, "/api/v1/repos/o/r/bounties"), + (Method::PUT, "/api/v1/profile"), + ] { + let mut req = Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .body(Body::from(r#"{"query":"{ __typename }"}"#)) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_ne!( + router.clone().oneshot(req).await.unwrap().status(), + StatusCode::TOO_MANY_REQUESTS, + "under-limit write to {uri} must pass the brake, not 429" + ); + } + } } From 98150db98cd9a27af5fd24a96af8a176f7b9bcab Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 14 Jul 2026 21:53:23 -0500 Subject: [PATCH 08/61] fix(node): reject path-traversal repo names in purge-spam before remove_dir_all A peer-mirror row skips API name validation, so a burst-owned repo can carry a '../' name; repo_disk_path joined it verbatim and the emptiness marker check passed on the traversed target, so purge-spam selected and remove_dir_all'd a repo OUTSIDE repos_dir. Validate the name via repo_store::validate_repo_name at selection (fail closed -> non-candidate) and assert canonical containment inside repos_dir immediately before the destructive remove. Adversarial RED->GREEN: a victim bare repo reachable via '../../victim' is deleted before, survives after; symlink-escape covered by path_within. --- crates/gitlawb-node/src/admin.rs | 100 +++++++++++++++++++++- crates/gitlawb-node/src/git/repo_store.rs | 2 +- 2 files changed, 99 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/admin.rs b/crates/gitlawb-node/src/admin.rs index 01c03432..90e3162a 100644 --- a/crates/gitlawb-node/src/admin.rs +++ b/crates/gitlawb-node/src/admin.rs @@ -124,6 +124,16 @@ fn on_disk_ref_count(repos_dir: &Path, repo: &RepoRecord) -> usize { /// Core of [`on_disk_ref_count`], keyed on owner+name so the execute path can /// re-verify emptiness right before deleting (using only a [`Candidate`]). fn ref_count_on_disk(repos_dir: &Path, owner_did: &str, name: &str) -> usize { + // Fail closed on an unsafe repo name BEFORE building any on-disk path. A + // peer-mirror row (which skips API name validation) can carry a `../` name; + // `repo_disk_path` would join it verbatim and resolve OUTSIDE `repos_dir`, + // pointing this "empty" check — and later the delete — at an unrelated repo. + // Reject it here so such a row is never a candidate (treated non-empty). + if let Err(e) = crate::git::repo_store::validate_repo_name(name) { + warn!(name = %name, err = %e, + "purge-spam: unsafe repo name — treating as non-empty (skipped)"); + return 1; + } let path = store::repo_disk_path(repos_dir, owner_did, name); if !path.join("HEAD").is_file() || !path.join("objects").is_dir() { // Not a bare git repo at the exact expected path. Do NOT trust a ref @@ -144,6 +154,17 @@ fn ref_count_on_disk(repos_dir: &Path, owner_did: &str, name: &str) -> usize { } } +/// Whether `path` resolves canonically inside `root`. Both are canonicalized so +/// symlinks and `..` segments are fully resolved before the containment test; a +/// path that does not exist (or a root that cannot be canonicalized) fails closed +/// to `false`. Used as the last gate before a destructive `remove_dir_all`. +fn path_within(path: &Path, root: &Path) -> bool { + match (std::fs::canonicalize(path), std::fs::canonicalize(root)) { + (Ok(p), Ok(r)) => p.starts_with(&r), + _ => false, + } +} + /// Split selected candidates into (delete, skip) by a fresh emptiness re-check, /// so a repo that gained a ref between selection and deletion (a TOCTOU push) /// is never deleted. Pure over the `recheck` closure so the skip branch is @@ -239,9 +260,16 @@ pub async fn run_purge_spam(db: &Db, repos_dir: &Path, execute: bool) -> Result< Ok(n) => { deleted += n; // Remove the now-orphaned on-disk bare repo (empty, so cheap) so - // the DB row and disk stay consistent. + // the DB row and disk stay consistent. Belt-and-suspenders: assert + // the resolved path is canonically INSIDE repos_dir before any + // remove_dir_all, so a symlinked slug dir or any residual traversal + // can never delete outside the repo root (the name is already + // validated at selection; this guards the destructive op itself). let path = store::repo_disk_path(&repos_dir, &c.owner_did, &c.name); - if let Err(e) = std::fs::remove_dir_all(&path) { + if !path_within(&path, &repos_dir) { + warn!(repo = %c.id, path = %path.display(), + "purge-spam: on-disk path escapes repos_dir — refusing to remove"); + } else if let Err(e) = std::fs::remove_dir_all(&path) { warn!(repo = %c.id, path = %path.display(), err = %e, "purge-spam: deleted DB row but could not remove on-disk repo dir"); } @@ -564,6 +592,47 @@ mod db_tests { assert!(!path.exists(), "on-disk bare repo dir must be removed too"); } + // U1 (M3): a burst-owned row whose NAME traverses out of repos_dir must never + // cause a delete OUTSIDE repos_dir. Adversarial must-not: a real empty bare repo + // planted as a "victim" beside repos_dir is reachable from repos_dir// via + // a `../../victim` name; because the traversed path IS a real bare repo, the + // marker check passes and the candidate is selected — then remove_dir_all would + // delete the victim. The name validator must reject it so the victim survives. + #[sqlx::test] + async fn traversal_name_cannot_delete_a_repo_outside_repos_dir(pool: PgPool) { + let db = db(pool).await; + let root = tempfile::TempDir::new().unwrap(); + let repos_dir = root.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + + // The burst DID's own slug dir must exist for the OS to resolve the `..` + // segments (a burst that owns any normal repo already has this dir). + let slug = SPAM_BURST_TARGET_DID.replace([':', '/'], "_"); + std::fs::create_dir_all(repos_dir.join(&slug)).unwrap(); + + // Victim: a real empty bare repo OUTSIDE repos_dir (sibling under root). + let victim = root.path().join("victim.git"); + store::init_bare(&victim).unwrap(); + assert!(victim.join("HEAD").is_file(), "victim precondition"); + + // Sanity: the evil name resolves from repos_dir// onto the victim. + let evil = rec("evil", SPAM_BURST_TARGET_DID, "../../victim"); + let traversed = store::repo_disk_path(&repos_dir, &evil.owner_did, &evil.name); + assert_eq!( + std::fs::canonicalize(&traversed).unwrap(), + std::fs::canonicalize(&victim).unwrap(), + "test setup: the evil name must resolve onto the victim" + ); + + db.create_repo(&evil).await.unwrap(); + run_purge_spam(&db, &repos_dir, true).await.unwrap(); + + assert!( + victim.join("HEAD").is_file(), + "a repo OUTSIDE repos_dir must never be deleted via a traversal name" + ); + } + // The DB query normalizes did:key form (OWNER_KEY_CASE_SQL), so a burst repo // stored in SHORT (bare) form is still found when querying by the full-form // target DID — the SQL side of the normalization fix. @@ -648,6 +717,33 @@ mod db_tests { assert_eq!(on_disk_ref_count(tmp.path(), &repo), 0); } + /// The belt-and-suspenders containment gate (`path_within`) must reject a path + /// that resolves outside repos_dir even when the *name* itself is innocuous — + /// e.g. the owner slug dir is a symlink pointing elsewhere. Layer 1 (name + /// validation) can't see this; only the canonical-containment check catches it. + #[test] + fn path_within_rejects_symlink_escape() { + let root = tempfile::TempDir::new().unwrap(); + let repos_dir = root.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + + // A real dir outside repos_dir, and a symlink INTO repos_dir that targets it. + let outside = root.path().join("outside.git"); + std::fs::create_dir_all(&outside).unwrap(); + let link = repos_dir.join("evil.git"); + std::os::unix::fs::symlink(&outside, &link).unwrap(); + + // The name is innocuous, but the path resolves outside repos_dir. + assert!( + !path_within(&link, &repos_dir), + "a symlink escaping repos_dir must fail the containment gate" + ); + // A genuine path inside repos_dir passes. + let inside = repos_dir.join("real.git"); + std::fs::create_dir_all(&inside).unwrap(); + assert!(path_within(&inside, &repos_dir), "an in-root path must pass"); + } + /// The exclusion gate and the target scope must never overlap: if the burst /// target were ever set to an excluded DID, the gate would fail to protect it. #[test] diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index a5c367e9..c7aa954a 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -324,7 +324,7 @@ fn validate_owner_did(owner_did: &str) -> Result<()> { Ok(()) } -fn validate_repo_name(repo_name: &str) -> Result<()> { +pub(crate) fn validate_repo_name(repo_name: &str) -> Result<()> { if repo_name.is_empty() { anyhow::bail!("repo_name is empty"); } From 1688bdc05d54e638f8290331008acec3e20ebc26 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 14 Jul 2026 21:58:31 -0500 Subject: [PATCH 09/61] fix(node): delete child rows transactionally in delete_repo_by_id delete_repo_by_id was a bare DELETE FROM repos with no cascade and the schema has no FKs, so every child row (ref_certificates, pull_requests + pr_reviews/ pr_comments, webhooks, push_events, protected_branches, repo_stars, repo_replicas, repo_labels, visibility_rules, encrypted_blobs, repo_icaptcha_proofs, agent_tasks, and the slug-keyed branch_cids/sync_queue/received_ref_updates/arweave_anchors) was orphaned and could re-join a later repo reusing the id. Wrap parent+child deletes in one transaction, resolving the owner_short/name slug for the repo-TEXT children. bounties (financial) and issue_comments (unmappable) are left intact by design. RED->GREEN test seeds a repo_id child, a slug child, and a PR grandchild. --- crates/gitlawb-node/src/db/mod.rs | 138 +++++++++++++++++++++++++++++- 1 file changed, 137 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 706d670b..5b2fb79e 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1286,10 +1286,77 @@ impl Db { /// the `purge-spam` tool deletes a vetted candidate list per-repo, never a /// blanket "delete all repos of owner X". pub async fn delete_repo_by_id(&self, id: &str) -> Result { + let mut tx = self.pool.begin().await?; + + // Resolve the row's identity so the `repo`-slug-keyed children (keyed on + // `normalize_owner_key(owner)/name`, not `repos.id`) can be matched. Absent + // row → nothing to delete; commit the empty tx and report 0. + let row: Option<(String, String)> = + sqlx::query_as("SELECT owner_did, name FROM repos WHERE id = $1") + .bind(id) + .fetch_optional(&mut *tx) + .await?; + let Some((owner_did, name)) = row else { + tx.commit().await?; + return Ok(0); + }; + let slug = format!("{}/{}", normalize_owner_key(&owner_did), name); + + // Grandchildren first: PR reviews/comments key on pr_id, so delete them + // before the parent PRs vanish. + for table in ["pr_reviews", "pr_comments"] { + sqlx::query(&format!( + "DELETE FROM {table} WHERE pr_id IN (SELECT id FROM pull_requests WHERE repo_id = $1)" + )) + .bind(id) + .execute(&mut *tx) + .await?; + } + + // Direct children keyed on repos.id. + for table in [ + "push_events", + "ref_certificates", + "pull_requests", + "webhooks", + "agent_tasks", + "protected_branches", + "repo_stars", + "repo_replicas", + "repo_labels", + "visibility_rules", + "encrypted_blobs", + "repo_icaptcha_proofs", + ] { + sqlx::query(&format!("DELETE FROM {table} WHERE repo_id = $1")) + .bind(id) + .execute(&mut *tx) + .await?; + } + + // Children keyed on the derived `owner_short/name` slug rather than the id. + for table in [ + "branch_cids", + "sync_queue", + "received_ref_updates", + "arweave_anchors", + ] { + sqlx::query(&format!("DELETE FROM {table} WHERE repo = $1")) + .bind(&slug) + .execute(&mut *tx) + .await?; + } + // NOTE: `bounties` (financial: amount/wallet/tx_hash) and `issue_comments` + // (no issues table to map issue_id → repo) are deliberately NOT cascaded + // here — dropping money records or unmappable rows on a repo delete would + // be wrong. They are left intact by design. + let result = sqlx::query("DELETE FROM repos WHERE id = $1") .bind(id) - .execute(&self.pool) + .execute(&mut *tx) .await?; + + tx.commit().await?; Ok(result.rows_affected()) } } @@ -3570,6 +3637,75 @@ mod dedup_db_tests { } } + /// U2 (M7): deleting a repo must not orphan its child rows. Seeds one child in + /// a `repo_id`-keyed table (ref_certificates), one in a `repo`-slug-keyed table + /// (branch_cids), and a PR with a grandchild review (`pr_id`-keyed). Before the + /// transactional cascade these all survive `delete_repo_by_id` (RED); after, the + /// row and every child are gone (GREEN). + #[sqlx::test] + async fn delete_repo_by_id_removes_child_rows(pool: PgPool) { + let db = db(pool).await; + let owner = "did:key:z6MkChildOwnerFixtureForCascadeDelete"; + let repo = rec( + "rid-cascade", + owner, + "victim", + "d", + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:00Z", + ); + db.create_repo(&repo).await.unwrap(); + let slug = format!("{}/victim", crate::db::normalize_owner_key(owner)); + + sqlx::query( + "INSERT INTO ref_certificates (id, repo_id, ref_name, old_sha, new_sha, pusher_did, node_did, signature, issued_at) + VALUES ('rc1', 'rid-cascade', 'refs/heads/main', '0', '1', 'p', 'n', 'sig', '2026-01-01T00:00:00Z')", + ).execute(db.pool()).await.unwrap(); + sqlx::query( + "INSERT INTO branch_cids (repo, ref_name, sha, cid, node_did, updated_at) + VALUES ($1, 'refs/heads/main', '1', 'cid', 'n', '2026-01-01T00:00:00Z')", + ).bind(&slug).execute(db.pool()).await.unwrap(); + sqlx::query( + "INSERT INTO pull_requests (id, repo_id, number, title, author_did, source_branch, target_branch, created_at, updated_at) + VALUES ('pr1', 'rid-cascade', 1, 't', 'a', 'b', 'main', '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z')", + ).execute(db.pool()).await.unwrap(); + sqlx::query( + "INSERT INTO pr_reviews (id, pr_id, reviewer_did, status, created_at) + VALUES ('rev1', 'pr1', 'r', 'approved', '2026-01-01T00:00:00Z')", + ).execute(db.pool()).await.unwrap(); + + let removed = db.delete_repo_by_id("rid-cascade").await.unwrap(); + assert_eq!(removed, 1, "parent repo row deleted"); + + async fn count(db: &Db, sql: &str, arg: &str) -> i64 { + sqlx::query_scalar::<_, i64>(sql) + .bind(arg) + .fetch_one(db.pool()) + .await + .unwrap() + } + assert_eq!( + count(&db, "SELECT COUNT(*) FROM ref_certificates WHERE repo_id=$1", "rid-cascade").await, + 0, + "repo_id-keyed child (ref_certificates) must be deleted" + ); + assert_eq!( + count(&db, "SELECT COUNT(*) FROM branch_cids WHERE repo=$1", &slug).await, + 0, + "slug-keyed child (branch_cids) must be deleted" + ); + assert_eq!( + count(&db, "SELECT COUNT(*) FROM pull_requests WHERE repo_id=$1", "rid-cascade").await, + 0, + "pull_requests must be deleted" + ); + assert_eq!( + count(&db, "SELECT COUNT(*) FROM pr_reviews WHERE pr_id=$1", "pr1").await, + 0, + "PR grandchild (pr_reviews) must be deleted with its parent PR" + ); + } + /// The canonical `did:key:` row and the short-owner mirror row of one logical /// repo collapse to a single deduped entry: the canonical row wins and inherits /// the group's most recent `updated_at`. From 4943430f58bd7494df7ef02ee62c534ead7d1a89 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:03:49 -0500 Subject: [PATCH 10/61] fix(node): reap write_rate_limiter in the periodic cleanup loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PR's new write_rate_limiter was the one limiter omitted from main()'s hand-maintained cleanup list, so its key map never shed expired entries until the 200k cap. Move the sweep into AppState::cleanup_rate_limiters (co-located with the limiter fields, single source of truth) and call it from the loop, so a newly-added limiter is reaped rather than silently dropped. Guard test seeds a reclaimable entry into write_rate_limiter and asserts the sweep reaps it — RED if write_rate_limiter is dropped from cleanup_rate_limiters (verified). --- crates/gitlawb-node/src/main.rs | 15 +++------ crates/gitlawb-node/src/rate_limit.rs | 18 +++++++++++ crates/gitlawb-node/src/state.rs | 44 +++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 10 deletions(-) diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 88984469..4d492db9 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -437,22 +437,17 @@ async fn main() -> Result<()> { // Periodic cleanup of expired rate limit entries + consumed-proof ledger { - let rl = state.rate_limiter.clone(); - let create_ip_rl = state.create_ip_rate_limiter.clone(); - let push_rl = state.push_rate_limiter.clone(); - let sync_trigger_rl = state.sync_trigger_rate_limiter.clone(); - let peer_write_rl = state.peer_write_rate_limiter.clone(); + // Clone the whole state so the sweep uses AppState::cleanup_rate_limiters + // (the single source of truth that reaps EVERY limiter, including + // write_rate_limiter) rather than a hand-maintained list that can omit one. + let state = state.clone(); let db = state.db.clone(); let mut shutdown_rx = state.subscribe_shutdown(); tokio::spawn(async move { loop { tokio::select! { _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { - rl.cleanup().await; - create_ip_rl.cleanup().await; - push_rl.cleanup().await; - sync_trigger_rl.cleanup().await; - peer_write_rl.cleanup().await; + state.cleanup_rate_limiters().await; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs() as i64) diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index 4c41575c..1d38a0ae 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -130,6 +130,24 @@ impl RateLimiter { !w.timestamps.is_empty() }); } + + /// Test-only: insert a fully-expired entry (no live timestamps) that the next + /// `cleanup()` must reclaim. Models a key whose window has aged out, without + /// depending on wall-clock sleeps or a short window. Used cross-module by the + /// AppState cleanup guard test (H2). + #[cfg(test)] + pub(crate) async fn insert_reclaimable_for_test(&self, key: &str) { + self.state + .lock() + .await + .insert(key.to_string(), Window { timestamps: vec![] }); + } + + /// Test-only: number of distinct keys currently tracked. + #[cfg(test)] + pub(crate) async fn tracked_keys(&self) -> usize { + self.state.lock().await.len() + } } pub async fn rate_limit_by_did(request: Request, next: Next) -> Response { diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index d38541f9..6c033392 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -127,4 +127,48 @@ impl AppState { pub fn is_shutting_down(&self) -> bool { *self.shutdown_tx.borrow() } + + /// Reclaim expired entries from EVERY rate limiter. Single source of truth for + /// the periodic cleanup loop (main.rs), co-located with the limiter fields so a + /// newly-added limiter is cleaned here rather than silently omitted from a + /// hand-maintained list in main() (H2: `write_rate_limiter` was the omitted + /// one). Every limiter field above MUST be swept here; the guard test + /// `cleanup_rate_limiters_reaps_the_write_limiter` fails if it is not. + pub(crate) async fn cleanup_rate_limiters(&self) { + self.rate_limiter.cleanup().await; + self.create_ip_rate_limiter.cleanup().await; + self.write_rate_limiter.cleanup().await; + self.push_rate_limiter.cleanup().await; + self.sync_trigger_rate_limiter.cleanup().await; + self.peer_write_rate_limiter.cleanup().await; + } +} + +#[cfg(test)] +mod tests { + /// H2 guard: the write-surface limiter must be reaped by the shared cleanup + /// routine. Seeds a reclaimable entry into `write_rate_limiter` and asserts + /// `cleanup_rate_limiters` removes it. Goes RED if `write_rate_limiter` is + /// dropped from `cleanup_rate_limiters` — the exact H2 omission, now guarded. + #[tokio::test] + async fn cleanup_rate_limiters_reaps_the_write_limiter() { + let state = crate::test_support::test_state_lazy(); + state + .write_rate_limiter + .insert_reclaimable_for_test("some-ip") + .await; + assert_eq!( + state.write_rate_limiter.tracked_keys().await, + 1, + "precondition: the reclaimable entry is tracked" + ); + + state.cleanup_rate_limiters().await; + + assert_eq!( + state.write_rate_limiter.tracked_keys().await, + 0, + "write_rate_limiter must be reaped by cleanup_rate_limiters (H2)" + ); + } } From 130516743ae5880bdf929324fdb8d9bed63c1065 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:14:15 -0500 Subject: [PATCH 11/61] fix(node): hold the per-repo advisory lock across purge recheck+delete MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit purge-spam took no write lock, so a concurrent receive-pack could land a ref between the emptiness recheck and the delete, silently losing a live push. Add RepoStore::try_lock_repo — a lock-only, non-blocking counterpart to acquire_write (no Tigris I/O) whose guard holds a dedicated pooled connection so the advisory lock and its unlock run on the same session. purge now locks each candidate, rechecks emptiness UNDER the lock, deletes, and releases; a repo held by a live writer is skipped, not force-deleted. RED->GREEN: a locked empty repo survives purge, then deletes once the writer releases (neuter-check confirms the lock is load-bearing). --- crates/gitlawb-node/src/admin.rs | 113 +++++++++++++++++++--- crates/gitlawb-node/src/git/repo_store.rs | 48 +++++++++ crates/gitlawb-node/src/main.rs | 9 +- 3 files changed, 156 insertions(+), 14 deletions(-) diff --git a/crates/gitlawb-node/src/admin.rs b/crates/gitlawb-node/src/admin.rs index 90e3162a..2d3911c1 100644 --- a/crates/gitlawb-node/src/admin.rs +++ b/crates/gitlawb-node/src/admin.rs @@ -194,7 +194,12 @@ where /// applies the exclusion gate, prints one dry-run row per candidate with owner + /// ref-count evidence, and — only when `execute` is true — deletes the DB row of /// each candidate one at a time. Dry-run (the default) deletes nothing. -pub async fn run_purge_spam(db: &Db, repos_dir: &Path, execute: bool) -> Result<()> { +pub async fn run_purge_spam( + db: &Db, + repo_store: &crate::git::repo_store::RepoStore, + repos_dir: &Path, + execute: bool, +) -> Result<()> { let repos_dir: PathBuf = repos_dir.to_path_buf(); let rows = db .list_repos_by_owner_did(SPAM_BURST_TARGET_DID) @@ -252,7 +257,32 @@ pub async fn run_purge_spam(db: &Db, repos_dir: &Path, execute: bool) -> Result< // Execute: delete per-repo, never a single blanket "delete all of owner X". // A per-repo failure warns and continues rather than aborting the batch. let mut deleted = 0u64; + let mut skipped_locked = 0usize; for c in &to_delete { + // Hold the per-repo advisory lock across the FINAL emptiness recheck and + // the delete, so a concurrent receive-pack cannot land a ref in the window + // between recheck and delete (M4). A repo currently locked by a live writer + // is skipped, never force-deleted out from under the push. + let guard = match repo_store.try_lock_repo(&c.owner_did, &c.name).await { + Ok(Some(g)) => g, + Ok(None) => { + warn!(repo = %c.id, "purge-spam: repo is locked by a live writer — skipped"); + skipped_locked += 1; + continue; + } + Err(e) => { + warn!(repo = %c.id, err = %e, "purge-spam: could not acquire repo lock — skipped"); + skipped_locked += 1; + continue; + } + }; + // Authoritative recheck UNDER the lock: a ref that landed before we locked + // makes the repo non-empty, so it must not be deleted. + if ref_count_on_disk(&repos_dir, &c.owner_did, &c.name) != 0 { + warn!(repo = %c.id, "purge-spam: repo no longer empty under lock — skipped (TOCTOU)"); + guard.release().await; + continue; + } match db.delete_repo_by_id(&c.id).await { Ok(0) => { warn!(repo = %c.id, "purge-spam: repo row already gone — nothing to delete"); @@ -279,9 +309,10 @@ pub async fn run_purge_spam(db: &Db, repos_dir: &Path, execute: bool) -> Result< warn!(repo = %c.id, err = %e, "purge-spam: failed to delete repo row — continuing"); } } + guard.release().await; } println!( - "purge-spam: deleted {deleted} repo row(s), skipped {} (no longer empty).", + "purge-spam: deleted {deleted} repo row(s), skipped {} (no longer empty), {skipped_locked} (locked by a live writer).", to_skip.len() ); Ok(()) @@ -463,12 +494,18 @@ mod db_tests { use chrono::Utc; use sqlx::PgPool; - async fn db(pool: PgPool) -> Db { - let db = Db::for_testing(pool); + async fn db(pool: &PgPool) -> Db { + let db = Db::for_testing(pool.clone()); db.run_migrations().await.unwrap(); db } + /// Lock-only RepoStore over a test pool + repos_dir (no Tigris), for the purge + /// callers that now take the advisory lock (M4). + fn test_store(repos_dir: &Path, pool: &PgPool) -> crate::git::repo_store::RepoStore { + crate::git::repo_store::RepoStore::for_testing(repos_dir.to_path_buf(), pool.clone()) + } + fn rec(id: &str, owner: &str, name: &str) -> RepoRecord { RepoRecord { id: id.to_string(), @@ -498,7 +535,7 @@ mod db_tests { // which is fine here: the assertion is that dry-run mutates no rows regardless. #[sqlx::test] async fn dry_run_deletes_nothing(pool: PgPool) { - let db = db(pool).await; + let db = db(&pool).await; for r in [ rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"), rec("t-nonempty", SPAM_BURST_TARGET_DID, "real"), @@ -513,7 +550,8 @@ mod db_tests { // Empty repos dir: no repo exists on disk, so nothing is provably empty. let tmp = tempfile::TempDir::new().unwrap(); - run_purge_spam(&db, tmp.path(), false).await.unwrap(); + let store = test_store(tmp.path(), &pool); + run_purge_spam(&db, &store, tmp.path(), false).await.unwrap(); let after = count_rows(&db).await; assert_eq!(after, before, "dry-run must not delete any repo rows"); @@ -524,7 +562,7 @@ mod db_tests { // repo. This exercises the DB wiring end-to-end with a real empty repo on disk. #[sqlx::test] async fn execute_deletes_only_the_empty_target_repo_on_disk(pool: PgPool) { - let db = db(pool).await; + let db = db(&pool).await; let tmp = tempfile::TempDir::new().unwrap(); // One empty target repo (real bare repo, zero refs) and one target repo @@ -548,7 +586,8 @@ mod db_tests { assert_eq!(store::list_refs(&empty_path).unwrap().len(), 0); assert!(!store::list_refs(&refs_path).unwrap().is_empty()); - run_purge_spam(&db, tmp.path(), true).await.unwrap(); + let repo_store = test_store(tmp.path(), &pool); + run_purge_spam(&db, &repo_store, tmp.path(), true).await.unwrap(); // Only the empty target repo row is gone. assert!(db @@ -572,7 +611,7 @@ mod db_tests { // consistent (no orphaned empty git dir left behind). #[sqlx::test] async fn execute_removes_the_on_disk_repo_dir(pool: PgPool) { - let db = db(pool).await; + let db = db(&pool).await; let tmp = tempfile::TempDir::new().unwrap(); let empty = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); db.create_repo(&empty).await.unwrap(); @@ -580,7 +619,8 @@ mod db_tests { store::init_bare(&path).unwrap(); assert!(path.exists(), "precondition: on-disk repo exists"); - run_purge_spam(&db, tmp.path(), true).await.unwrap(); + let repo_store = test_store(tmp.path(), &pool); + run_purge_spam(&db, &repo_store, tmp.path(), true).await.unwrap(); assert!( db.get_repo(SPAM_BURST_TARGET_DID, "spam1") @@ -592,6 +632,52 @@ mod db_tests { assert!(!path.exists(), "on-disk bare repo dir must be removed too"); } + // U4 (M4): a repo whose per-repo advisory lock is held by a live writer must be + // SKIPPED by purge, not deleted out from under the push. Holds the lock via the + // same RepoStore (a separate pooled connection), runs execute, and asserts the + // row + on-disk dir survive; once the writer releases, purge deletes it. + #[sqlx::test] + async fn locked_repo_is_skipped_not_deleted(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let empty = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&empty).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &empty.owner_did, &empty.name); + store::init_bare(&path).unwrap(); + + let repo_store = test_store(tmp.path(), &pool); + // A live writer holds the per-repo advisory lock. + let held = repo_store + .try_lock_repo(&empty.owner_did, &empty.name) + .await + .unwrap() + .expect("lock should be free initially"); + + run_purge_spam(&db, &repo_store, tmp.path(), true).await.unwrap(); + + // Locked → skipped: both the row and the on-disk dir survive. + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_some(), + "a repo locked by a live writer must NOT be deleted" + ); + assert!(path.exists(), "on-disk dir must survive while locked"); + + // Once the writer releases, the empty repo is deleted (the lock was the + // only thing protecting it — baseline both ways). + held.release().await; + run_purge_spam(&db, &repo_store, tmp.path(), true).await.unwrap(); + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none(), + "once unlocked, the empty repo is deleted" + ); + } + // U1 (M3): a burst-owned row whose NAME traverses out of repos_dir must never // cause a delete OUTSIDE repos_dir. Adversarial must-not: a real empty bare repo // planted as a "victim" beside repos_dir is reachable from repos_dir// via @@ -600,7 +686,7 @@ mod db_tests { // delete the victim. The name validator must reject it so the victim survives. #[sqlx::test] async fn traversal_name_cannot_delete_a_repo_outside_repos_dir(pool: PgPool) { - let db = db(pool).await; + let db = db(&pool).await; let root = tempfile::TempDir::new().unwrap(); let repos_dir = root.path().join("repos"); std::fs::create_dir_all(&repos_dir).unwrap(); @@ -625,7 +711,8 @@ mod db_tests { ); db.create_repo(&evil).await.unwrap(); - run_purge_spam(&db, &repos_dir, true).await.unwrap(); + let repo_store = test_store(&repos_dir, &pool); + run_purge_spam(&db, &repo_store, &repos_dir, true).await.unwrap(); assert!( victim.join("HEAD").is_file(), @@ -638,7 +725,7 @@ mod db_tests { // target DID — the SQL side of the normalization fix. #[sqlx::test] async fn list_repos_by_owner_did_matches_short_form(pool: PgPool) { - let db = db(pool).await; + let db = db(&pool).await; let short = crate::db::normalize_owner_key(SPAM_BURST_TARGET_DID); assert_ne!(short, SPAM_BURST_TARGET_DID, "fixture must be short form"); let repo = rec("short-owned", short, "spam"); diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index c7aa954a..669d8d51 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -207,6 +207,35 @@ impl RepoStore { }) } + /// Try to acquire ONLY the per-repo advisory lock, non-blocking, with no + /// Tigris I/O — the lightweight counterpart to [`acquire_write`](Self::acquire_write) + /// for out-of-band admin ops (purge-spam) that must mutually exclude a live + /// push during a destructive delete but never download/re-upload the repo. + /// + /// Returns `Some(guard)` if the lock was free, `None` if another writer holds + /// it (so the caller can skip rather than block). The guard holds a dedicated + /// pooled connection for its whole lifetime, so the lock lives on that one + /// connection and `release()` unlocks it on the SAME connection — a plain + /// pool query could unlock on a different connection and silently fail. + pub async fn try_lock_repo( + &self, + owner_did: &str, + repo_name: &str, + ) -> Result> { + let (owner_slug, _local) = self.local_path(owner_did, repo_name)?; + let lock_key = advisory_lock_key(&owner_slug, repo_name); + let mut conn = self.pool.acquire().await.context("acquiring lock connection")?; + let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(lock_key) + .fetch_one(&mut *conn) + .await + .context("try advisory lock")?; + if !row.0 { + return Ok(None); + } + Ok(Some(RepoLockGuard { conn, lock_key })) + } + /// Initialize a new bare repo on local disk and upload to Tigris. pub async fn init(&self, owner_did: &str, repo_name: &str) -> Result { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; @@ -392,6 +421,25 @@ impl RepoWriteGuard { } } +/// Lock-only guard from [`RepoStore::try_lock_repo`]. Holds the Postgres advisory +/// lock (and the dedicated connection it lives on) until `release()`. No Tigris +/// I/O — unlike [`RepoWriteGuard`], releasing does NOT upload anything. +pub struct RepoLockGuard { + conn: sqlx::pool::PoolConnection, + lock_key: i64, +} + +impl RepoLockGuard { + /// Release the advisory lock on the same connection that took it, then return + /// the connection to the pool. + pub async fn release(mut self) { + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(self.lock_key) + .execute(&mut *self.conn) + .await; + } +} + /// Compute a stable i64 hash for a Postgres advisory lock key. fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { use std::hash::{Hash, Hasher}; diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 4d492db9..ff53a927 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -601,7 +601,14 @@ async fn run_admin_command(command: config::Command, config: &Config) -> Result< ) .await .context("connecting to database for purge-spam")?; - admin::run_purge_spam(&db, &config.repos_dir, execute).await + // Lock-only RepoStore (no Tigris): purge takes the per-repo advisory + // lock to exclude a live push, but never downloads/uploads archives. + let repo_store = git::repo_store::RepoStore::new( + config.repos_dir.clone(), + None, + db.pool().clone(), + ); + admin::run_purge_spam(&db, &repo_store, &config.repos_dir, execute).await } } } From aab70ec25d542e050271b238229badd5b7fb1b29 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:20:09 -0500 Subject: [PATCH 12/61] fix(node): count purge-spam disk-removal failures separately from deletes The Ok(n) arm incremented the deleted total before remove_dir_all, whose failure only warn!'d, so the summary reported N deleted even when the on-disk dir survived (DB/disk drift shown as clean success). run_purge_spam now returns a PurgeSummary and counts a failed (or containment-refused) on-disk removal in disk_failed, never folding it into deleted; the summary line surfaces 'M on-disk removal(s) failed'. RED->GREEN: a read-only parent dir forces the removal to fail; deleted=1 AND disk_failed=1 with the dir surviving (neuter-check confirms the counter is load-bearing). --- crates/gitlawb-node/src/admin.rs | 92 +++++++++++++++++++++++++++----- crates/gitlawb-node/src/main.rs | 4 +- 2 files changed, 83 insertions(+), 13 deletions(-) diff --git a/crates/gitlawb-node/src/admin.rs b/crates/gitlawb-node/src/admin.rs index 2d3911c1..2ab52529 100644 --- a/crates/gitlawb-node/src/admin.rs +++ b/crates/gitlawb-node/src/admin.rs @@ -29,6 +29,25 @@ pub const EXCLUDED_DIDS: &[&str] = &[ "did:key:z6MkqRzACJ5iCDdkiymAPK3gq18z2iecZHeAuUyW6JnwRfoM", ]; +/// Outcome tally of a `run_purge_spam` execute pass. Returned so the DB-delete +/// count is never conflated with full success: `disk_failed` records repos whose +/// row was deleted but whose on-disk dir could not be removed (or escaped the +/// repos_dir containment check), so an operator sees the DB/disk drift instead of +/// a clean "N deleted" summary. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct PurgeSummary { + /// Repo rows actually deleted from the DB. + pub deleted: u64, + /// Candidates skipped because they were no longer empty (pre-filter or the + /// authoritative recheck under the lock). + pub skipped_not_empty: usize, + /// Candidates skipped because a live writer held the per-repo lock. + pub skipped_locked: usize, + /// Rows deleted whose on-disk dir removal FAILED (or was refused by the + /// containment guard) — DB/disk drift the operator must reconcile. + pub disk_failed: usize, +} + /// A repo selected for purge, with the evidence that qualified it. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Candidate { @@ -199,7 +218,7 @@ pub async fn run_purge_spam( repo_store: &crate::git::repo_store::RepoStore, repos_dir: &Path, execute: bool, -) -> Result<()> { +) -> Result { let repos_dir: PathBuf = repos_dir.to_path_buf(); let rows = db .list_repos_by_owner_did(SPAM_BURST_TARGET_DID) @@ -220,7 +239,7 @@ pub async fn run_purge_spam( if candidates.is_empty() { println!("purge-spam: no empty spam-burst repos found for {SPAM_BURST_TARGET_DID}"); - return Ok(()); + return Ok(PurgeSummary::default()); } println!( @@ -241,7 +260,7 @@ pub async fn run_purge_spam( "purge-spam: dry-run — nothing deleted. Re-run with --execute to delete the {} candidate(s).", candidates.len() ); - return Ok(()); + return Ok(PurgeSummary::default()); } // Re-verify emptiness immediately before deleting: a push may have landed @@ -256,8 +275,10 @@ pub async fn run_purge_spam( // Execute: delete per-repo, never a single blanket "delete all of owner X". // A per-repo failure warns and continues rather than aborting the batch. - let mut deleted = 0u64; - let mut skipped_locked = 0usize; + let mut summary = PurgeSummary { + skipped_not_empty: to_skip.len(), + ..PurgeSummary::default() + }; for c in &to_delete { // Hold the per-repo advisory lock across the FINAL emptiness recheck and // the delete, so a concurrent receive-pack cannot land a ref in the window @@ -267,12 +288,12 @@ pub async fn run_purge_spam( Ok(Some(g)) => g, Ok(None) => { warn!(repo = %c.id, "purge-spam: repo is locked by a live writer — skipped"); - skipped_locked += 1; + summary.skipped_locked += 1; continue; } Err(e) => { warn!(repo = %c.id, err = %e, "purge-spam: could not acquire repo lock — skipped"); - skipped_locked += 1; + summary.skipped_locked += 1; continue; } }; @@ -280,6 +301,7 @@ pub async fn run_purge_spam( // makes the repo non-empty, so it must not be deleted. if ref_count_on_disk(&repos_dir, &c.owner_did, &c.name) != 0 { warn!(repo = %c.id, "purge-spam: repo no longer empty under lock — skipped (TOCTOU)"); + summary.skipped_not_empty += 1; guard.release().await; continue; } @@ -288,22 +310,28 @@ pub async fn run_purge_spam( warn!(repo = %c.id, "purge-spam: repo row already gone — nothing to delete"); } Ok(n) => { - deleted += n; + summary.deleted += n; // Remove the now-orphaned on-disk bare repo (empty, so cheap) so // the DB row and disk stay consistent. Belt-and-suspenders: assert // the resolved path is canonically INSIDE repos_dir before any // remove_dir_all, so a symlinked slug dir or any residual traversal // can never delete outside the repo root (the name is already // validated at selection; this guards the destructive op itself). + // A disk-removal failure is counted separately, NOT folded into the + // deleted total, so the summary never reports a clean success while + // an on-disk dir survives (DB/disk drift the operator must fix). let path = store::repo_disk_path(&repos_dir, &c.owner_did, &c.name); if !path_within(&path, &repos_dir) { warn!(repo = %c.id, path = %path.display(), "purge-spam: on-disk path escapes repos_dir — refusing to remove"); + summary.disk_failed += 1; } else if let Err(e) = std::fs::remove_dir_all(&path) { warn!(repo = %c.id, path = %path.display(), err = %e, "purge-spam: deleted DB row but could not remove on-disk repo dir"); + summary.disk_failed += 1; + } else { + info!(repo = %c.id, rows = n, "purge-spam: deleted repo row + on-disk dir"); } - info!(repo = %c.id, rows = n, "purge-spam: deleted repo row + on-disk dir"); } Err(e) => { warn!(repo = %c.id, err = %e, "purge-spam: failed to delete repo row — continuing"); @@ -312,10 +340,10 @@ pub async fn run_purge_spam( guard.release().await; } println!( - "purge-spam: deleted {deleted} repo row(s), skipped {} (no longer empty), {skipped_locked} (locked by a live writer).", - to_skip.len() + "purge-spam: deleted {} repo row(s); skipped {} (no longer empty), {} (locked by a live writer); {} on-disk removal(s) failed.", + summary.deleted, summary.skipped_not_empty, summary.skipped_locked, summary.disk_failed ); - Ok(()) + Ok(summary) } #[cfg(test)] @@ -678,6 +706,46 @@ mod db_tests { ); } + // U5 (M6): a repo whose DB row is deleted but whose on-disk removal FAILS must + // be counted in `disk_failed`, never folded into a clean "deleted" success — + // else the summary reports success while the on-disk dir survives (DB/disk + // drift). Forces the failure by making the parent (slug) dir read-only. + #[sqlx::test] + async fn disk_removal_failure_is_counted_not_reported_as_success(pool: PgPool) { + use std::os::unix::fs::PermissionsExt; + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let empty = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&empty).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &empty.owner_did, &empty.name); + store::init_bare(&path).unwrap(); + + // Read-only parent (slug) dir: remove_dir_all cannot unlink the repo dir. + let slug_dir = path.parent().unwrap().to_path_buf(); + let orig = std::fs::metadata(&slug_dir).unwrap().permissions(); + std::fs::set_permissions(&slug_dir, std::fs::Permissions::from_mode(0o555)).unwrap(); + + let repo_store = test_store(tmp.path(), &pool); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true).await.unwrap(); + + // Restore perms so TempDir cleanup works regardless of assertion outcome. + std::fs::set_permissions(&slug_dir, orig).unwrap(); + + assert_eq!(summary.deleted, 1, "the DB row was deleted"); + assert_eq!( + summary.disk_failed, 1, + "a failed on-disk removal must be counted, not reported as clean success" + ); + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none(), + "DB row is gone (delete succeeded)" + ); + assert!(path.exists(), "on-disk dir survived the failed removal (drift)"); + } + // U1 (M3): a burst-owned row whose NAME traverses out of repos_dir must never // cause a delete OUTSIDE repos_dir. Adversarial must-not: a real empty bare repo // planted as a "victim" beside repos_dir is reachable from repos_dir// via diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index ff53a927..90281611 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -608,7 +608,9 @@ async fn run_admin_command(command: config::Command, config: &Config) -> Result< None, db.pool().clone(), ); - admin::run_purge_spam(&db, &repo_store, &config.repos_dir, execute).await + admin::run_purge_spam(&db, &repo_store, &config.repos_dir, execute) + .await + .map(|_| ()) } } } From cecca67f20c89b71f458cde602b7157fac0b6aa0 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:22:32 -0500 Subject: [PATCH 13/61] fix(node): warn on unparseable rate-limit env vars instead of silently defaulting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GITLAWB_CREATE/WRITE/PUSH_RATE_LIMIT parsed with .ok().and_then(parse).unwrap_or, so a typo (e.g. 5O with a letter O) silently reverted to the default and left the operator believing a stricter cap was in force. Route all three through rate_limit_from_env, which distinguishes absent/empty (default, silent) from present-but-unparseable (default, WARN). Pure resolve_rate_limit is unit-tested across absent/empty/valid/zero/unparseable — 0 stays valid (its own ==0 warn handles the disable case). --- crates/gitlawb-node/src/main.rs | 80 ++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 12 deletions(-) diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 90281611..e8824ca2 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -301,10 +301,7 @@ async fn main() -> Result<()> { // capped regardless of how many identities it mints. Sized well above any // legitimate per-IP creation rate; GITLAWB_CREATE_RATE_LIMIT overrides, 0 // disables. Bounded key set — the key is a client-influenced IP. - let create_limit = std::env::var("GITLAWB_CREATE_RATE_LIMIT") - .ok() - .and_then(|v| v.trim().parse::().ok()) - .unwrap_or(120); + let create_limit = rate_limit_from_env("GITLAWB_CREATE_RATE_LIMIT", 120); let create_ip_rate_limiter = rate_limit::RateLimiter::new_bounded( create_limit, std::time::Duration::from_secs(3600), @@ -322,10 +319,7 @@ async fn main() -> Result<()> { // any legitimate per-IP write rate — real agent automation makes many small // writes per hour. GITLAWB_WRITE_RATE_LIMIT overrides; 0 disables. Bounded // key set — the key is a client-influenced IP. - let write_limit = std::env::var("GITLAWB_WRITE_RATE_LIMIT") - .ok() - .and_then(|v| v.trim().parse::().ok()) - .unwrap_or(600); + let write_limit = rate_limit_from_env("GITLAWB_WRITE_RATE_LIMIT", 600); let write_rate_limiter = rate_limit::RateLimiter::new_bounded( write_limit, std::time::Duration::from_secs(3600), @@ -340,10 +334,7 @@ async fn main() -> Result<()> { // for heavy agent automation while still stopping flood traffic (the June // 2026 attack pushed several times per second per IP). GITLAWB_PUSH_RATE_LIMIT // overrides; 0 disables. Bounded key set — the key is a client-influenced IP. - let push_limit = std::env::var("GITLAWB_PUSH_RATE_LIMIT") - .ok() - .and_then(|v| v.trim().parse::().ok()) - .unwrap_or(600); + let push_limit = rate_limit_from_env("GITLAWB_PUSH_RATE_LIMIT", 600); let push_rate_limiter = rate_limit::RateLimiter::new_bounded( push_limit, std::time::Duration::from_secs(3600), @@ -588,6 +579,37 @@ async fn main() -> Result<()> { Ok(()) } +/// Pure resolver for a usize rate-limit env value. Returns `(resolved, invalid)`: +/// an absent or empty value resolves to `default` and is NOT invalid; a non-empty +/// value that fails to parse resolves to `default` and IS invalid (so the caller +/// warns rather than silently disabling the intended, usually stricter, brake). +fn resolve_rate_limit(raw: Option<&str>, default: usize) -> (usize, bool) { + match raw.map(str::trim) { + None | Some("") => (default, false), + Some(t) => match t.parse::() { + Ok(n) => (n, false), + Err(_) => (default, true), + }, + } +} + +/// Read a rate-limit env var, warning on a present-but-unparseable value. A typo +/// like `GITLAWB_WRITE_RATE_LIMIT=5O` must not silently revert to the default and +/// leave the operator believing a stricter cap is in force. +fn rate_limit_from_env(var: &str, default: usize) -> usize { + let raw = std::env::var(var).ok(); + let (value, invalid) = resolve_rate_limit(raw.as_deref(), default); + if invalid { + tracing::warn!( + var, + value = raw.as_deref().unwrap_or(""), + default, + "invalid rate-limit value — using default; the intended brake is NOT applied" + ); + } + value +} + /// Dispatch an admin subcommand. Connects the database directly (no server, no /// p2p, no metrics) and runs the requested tool to completion, then exits. async fn run_admin_command(command: config::Command, config: &Config) -> Result<()> { @@ -1075,6 +1097,40 @@ fn load_or_create_keypair(config: &Config) -> Result { } } +#[cfg(test)] +mod rate_limit_env_tests { + use super::resolve_rate_limit; + + #[test] + fn absent_uses_default_without_warning() { + assert_eq!(resolve_rate_limit(None, 600), (600, false)); + } + + #[test] + fn empty_or_whitespace_uses_default_without_warning() { + assert_eq!(resolve_rate_limit(Some(""), 600), (600, false)); + assert_eq!(resolve_rate_limit(Some(" "), 600), (600, false)); + } + + #[test] + fn valid_value_parses() { + assert_eq!(resolve_rate_limit(Some("50"), 600), (50, false)); + assert_eq!(resolve_rate_limit(Some(" 50 "), 600), (50, false)); + // 0 is a VALID value (disables the brake); the ==0 warning is handled + // separately at the call site. It must NOT be flagged invalid here. + assert_eq!(resolve_rate_limit(Some("0"), 600), (0, false)); + } + + #[test] + fn present_but_unparseable_defaults_and_flags_invalid() { + // The L8 case: a fat-fingered stricter cap ("5O", letter O) must default + // AND be flagged so the caller warns, not silently disable the brake. + assert_eq!(resolve_rate_limit(Some("5O"), 600), (600, true)); + assert_eq!(resolve_rate_limit(Some("abc"), 600), (600, true)); + assert_eq!(resolve_rate_limit(Some("-1"), 600), (600, true)); + } +} + #[cfg(test)] mod gossip_ssrf_tests { use super::ping_peer_health; From f4d5bec549262ba89f662cb5329ab4bd6d7ed03e Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:23:31 -0500 Subject: [PATCH 14/61] docs(node): document GITLAWB_IPFS_API/TIGRIS_BUCKET/METRICS_ADDR/SHUTDOWN_GRACE_SECS in .env.example Four config knobs read in config.rs were absent from .env.example, so operators had no template for them. Add each in its matching section with the config default (INV-24: operator docs match the config the code reads, in the same PR that touches the file). --- .env.example | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.env.example b/.env.example index 5987a527..e4593025 100644 --- a/.env.example +++ b/.env.example @@ -13,9 +13,19 @@ GITLAWB_PUBLIC_URL=https://your-node.example.com # ── Server ──────────────────────────────────────────────────────────────── GITLAWB_HOST=0.0.0.0 GITLAWB_PORT=7545 +# Optional address to bind a Prometheus /metrics exposition endpoint on (e.g. +# 127.0.0.1:9091). Leave empty (default) to disable. Bind to localhost or a +# private interface — the metrics endpoint is unauthenticated. +GITLAWB_METRICS_ADDR= +# Max seconds to wait for in-flight requests to drain on shutdown before the +# server returns 503 to anything still in flight and exits. Default: 30. +GITLAWB_SHUTDOWN_GRACE_SECS=30 # ── Storage ─────────────────────────────────────────────────────────────── GITLAWB_REPOS_DIR=/data/repos +# Tigris (S3-compatible) bucket for repo storage. Leave empty (default) to +# disable Tigris and use local-only storage. +GITLAWB_TIGRIS_BUCKET= # PostgreSQL connection URL. Required. # When using the bundled docker-compose, this is wired automatically. @@ -39,6 +49,9 @@ GITLAWB_DB_RETRY_INITIAL_SECS=5 GITLAWB_DB_RETRY_MAX_SECS=60 # ── IPFS pinning (Pinata) ───────────────────────────────────────────────── +# URL of a local IPFS/Kubo node HTTP API (e.g. http://127.0.0.1:5001). Leave +# empty (default) to disable local IPFS. +GITLAWB_IPFS_API= # Get a JWT at https://app.pinata.cloud/developers/api-keys GITLAWB_PINATA_JWT= GITLAWB_PINATA_UPLOAD_URL=https://uploads.pinata.cloud/v3/files From 6da0cf5dcfcde4eb3a06e77bd37f312721f4f289 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:26:29 -0500 Subject: [PATCH 15/61] test(node): exercise the dry-run guard with a real on-disk candidate present MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dry_run_deletes_nothing ran against an empty repos_dir, so select_spam_candidates found nothing and run_purge_spam hit the no-candidate early return — the if !execute { return } guard was never exercised with candidates present (a vacuous guard, INV-21). Materialize a real empty bare repo so a genuine candidate exists, then assert dry-run leaves both the row and the on-disk dir. Verified load-bearing: removing the !execute guard turns the test RED (dry-run deletes). --- crates/gitlawb-node/src/admin.rs | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/admin.rs b/crates/gitlawb-node/src/admin.rs index 2ab52529..005645ca 100644 --- a/crates/gitlawb-node/src/admin.rs +++ b/crates/gitlawb-node/src/admin.rs @@ -576,13 +576,39 @@ mod db_tests { let before = count_rows(&db).await; assert_eq!(before, 5); - // Empty repos dir: no repo exists on disk, so nothing is provably empty. + // Materialize a REAL empty bare repo for the empty target so it is a genuine + // purge candidate. Without an on-disk candidate, run_purge_spam hits the + // no-candidate early return and the `if !execute { return }` guard is never + // exercised with candidates present — the L10 gap this test now closes. let tmp = tempfile::TempDir::new().unwrap(); + let target_dir = + store::repo_disk_path(tmp.path(), SPAM_BURST_TARGET_DID, "spam1"); + store::init_bare(&target_dir).unwrap(); + assert_eq!( + store::list_refs(&target_dir).unwrap().len(), + 0, + "precondition: the candidate is a real empty bare repo" + ); + let store = test_store(tmp.path(), &pool); - run_purge_spam(&db, &store, tmp.path(), false).await.unwrap(); + let summary = run_purge_spam(&db, &store, tmp.path(), false).await.unwrap(); + // A candidate existed, but dry-run deletes nothing — DB row and on-disk dir + // both survive. RED if the `if !execute` guard is removed. + assert_eq!(summary.deleted, 0, "dry-run must delete no rows"); let after = count_rows(&db).await; assert_eq!(after, before, "dry-run must not delete any repo rows"); + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_some(), + "the candidate row must survive a dry-run" + ); + assert!( + target_dir.exists(), + "dry-run must not remove the on-disk repo dir" + ); } // The DB accessor lists exactly the target DID's rows (exact owner match), and From ffdc1672b592f7c7892794175e646c4700474bd5 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:27:00 -0500 Subject: [PATCH 16/61] style(node): cargo fmt for the #196 fix set --- crates/gitlawb-node/src/admin.rs | 41 +++++++++++++++++------ crates/gitlawb-node/src/db/mod.rs | 25 +++++++++++--- crates/gitlawb-node/src/git/repo_store.rs | 6 +++- crates/gitlawb-node/src/main.rs | 7 ++-- 4 files changed, 58 insertions(+), 21 deletions(-) diff --git a/crates/gitlawb-node/src/admin.rs b/crates/gitlawb-node/src/admin.rs index 005645ca..a03e93ed 100644 --- a/crates/gitlawb-node/src/admin.rs +++ b/crates/gitlawb-node/src/admin.rs @@ -581,8 +581,7 @@ mod db_tests { // no-candidate early return and the `if !execute { return }` guard is never // exercised with candidates present — the L10 gap this test now closes. let tmp = tempfile::TempDir::new().unwrap(); - let target_dir = - store::repo_disk_path(tmp.path(), SPAM_BURST_TARGET_DID, "spam1"); + let target_dir = store::repo_disk_path(tmp.path(), SPAM_BURST_TARGET_DID, "spam1"); store::init_bare(&target_dir).unwrap(); assert_eq!( store::list_refs(&target_dir).unwrap().len(), @@ -591,7 +590,9 @@ mod db_tests { ); let store = test_store(tmp.path(), &pool); - let summary = run_purge_spam(&db, &store, tmp.path(), false).await.unwrap(); + let summary = run_purge_spam(&db, &store, tmp.path(), false) + .await + .unwrap(); // A candidate existed, but dry-run deletes nothing — DB row and on-disk dir // both survive. RED if the `if !execute` guard is removed. @@ -641,7 +642,9 @@ mod db_tests { assert!(!store::list_refs(&refs_path).unwrap().is_empty()); let repo_store = test_store(tmp.path(), &pool); - run_purge_spam(&db, &repo_store, tmp.path(), true).await.unwrap(); + run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); // Only the empty target repo row is gone. assert!(db @@ -674,7 +677,9 @@ mod db_tests { assert!(path.exists(), "precondition: on-disk repo exists"); let repo_store = test_store(tmp.path(), &pool); - run_purge_spam(&db, &repo_store, tmp.path(), true).await.unwrap(); + run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); assert!( db.get_repo(SPAM_BURST_TARGET_DID, "spam1") @@ -707,7 +712,9 @@ mod db_tests { .unwrap() .expect("lock should be free initially"); - run_purge_spam(&db, &repo_store, tmp.path(), true).await.unwrap(); + run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); // Locked → skipped: both the row and the on-disk dir survive. assert!( @@ -722,7 +729,9 @@ mod db_tests { // Once the writer releases, the empty repo is deleted (the lock was the // only thing protecting it — baseline both ways). held.release().await; - run_purge_spam(&db, &repo_store, tmp.path(), true).await.unwrap(); + run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); assert!( db.get_repo(SPAM_BURST_TARGET_DID, "spam1") .await @@ -752,7 +761,9 @@ mod db_tests { std::fs::set_permissions(&slug_dir, std::fs::Permissions::from_mode(0o555)).unwrap(); let repo_store = test_store(tmp.path(), &pool); - let summary = run_purge_spam(&db, &repo_store, tmp.path(), true).await.unwrap(); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); // Restore perms so TempDir cleanup works regardless of assertion outcome. std::fs::set_permissions(&slug_dir, orig).unwrap(); @@ -769,7 +780,10 @@ mod db_tests { .is_none(), "DB row is gone (delete succeeded)" ); - assert!(path.exists(), "on-disk dir survived the failed removal (drift)"); + assert!( + path.exists(), + "on-disk dir survived the failed removal (drift)" + ); } // U1 (M3): a burst-owned row whose NAME traverses out of repos_dir must never @@ -806,7 +820,9 @@ mod db_tests { db.create_repo(&evil).await.unwrap(); let repo_store = test_store(&repos_dir, &pool); - run_purge_spam(&db, &repo_store, &repos_dir, true).await.unwrap(); + run_purge_spam(&db, &repo_store, &repos_dir, true) + .await + .unwrap(); assert!( victim.join("HEAD").is_file(), @@ -922,7 +938,10 @@ mod db_tests { // A genuine path inside repos_dir passes. let inside = repos_dir.join("real.git"); std::fs::create_dir_all(&inside).unwrap(); - assert!(path_within(&inside, &repos_dir), "an in-root path must pass"); + assert!( + path_within(&inside, &repos_dir), + "an in-root path must pass" + ); } /// The exclusion gate and the target scope must never overlap: if the burst diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 5b2fb79e..a963f544 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -3664,7 +3664,11 @@ mod dedup_db_tests { sqlx::query( "INSERT INTO branch_cids (repo, ref_name, sha, cid, node_did, updated_at) VALUES ($1, 'refs/heads/main', '1', 'cid', 'n', '2026-01-01T00:00:00Z')", - ).bind(&slug).execute(db.pool()).await.unwrap(); + ) + .bind(&slug) + .execute(db.pool()) + .await + .unwrap(); sqlx::query( "INSERT INTO pull_requests (id, repo_id, number, title, author_did, source_branch, target_branch, created_at, updated_at) VALUES ('pr1', 'rid-cascade', 1, 't', 'a', 'b', 'main', '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z')", @@ -3672,7 +3676,10 @@ mod dedup_db_tests { sqlx::query( "INSERT INTO pr_reviews (id, pr_id, reviewer_did, status, created_at) VALUES ('rev1', 'pr1', 'r', 'approved', '2026-01-01T00:00:00Z')", - ).execute(db.pool()).await.unwrap(); + ) + .execute(db.pool()) + .await + .unwrap(); let removed = db.delete_repo_by_id("rid-cascade").await.unwrap(); assert_eq!(removed, 1, "parent repo row deleted"); @@ -3685,7 +3692,12 @@ mod dedup_db_tests { .unwrap() } assert_eq!( - count(&db, "SELECT COUNT(*) FROM ref_certificates WHERE repo_id=$1", "rid-cascade").await, + count( + &db, + "SELECT COUNT(*) FROM ref_certificates WHERE repo_id=$1", + "rid-cascade" + ) + .await, 0, "repo_id-keyed child (ref_certificates) must be deleted" ); @@ -3695,7 +3707,12 @@ mod dedup_db_tests { "slug-keyed child (branch_cids) must be deleted" ); assert_eq!( - count(&db, "SELECT COUNT(*) FROM pull_requests WHERE repo_id=$1", "rid-cascade").await, + count( + &db, + "SELECT COUNT(*) FROM pull_requests WHERE repo_id=$1", + "rid-cascade" + ) + .await, 0, "pull_requests must be deleted" ); diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 669d8d51..5ab8f270 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -224,7 +224,11 @@ impl RepoStore { ) -> Result> { let (owner_slug, _local) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); - let mut conn = self.pool.acquire().await.context("acquiring lock connection")?; + let mut conn = self + .pool + .acquire() + .await + .context("acquiring lock connection")?; let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") .bind(lock_key) .fetch_one(&mut *conn) diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index e8824ca2..2fe8164f 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -625,11 +625,8 @@ async fn run_admin_command(command: config::Command, config: &Config) -> Result< .context("connecting to database for purge-spam")?; // Lock-only RepoStore (no Tigris): purge takes the per-repo advisory // lock to exclude a live push, but never downloads/uploads archives. - let repo_store = git::repo_store::RepoStore::new( - config.repos_dir.clone(), - None, - db.pool().clone(), - ); + let repo_store = + git::repo_store::RepoStore::new(config.repos_dir.clone(), None, db.pool().clone()); admin::run_purge_spam(&db, &repo_store, &config.repos_dir, execute) .await .map(|_| ()) From 010c625f936d25dfd0d5ef0b74cdbb1482dbba09 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 16:43:59 -0500 Subject: [PATCH 17/61] fix(node): pin the writer's advisory-lock connection in RepoWriteGuard acquire_write took the session-scoped advisory lock through the shared pool and returned the lock-owning connection to the pool, so release unlocked on a different connection (silent no-op) and a concurrent try_lock_repo could be handed the idle lock-owning connection and reentrantly re-grab the lock, defeating the purge tool's mutual exclusion. Pin a dedicated PoolConnection for the guard's lifetime and unlock on it, mirroring RepoLockGuard. Load-bearing contention test (single-connection pool to force the reentrant grab deterministic): RED (Ok(Some)) before, GREEN after. --- crates/gitlawb-node/src/git/repo_store.rs | 89 +++++++++++++++++++++-- 1 file changed, 83 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 5ab8f270..ed691960 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -157,13 +157,26 @@ impl RepoStore { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); + // Pin a dedicated connection for the guard's whole lifetime so the + // session-scoped advisory lock lives on ONE connection and `release` + // unlocks it on that SAME connection. A plain `&self.pool` query returns + // its connection to the pool between calls, which (a) leaves the lock on + // an idle connection `release`'s unlock never reaches, and (b) lets + // `try_lock_repo`'s `pool.acquire()` be handed that idle connection and + // reentrantly re-grab the lock. Mirrors `RepoLockGuard`. + let mut conn = self + .pool + .acquire() + .await + .context("acquiring write-lock connection")?; + // Acquire Postgres advisory lock with retry using pg_try_advisory_lock // to avoid blocking indefinitely on stale locks from crashed connections. let mut acquired = false; for attempt in 0..60 { let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") .bind(lock_key) - .fetch_one(&self.pool) + .fetch_one(&mut *conn) .await .context("trying advisory lock")?; if row.0 { @@ -202,7 +215,7 @@ impl RepoStore { repo_name: repo_name.to_string(), local_path, lock_key, - pool: self.pool.clone(), + conn, tigris: self.tigris.clone(), }) } @@ -387,7 +400,11 @@ pub struct RepoWriteGuard { repo_name: String, pub local_path: PathBuf, lock_key: i64, - pool: PgPool, + /// Dedicated pooled connection the advisory lock lives on. Held for the + /// guard's lifetime so `release` unlocks on the SAME session that locked, + /// and so a concurrent `try_lock_repo` cannot be handed this connection and + /// reentrantly re-grab the lock. + conn: sqlx::pool::PoolConnection, tigris: Option, } @@ -402,7 +419,7 @@ impl RepoWriteGuard { /// half-applied or otherwise inconsistent repo would propagate corruption to /// Tigris (and to every node that later downloads it). The lock is always /// released regardless, to avoid stale locks blocking future writes. - pub async fn release(self, success: bool) { + pub async fn release(mut self, success: bool) { // Upload to Tigris only on success. if success { if let Some(ref tigris) = self.tigris { @@ -417,10 +434,11 @@ impl RepoWriteGuard { warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo"); } - // Release advisory lock + // Release the advisory lock on the SAME connection that took it, then + // the connection returns to the pool on drop. let _ = sqlx::query("SELECT pg_advisory_unlock($1)") .bind(self.lock_key) - .execute(&self.pool) + .execute(&mut *self.conn) .await; } } @@ -614,4 +632,63 @@ mod tests { ); } } + + // ── advisory-lock mutual exclusion (P1a) ──────────────────────────────── + + // A live `acquire_write` guard must exclude a concurrent `try_lock_repo` + // (the purge path) on the same repo. This is load-bearing for the purge + // tool's M4 mutual exclusion. + // + // The pool MUST be single-connection. `try_lock_repo`'s own return is the + // only observable that exposes the bug (it acquires through the pool), and + // it is only deterministic when the writer's connection is the ONLY one, so + // `try_lock_repo`'s `pool.acquire()` is forced onto it: + // * Pre-fix, `acquire_write` locks via `.fetch_one(&self.pool)` and returns + // the lock-owning connection to the pool. `try_lock_repo` re-acquires + // that same connection and `pg_try_advisory_lock` re-locks it + // reentrantly -> Ok(Some) (RED). (On a multi-connection pool the reentrant + // grab is non-deterministic: `acquire` may hand back a different idle + // connection, so the bug hides — verified by execution.) + // * Post-fix, `acquire_write` pins the connection, so `try_lock_repo`'s + // `pool.acquire()` cannot get one and times out -> Err. Either way the + // fix guarantees try_lock does NOT reentrantly grab the held lock. + #[sqlx::test] + async fn acquire_write_guard_blocks_try_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + let pool = pool_opts + .max_connections(1) + .acquire_timeout(std::time::Duration::from_secs(3)) + .connect_with(connect_opts) + .await + .unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(tmp.path().to_path_buf(), pool); + let owner = "did:key:z6MkWriterLockOwnerAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "locktest"; + + // A live writer holds the per-repo advisory lock. + let guard = store.acquire_write(owner, name).await.unwrap(); + + // The purge path must NOT reentrantly acquire the same lock while held. + // Pre-fix: Ok(Some) (reentrant grab). Post-fix: Err (connection pinned). + // The invariant is "not Ok(Some)". + let contended = store.try_lock_repo(owner, name).await; + assert!( + !matches!(contended, Ok(Some(_))), + "try_lock_repo must not reentrantly acquire a lock a live acquire_write \ + guard holds (got Ok(Some) — the reentrant-grab bug)" + ); + + // After the writer releases (on its pinned connection), the lock is free + // and the single connection is back in the pool. + guard.release(true).await; + let after = store.try_lock_repo(owner, name).await.unwrap(); + assert!( + after.is_some(), + "lock must be free once the writer releases it" + ); + after.unwrap().release().await; + } } From c192c1f18d4186024ddd136c146f33b11f43acf2 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 16:54:48 -0500 Subject: [PATCH 18/61] refactor(node): introduce ObjectStore trait seam over TigrisClient MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract an async ObjectStore trait (exists/upload/download/delete) that TigrisClient implements, and hold Option> in RepoStore instead of Option. A dyn async trait needs async-trait (added as a direct dep; already transitively present), which E0038 would otherwise reject. Pure refactor — no behavior change; full suite stays green. Enables an in-test fake for the purge Tigris work (U3). Cargo.lock also picks up the pre-existing 0.5.0->0.5.1 version sync (#185: the committed lock was stale vs the manifests) that any cargo run regenerates. --- Cargo.lock | 11 ++++--- crates/gitlawb-node/Cargo.toml | 1 + crates/gitlawb-node/src/git/repo_store.rs | 38 +++++++++++++---------- crates/gitlawb-node/src/git/tigris.rs | 33 ++++++++++++++------ crates/gitlawb-node/src/main.rs | 4 ++- 5 files changed, 54 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d12daa43..47f2824f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3300,7 +3300,7 @@ dependencies = [ [[package]] name = "git-remote-gitlawb" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "gitlawb-core", @@ -3312,7 +3312,7 @@ dependencies = [ [[package]] name = "gitlawb-attest" -version = "0.5.0" +version = "0.5.1" dependencies = [ "base64", "ed25519-dalek", @@ -3329,7 +3329,7 @@ dependencies = [ [[package]] name = "gitlawb-core" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "base64", @@ -3356,13 +3356,14 @@ dependencies = [ [[package]] name = "gitlawb-node" -version = "0.5.0" +version = "0.5.1" dependencies = [ "alloy", "anyhow", "async-compression", "async-graphql", "async-graphql-axum", + "async-trait", "aws-config", "aws-sdk-s3", "axum", @@ -3412,7 +3413,7 @@ dependencies = [ [[package]] name = "gl" -version = "0.5.0" +version = "0.5.1" dependencies = [ "alloy", "anyhow", diff --git a/crates/gitlawb-node/Cargo.toml b/crates/gitlawb-node/Cargo.toml index 9e42c084..3e721181 100644 --- a/crates/gitlawb-node/Cargo.toml +++ b/crates/gitlawb-node/Cargo.toml @@ -12,6 +12,7 @@ path = "src/main.rs" [dependencies] gitlawb-core = { path = "../gitlawb-core" } +async-trait = "0.1" ed25519-dalek = { workspace = true } base64 = { workspace = true } tokio = { workspace = true } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index ed691960..66ef055a 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -18,17 +18,17 @@ use tokio::sync::Mutex; use tracing::{debug, info, warn}; use super::store; -use super::tigris::TigrisClient; +use super::tigris::ObjectStore; -/// Centralized repo storage: local disk cache + optional Tigris backend. +/// Centralized repo storage: local disk cache + optional object-store backend. #[derive(Clone)] pub struct RepoStore { repos_dir: PathBuf, - tigris: Option, + object_store: Option>, /// Shared Postgres pool for advisory locks. pool: PgPool, - /// Tracks repos already confirmed to exist in Tigris — avoids redundant - /// HEAD checks and background uploads for repos we've already migrated. + /// Tracks repos already confirmed to exist in the object store — avoids + /// redundant HEAD checks and background uploads for repos we've migrated. migrated: Arc>>, } @@ -37,16 +37,20 @@ impl RepoStore { pub fn for_testing(repos_dir: PathBuf, pool: PgPool) -> Self { Self { repos_dir, - tigris: None, + object_store: None, pool, migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), } } - pub fn new(repos_dir: PathBuf, tigris: Option, pool: PgPool) -> Self { + pub fn new( + repos_dir: PathBuf, + object_store: Option>, + pool: PgPool, + ) -> Self { Self { repos_dir, - tigris, + object_store, pool, migrated: Arc::new(Mutex::new(HashSet::new())), } @@ -63,7 +67,7 @@ impl RepoStore { if local_path.exists() { // Lazy migration: if Tigris is enabled and we haven't confirmed this // repo is in Tigris yet, check and upload in the background. - if let Some(ref tigris) = self.tigris { + if let Some(ref tigris) = self.object_store { let key = format!("{owner_slug}/{repo_name}"); let already_migrated = self.migrated.lock().await.contains(&key); if !already_migrated { @@ -99,7 +103,7 @@ impl RepoStore { } // Try downloading from Tigris - if let Some(ref tigris) = self.tigris { + if let Some(ref tigris) = self.object_store { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "cache miss — downloading from tigris"); tigris @@ -127,7 +131,7 @@ impl RepoStore { pub async fn acquire_fresh(&self, owner_did: &str, repo_name: &str) -> Result { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; - if let Some(ref tigris) = self.tigris { + if let Some(ref tigris) = self.object_store { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "acquire_fresh: downloading latest from tigris"); if let Err(e) = tigris.download(&owner_slug, repo_name, &local_path).await { @@ -193,7 +197,7 @@ impl RepoStore { // Always download the latest from Tigris before writing. // Local disk may be stale if another machine pushed since our last access. - if let Some(ref tigris) = self.tigris { + if let Some(ref tigris) = self.object_store { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); if let Err(e) = tigris.download(&owner_slug, repo_name, &local_path).await { @@ -216,7 +220,7 @@ impl RepoStore { local_path, lock_key, conn, - tigris: self.tigris.clone(), + object_store: self.object_store.clone(), }) } @@ -260,7 +264,7 @@ impl RepoStore { store::init_bare(&local_path).context("initializing bare repo")?; // Upload to Tigris in background - if let Some(ref tigris) = self.tigris { + if let Some(ref tigris) = self.object_store { let tigris = tigris.clone(); let owner_slug = owner_slug.clone(); let repo_name = repo_name.to_string(); @@ -278,7 +282,7 @@ impl RepoStore { /// Upload a repo to Tigris after a write operation (push, merge, fork, etc.). /// Call this after any operation that modifies the git repo on disk. pub async fn release_after_write(&self, owner_did: &str, repo_name: &str) { - if let Some(ref tigris) = self.tigris { + if let Some(ref tigris) = self.object_store { let (owner_slug, local_path) = match self.local_path(owner_did, repo_name) { Ok(p) => p, Err(e) => { @@ -405,7 +409,7 @@ pub struct RepoWriteGuard { /// and so a concurrent `try_lock_repo` cannot be handed this connection and /// reentrantly re-grab the lock. conn: sqlx::pool::PoolConnection, - tigris: Option, + object_store: Option>, } impl RepoWriteGuard { @@ -422,7 +426,7 @@ impl RepoWriteGuard { pub async fn release(mut self, success: bool) { // Upload to Tigris only on success. if success { - if let Some(ref tigris) = self.tigris { + if let Some(ref tigris) = self.object_store { if let Err(e) = tigris .upload(&self.owner_slug, &self.repo_name, &self.local_path) .await diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index ad26ddc5..c4f54fb0 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -11,6 +11,22 @@ use anyhow::{Context, Result}; use aws_sdk_s3::Client as S3Client; use tracing::{debug, info}; +/// Object storage for git bare-repo archives. The seam that lets `RepoStore` +/// (and tests) depend on storage behavior without a live bucket. `TigrisClient` +/// is the production implementation. +#[async_trait::async_trait] +pub trait ObjectStore: Send + Sync { + /// Whether an archive exists for this repo. + async fn exists(&self, owner_slug: &str, repo_name: &str) -> Result; + /// Upload a local bare repo directory as an archive. + async fn upload(&self, owner_slug: &str, repo_name: &str, local_path: &Path) -> Result<()>; + /// Download and extract a repo archive to local disk. + async fn download(&self, owner_slug: &str, repo_name: &str, local_path: &Path) -> Result<()>; + /// Delete a repo archive. (Wired into purge-spam in U3.) + #[allow(dead_code)] + async fn delete(&self, owner_slug: &str, repo_name: &str) -> Result<()>; +} + /// Wrapper around the S3 client with the configured bucket. #[derive(Clone)] pub struct TigrisClient { @@ -35,9 +51,12 @@ impl TigrisClient { fn repo_key(owner_slug: &str, repo_name: &str) -> String { format!("repos/v1/{owner_slug}/{repo_name}.tar.zst") } +} +#[async_trait::async_trait] +impl ObjectStore for TigrisClient { /// Check if a repo archive exists in Tigris. - pub async fn exists(&self, owner_slug: &str, repo_name: &str) -> Result { + async fn exists(&self, owner_slug: &str, repo_name: &str) -> Result { let key = Self::repo_key(owner_slug, repo_name); match self .s3 @@ -59,7 +78,7 @@ impl TigrisClient { } /// Upload a local bare repo directory to Tigris as a tar.zst archive. - pub async fn upload(&self, owner_slug: &str, repo_name: &str, local_path: &Path) -> Result<()> { + async fn upload(&self, owner_slug: &str, repo_name: &str, local_path: &Path) -> Result<()> { let key = Self::repo_key(owner_slug, repo_name); debug!(key = %key, path = %local_path.display(), "uploading repo to tigris"); @@ -89,12 +108,7 @@ impl TigrisClient { } /// Download a repo archive from Tigris and extract to local disk. - pub async fn download( - &self, - owner_slug: &str, - repo_name: &str, - local_path: &Path, - ) -> Result<()> { + async fn download(&self, owner_slug: &str, repo_name: &str, local_path: &Path) -> Result<()> { let key = Self::repo_key(owner_slug, repo_name); debug!(key = %key, path = %local_path.display(), "downloading repo from tigris"); @@ -128,8 +142,7 @@ impl TigrisClient { } /// Delete a repo archive from Tigris. - #[allow(dead_code)] - pub async fn delete(&self, owner_slug: &str, repo_name: &str) -> Result<()> { + async fn delete(&self, owner_slug: &str, repo_name: &str) -> Result<()> { let key = Self::repo_key(owner_slug, repo_name); self.s3 .delete_object() diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 2fe8164f..32063b9a 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -286,8 +286,10 @@ async fn main() -> Result<()> { None }; + let object_store = + tigris.map(|t| std::sync::Arc::new(t) as std::sync::Arc); let repo_store = - git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, db.pool().clone()); + git::repo_store::RepoStore::new(config.repos_dir.clone(), object_store, db.pool().clone()); // Per-DID limiter for the creation endpoints. Keyed on the authenticated // DID (attacker-varied), so bound its key set to cap memory. From 2155c958ac29edaa6d92f3dd0526a851aafa27cb Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 17:07:47 -0500 Subject: [PATCH 19/61] fix(node): make purge-spam Tigris-authoritative and archive-deleting purge-spam built its RepoStore with no object store, so on a Tigris deployment it rechecked emptiness against possibly-stale local disk (deleting a repo with live remote refs) and left the archive behind (to be re-downloaded into a later same-owner/name repo). Give the admin command the configured object store, refresh the local copy from the authoritative archive under the per-repo lock before the recheck (fail-closed on a store error via a new skipped_store_error), and delete the archive on a successful purge (failures counted in a new archive_failed, never folded into success). When no bucket is configured the None path is byte-for-byte unchanged. Three load-bearing guards verified RED->GREEN on the Tigris-blind purge: remote-refs-not-deleted, archive-deleted, fail-closed-on-unreachable. --- crates/gitlawb-node/src/admin.rs | 277 +++++++++++++++++++++- crates/gitlawb-node/src/git/repo_store.rs | 28 +++ crates/gitlawb-node/src/git/tigris.rs | 3 +- crates/gitlawb-node/src/main.rs | 33 ++- 4 files changed, 332 insertions(+), 9 deletions(-) diff --git a/crates/gitlawb-node/src/admin.rs b/crates/gitlawb-node/src/admin.rs index a03e93ed..35016424 100644 --- a/crates/gitlawb-node/src/admin.rs +++ b/crates/gitlawb-node/src/admin.rs @@ -43,9 +43,17 @@ pub struct PurgeSummary { pub skipped_not_empty: usize, /// Candidates skipped because a live writer held the per-repo lock. pub skipped_locked: usize, + /// Candidates skipped because the object store could not be consulted for the + /// authoritative emptiness recheck (fail-closed: never delete on a store + /// error rather than risk deleting a repo with live remote refs). + pub skipped_store_error: usize, /// Rows deleted whose on-disk dir removal FAILED (or was refused by the /// containment guard) — DB/disk drift the operator must reconcile. pub disk_failed: usize, + /// Rows+dirs deleted whose object-store archive removal FAILED — the archive + /// survives and could be re-downloaded into a later same-owner/name repo, so + /// this is tracked separately and never folded into a clean success. + pub archive_failed: usize, } /// A repo selected for purge, with the evidence that qualified it. @@ -297,8 +305,21 @@ pub async fn run_purge_spam( continue; } }; + // Refresh the local copy from the authoritative object store (if any) + // before the recheck: on a Tigris deployment the admin node's local disk + // can be stale, so an emptiness check against local alone could delete a + // repo that has live remote refs. Fail closed on any store error — never + // delete on an unverified view. + if let Err(e) = repo_store.refresh_from_archive(&c.owner_did, &c.name).await { + warn!(repo = %c.id, err = %e, + "purge-spam: could not consult the object store — skipped (fail-closed)"); + summary.skipped_store_error += 1; + guard.release().await; + continue; + } // Authoritative recheck UNDER the lock: a ref that landed before we locked - // makes the repo non-empty, so it must not be deleted. + // (or that the object-store refresh surfaced) makes the repo non-empty, so + // it must not be deleted. if ref_count_on_disk(&repos_dir, &c.owner_did, &c.name) != 0 { warn!(repo = %c.id, "purge-spam: repo no longer empty under lock — skipped (TOCTOU)"); summary.skipped_not_empty += 1; @@ -332,6 +353,15 @@ pub async fn run_purge_spam( } else { info!(repo = %c.id, rows = n, "purge-spam: deleted repo row + on-disk dir"); } + // Delete the object-store archive too (a no-op when no store is + // configured), else it survives and can be downloaded into a + // later repo created with the same owner/name. Counted separately + // so a surviving archive never reads as a clean success. + if let Err(e) = repo_store.delete_archive(&c.owner_did, &c.name).await { + warn!(repo = %c.id, err = %e, + "purge-spam: deleted row + dir but could not delete the object-store archive"); + summary.archive_failed += 1; + } } Err(e) => { warn!(repo = %c.id, err = %e, "purge-spam: failed to delete repo row — continuing"); @@ -340,8 +370,13 @@ pub async fn run_purge_spam( guard.release().await; } println!( - "purge-spam: deleted {} repo row(s); skipped {} (no longer empty), {} (locked by a live writer); {} on-disk removal(s) failed.", - summary.deleted, summary.skipped_not_empty, summary.skipped_locked, summary.disk_failed + "purge-spam: deleted {} repo row(s); skipped {} (no longer empty), {} (locked by a live writer), {} (object store unreachable); {} on-disk removal(s) failed, {} archive removal(s) failed.", + summary.deleted, + summary.skipped_not_empty, + summary.skipped_locked, + summary.skipped_store_error, + summary.disk_failed, + summary.archive_failed ); Ok(summary) } @@ -979,4 +1014,240 @@ mod db_tests { "an empty repo owned by a short-form excluded DID must never be a candidate" ); } + + // ── Tigris-authoritative purge (P1b) ─────────────────────────────────── + + /// In-test object store. `download` materializes a bare repo with (or + /// without) a ref so the purge tool's authoritative recheck can be driven + /// without a live bucket; error flags exercise the fail-closed and + /// archive-delete-failure paths. + struct FakeStore { + has_archive: bool, + archive_has_refs: bool, + fail_recheck: bool, + fail_delete: bool, + deleted: std::sync::Arc, + } + + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for FakeStore { + async fn exists(&self, _owner: &str, _repo: &str) -> Result { + if self.fail_recheck { + anyhow::bail!("fake object store unreachable"); + } + Ok(self.has_archive) + } + async fn upload(&self, _owner: &str, _repo: &str, _path: &Path) -> Result<()> { + Ok(()) + } + async fn download(&self, _owner: &str, _repo: &str, local_path: &Path) -> Result<()> { + if self.fail_recheck { + anyhow::bail!("fake object store unreachable"); + } + // Materialize the authoritative archive on local disk so + // ref_count_on_disk reflects remote state. + let _ = std::fs::remove_dir_all(local_path); + store::init_bare(local_path).unwrap(); + if self.archive_has_refs { + seed_one_ref(local_path); + } + Ok(()) + } + async fn delete(&self, _owner: &str, _repo: &str) -> Result<()> { + self.deleted + .store(true, std::sync::atomic::Ordering::SeqCst); + if self.fail_delete { + anyhow::bail!("fake object store delete failed"); + } + Ok(()) + } + } + + fn store_backed( + repos_dir: &Path, + pool: &PgPool, + fake: FakeStore, + ) -> crate::git::repo_store::RepoStore { + crate::git::repo_store::RepoStore::new( + repos_dir.to_path_buf(), + Some(std::sync::Arc::new(fake)), + pool.clone(), + ) + } + + fn fake( + has_archive: bool, + archive_has_refs: bool, + fail_recheck: bool, + fail_delete: bool, + ) -> (FakeStore, std::sync::Arc) { + let deleted = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + ( + FakeStore { + has_archive, + archive_has_refs, + fail_recheck, + fail_delete, + deleted: deleted.clone(), + }, + deleted, + ) + } + + // A locally-empty repo whose authoritative archive HAS refs (pushed via + // another machine) must NOT be purged on the stale-local view. Load-bearing: + // RED on the Tigris-blind purge (deletes it), GREEN once the recheck + // consults the object store. + #[sqlx::test] + async fn purge_skips_repo_with_remote_refs_when_local_empty(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-remote", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &target.owner_did, &target.name); + store::init_bare(&path).unwrap(); + assert_eq!( + store::list_refs(&path).unwrap().len(), + 0, + "local starts empty" + ); + + let (f, _deleted) = fake(true, true, false, false); + let repo_store = store_backed(tmp.path(), &pool, f); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_some(), + "a repo with live remote refs must not be purged on a stale-local view" + ); + assert_eq!(summary.deleted, 0); + assert_eq!(summary.skipped_not_empty, 1); + } + + // A genuinely-empty repo (empty archive) is deleted AND its archive removed, + // else the archive can be re-downloaded into a later same-name repo. + // Load-bearing: RED on the Tigris-blind purge (archive survives), GREEN once + // the delete wires the archive removal. + #[sqlx::test] + async fn purge_deletes_archive_on_successful_delete(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &target.owner_did, &target.name); + store::init_bare(&path).unwrap(); + + let (f, deleted) = fake(true, false, false, false); + let repo_store = store_backed(tmp.path(), &pool, f); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none(), + "a genuinely-empty repo is deleted" + ); + assert!( + deleted.load(std::sync::atomic::Ordering::SeqCst), + "the object-store archive must be deleted on a successful purge" + ); + assert_eq!(summary.deleted, 1); + assert_eq!(summary.archive_failed, 0); + } + + // An unreachable object store during the recheck must fail closed (skip), + // never delete on an unverified view. Load-bearing: RED on the Tigris-blind + // purge (deletes), GREEN once the recheck consults (and fails closed on) the + // store. + #[sqlx::test] + async fn purge_fails_closed_when_store_unreachable(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &target.owner_did, &target.name); + store::init_bare(&path).unwrap(); + + let (f, _deleted) = fake(true, false, true, false); + let repo_store = store_backed(tmp.path(), &pool, f); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_some(), + "must not delete when the object store is unreachable (fail-closed)" + ); + assert_eq!(summary.deleted, 0); + assert_eq!(summary.skipped_store_error, 1); + } + + // An archive-delete failure after the row+dir are removed is counted + // separately, never folded into a clean success. + #[sqlx::test] + async fn purge_archive_delete_failure_counted_separately(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &target.owner_did, &target.name); + store::init_bare(&path).unwrap(); + + let (f, deleted) = fake(true, false, false, true); + let repo_store = store_backed(tmp.path(), &pool, f); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none(), + "the row+dir are still deleted" + ); + assert!(deleted.load(std::sync::atomic::Ordering::SeqCst)); + assert_eq!(summary.deleted, 1); + assert_eq!( + summary.archive_failed, 1, + "a surviving archive must be counted, not reported as clean success" + ); + } + + // R7: with no object store configured (single-machine), an empty repo is + // deleted exactly as before and nothing touches an archive. + #[sqlx::test] + async fn purge_tigris_disabled_deletes_empty_unchanged(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &target.owner_did, &target.name); + store::init_bare(&path).unwrap(); + + let repo_store = test_store(tmp.path(), &pool); // Tigris = None + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!(db + .get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none()); + assert_eq!(summary.deleted, 1); + assert_eq!(summary.skipped_store_error, 0); + assert_eq!(summary.archive_failed, 0); + } } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 66ef055a..9a3a8814 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -341,6 +341,34 @@ impl RepoStore { Ok((owner_slug, local_path)) } + + /// Refresh the local copy from the authoritative object-store archive so an + /// emptiness recheck reflects remote state (used by purge-spam on a Tigris + /// deployment, where the admin node's local disk can be stale). Downloads + /// the archive over the local path when it exists; a no-op when no object + /// store is configured (single-machine). Errors propagate so the caller can + /// fail closed rather than delete on a stale-local view. + pub async fn refresh_from_archive(&self, owner_did: &str, repo_name: &str) -> Result<()> { + let Some(ref store) = self.object_store else { + return Ok(()); + }; + let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; + if store.exists(&owner_slug, repo_name).await? { + store.download(&owner_slug, repo_name, &local_path).await?; + } + Ok(()) + } + + /// Delete the object-store archive for a repo. A no-op when no object store + /// is configured. Used by purge-spam so a deleted repo's archive cannot be + /// downloaded into a later repo created with the same owner/name. + pub async fn delete_archive(&self, owner_did: &str, repo_name: &str) -> Result<()> { + let Some(ref store) = self.object_store else { + return Ok(()); + }; + let (owner_slug, _local_path) = self.local_path(owner_did, repo_name)?; + store.delete(&owner_slug, repo_name).await + } } /// Strict allowlist validator for `owner_did` and `repo_name`. diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index c4f54fb0..9b447696 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -22,8 +22,7 @@ pub trait ObjectStore: Send + Sync { async fn upload(&self, owner_slug: &str, repo_name: &str, local_path: &Path) -> Result<()>; /// Download and extract a repo archive to local disk. async fn download(&self, owner_slug: &str, repo_name: &str, local_path: &Path) -> Result<()>; - /// Delete a repo archive. (Wired into purge-spam in U3.) - #[allow(dead_code)] + /// Delete a repo archive. async fn delete(&self, owner_slug: &str, repo_name: &str) -> Result<()>; } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 32063b9a..f85eaded 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -625,10 +625,35 @@ async fn run_admin_command(command: config::Command, config: &Config) -> Result< ) .await .context("connecting to database for purge-spam")?; - // Lock-only RepoStore (no Tigris): purge takes the per-repo advisory - // lock to exclude a live push, but never downloads/uploads archives. - let repo_store = - git::repo_store::RepoStore::new(config.repos_dir.clone(), None, db.pool().clone()); + // Build the object store so purge is authoritative against remote + // state: the emptiness recheck consults the archive (not just + // possibly-stale local disk) and a successful purge deletes the + // archive too. When no bucket is configured this stays None and purge + // is local-only, exactly as before. + let object_store: Option> = if config + .tigris_bucket + .is_empty() + { + None + } else { + match git::tigris::TigrisClient::new(&config.tigris_bucket).await { + Ok(client) => Some(std::sync::Arc::new(client)), + Err(e) => { + // Fail closed: without the configured store we cannot + // do the authoritative recheck, and a local-only purge + // on a Tigris deployment is the stale-view delete this + // guards against. Refuse to run rather than fall back. + anyhow::bail!( + "purge-spam: GITLAWB_TIGRIS_BUCKET is set but the object-store client failed to initialize ({e}); refusing to run a local-only purge on a Tigris deployment" + ); + } + } + }; + let repo_store = git::repo_store::RepoStore::new( + config.repos_dir.clone(), + object_store, + db.pool().clone(), + ); admin::run_purge_spam(&db, &repo_store, &config.repos_dir, execute) .await .map(|_| ()) From 8842272560d36291a274592ced9c0260e56387bd Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 17:37:43 -0500 Subject: [PATCH 20/61] fix(review): don't hold/leak the pool connection in acquire_write Two regressions the connection-pinning fix (010c625) introduced, caught in pre-push review: - Pool pressure: acquire_write held its pinned connection through the whole 60s lock-retry backoff; pre-change the retry probed via the pool and returned the connection between attempts. Under a same-repo write burst every spinner now held a pool connection through its sleeps and could starve the shared pool. Restructure the loop to return the connection between FAILED attempts and keep it only on the winning one. - Lock leak: on the object-store download-error early-return the pinned connection dropped back to the pool with the advisory lock still held (no guard Drop runs pre-construction, and sqlx does not release session locks on connection return), blocking every future write to that repo until the pool reaped the connection. Unlock on the pinned connection before bailing. Add acquire_write_guard_excludes_try_lock_across_connections: a multi-connection test proving the exclusion holds across sessions (the production topology), which also catches a pin-but-don't-lock regression the single-connection test cannot. --- crates/gitlawb-node/src/git/repo_store.rs | 128 +++++++++++++++------- 1 file changed, 89 insertions(+), 39 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 9a3a8814..b1f64b50 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -161,54 +161,73 @@ impl RepoStore { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); - // Pin a dedicated connection for the guard's whole lifetime so the - // session-scoped advisory lock lives on ONE connection and `release` - // unlocks it on that SAME connection. A plain `&self.pool` query returns - // its connection to the pool between calls, which (a) leaves the lock on - // an idle connection `release`'s unlock never reaches, and (b) lets - // `try_lock_repo`'s `pool.acquire()` be handed that idle connection and - // reentrantly re-grab the lock. Mirrors `RepoLockGuard`. - let mut conn = self - .pool - .acquire() - .await - .context("acquiring write-lock connection")?; - - // Acquire Postgres advisory lock with retry using pg_try_advisory_lock - // to avoid blocking indefinitely on stale locks from crashed connections. - let mut acquired = false; - for attempt in 0..60 { - let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") - .bind(lock_key) - .fetch_one(&mut *conn) - .await - .context("trying advisory lock")?; - if row.0 { - acquired = true; - break; + // Acquire the advisory lock on a DEDICATED connection, held for the + // guard's whole lifetime so the session-scoped lock lives on ONE + // connection and `release` unlocks it on that SAME connection, and so a + // concurrent `try_lock_repo`'s `pool.acquire()` cannot be handed the + // lock-owning connection and reentrantly re-grab it. Mirrors + // `RepoLockGuard`. Use pg_try_advisory_lock with retry to avoid blocking + // indefinitely on a stale lock from a crashed connection. + // + // Between FAILED attempts the connection is returned to the pool (the + // `drop` below) so a writer spinning on a contended repo does NOT hold a + // pool connection through the retry backoff — holding one per spinner + // would starve the shared pool under a same-repo write burst. Only the + // WINNING attempt keeps its connection (the lock lives on it). + let mut conn = { + let mut acquired = None; + for attempt in 0..60 { + let mut c = self + .pool + .acquire() + .await + .context("acquiring write-lock connection")?; + let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(lock_key) + .fetch_one(&mut *c) + .await + .context("trying advisory lock")?; + if row.0 { + acquired = Some(c); + break; + } + drop(c); + if attempt < 59 { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } } - if attempt < 59 { - tokio::time::sleep(std::time::Duration::from_secs(1)).await; + match acquired { + Some(c) => c, + None => anyhow::bail!( + "could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}" + ), } - } - if !acquired { - anyhow::bail!("could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}"); - } + }; - // Always download the latest from Tigris before writing. + // Always download the latest from the object store before writing. // Local disk may be stale if another machine pushed since our last access. - if let Some(ref tigris) = self.object_store { - if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { - debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); - if let Err(e) = tigris.download(&owner_slug, repo_name, &local_path).await { + if let Some(ref store) = self.object_store { + if store.exists(&owner_slug, repo_name).await.unwrap_or(false) { + debug!(repo = %repo_name, "write acquire: downloading latest from object store"); + if let Err(e) = store.download(&owner_slug, repo_name, &local_path).await { // Same self-healing fallback as acquire_fresh: a corrupt/unreadable - // Tigris archive must not block a write when a valid local copy + // archive must not block a write when a valid local copy // exists — release(success) will re-upload a good archive. if local_path.exists() { warn!(repo = %repo_name, err = %e, - "write acquire: tigris download failed — falling back to local copy"); + "write acquire: object-store download failed — falling back to local copy"); } else { - return Err(e).context("downloading repo from tigris for write"); + // Unlock on the pinned connection before bailing, else the + // advisory lock orphans on it: the guard is not constructed + // yet (so its release never runs), and sqlx does not release + // session locks when a connection returns to the pool — the + // lock would block every future write to this repo until the + // pool reaps the connection. + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(lock_key) + .execute(&mut *conn) + .await; + return Err(e).context("downloading repo from object store for write"); } } } @@ -723,4 +742,35 @@ mod tests { ); after.unwrap().release().await; } + + // The exclusion is real ACROSS sessions, not just the reentrant + // same-connection case: on a multi-connection pool the writer pins its + // connection, so `try_lock_repo`'s `pool.acquire()` gets a DIFFERENT + // connection whose `pg_try_advisory_lock` correctly observes the lock held + // -> Ok(None). This is the actual production topology (writer and purge on + // different sessions), and it also catches a pin-but-don't-actually-lock + // regression that the single-connection test (which passes via a + // pool-exhaustion Err) cannot distinguish from a real held lock. + #[sqlx::test] + async fn acquire_write_guard_excludes_try_lock_across_connections(pool: PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(tmp.path().to_path_buf(), pool); + let owner = "did:key:z6MkCrossConnExclusionAAAAAAAAAAAAAAAAAAAAA"; + let name = "xconn"; + + let guard = store.acquire_write(owner, name).await.unwrap(); + + // A second, distinct pooled connection must see the lock held. + let contended = store.try_lock_repo(owner, name).await.unwrap(); + assert!( + contended.is_none(), + "a live acquire_write guard must exclude try_lock_repo on a different \ + connection (Ok(None)); Ok(Some) would mean the lock is not held" + ); + + guard.release(true).await; + let after = store.try_lock_repo(owner, name).await.unwrap(); + assert!(after.is_some(), "lock is free after release"); + after.unwrap().release().await; + } } From 1d011beeeb9f51c0a9665da7a3a2bc7fd3d4d73d Mon Sep 17 00:00:00 2001 From: t Date: Thu, 16 Jul 2026 09:38:54 -0500 Subject: [PATCH 21/61] fix(node): make advisory-lock guards drop-safe and reorder acquire_write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RepoWriteGuard and RepoLockGuard now hold their pooled connection as an Option: release() takes it (returns it to the pool) while a Drop impl closes the connection when release() was never reached — an early ?, a panic, or the handler future cancelled on client disconnect. Closing the connection ends the Postgres session so the server frees the session-scoped advisory lock, instead of returning a lock-holding connection to the pool where it would block every later write to that repo until the connection is reaped (up to ~30min at the pool's default max_lifetime). acquire_write now wraps the winning connection in the guard BEFORE the freshness download, so a cancellation during that await is covered by the guard's Drop rather than orphaning the lock on a bare connection. Tests observe the leak by disabling the pool's ambient reaping (which otherwise masks it ~1.8s after drop), so the lock stays held until the fix's own unlock. RED before / GREEN after for both guards and the acquire-side cancel; each protection line proven load-bearing by revert (INV-21). --- crates/gitlawb-node/src/git/repo_store.rs | 347 +++++++++++++++++++--- 1 file changed, 309 insertions(+), 38 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index b1f64b50..04e5dfc3 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -174,7 +174,7 @@ impl RepoStore { // pool connection through the retry backoff — holding one per spinner // would starve the shared pool under a same-repo write burst. Only the // WINNING attempt keeps its connection (the lock lives on it). - let mut conn = { + let conn = { let mut acquired = None; for attempt in 0..60 { let mut c = self @@ -204,43 +204,51 @@ impl RepoStore { } }; + // Wrap the winning connection in the guard IMMEDIATELY, before any + // further await. The download below can be cancelled (the caller's + // future dropped mid-op) or fail; once the connection lives inside the + // guard, its `Drop` frees the advisory lock on every such exit. Before + // this ordering the lock orphaned on a bare connection returned to the + // pool holding it. (KTD-2.) + let guard = RepoWriteGuard { + owner_slug, + repo_name: repo_name.to_string(), + local_path, + lock_key, + conn: Some(conn), + object_store: self.object_store.clone(), + }; + // Always download the latest from the object store before writing. // Local disk may be stale if another machine pushed since our last access. if let Some(ref store) = self.object_store { - if store.exists(&owner_slug, repo_name).await.unwrap_or(false) { - debug!(repo = %repo_name, "write acquire: downloading latest from object store"); - if let Err(e) = store.download(&owner_slug, repo_name, &local_path).await { + if store + .exists(&guard.owner_slug, &guard.repo_name) + .await + .unwrap_or(false) + { + debug!(repo = %guard.repo_name, "write acquire: downloading latest from object store"); + if let Err(e) = store + .download(&guard.owner_slug, &guard.repo_name, &guard.local_path) + .await + { // Same self-healing fallback as acquire_fresh: a corrupt/unreadable // archive must not block a write when a valid local copy // exists — release(success) will re-upload a good archive. - if local_path.exists() { - warn!(repo = %repo_name, err = %e, + if guard.local_path.exists() { + warn!(repo = %guard.repo_name, err = %e, "write acquire: object-store download failed — falling back to local copy"); } else { - // Unlock on the pinned connection before bailing, else the - // advisory lock orphans on it: the guard is not constructed - // yet (so its release never runs), and sqlx does not release - // session locks when a connection returns to the pool — the - // lock would block every future write to this repo until the - // pool reaps the connection. - let _ = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(lock_key) - .execute(&mut *conn) - .await; + // Dropping the guard frees the advisory lock (its `Drop` + // closes the connection). No Tigris upload on a bail, + // unlike release(true). return Err(e).context("downloading repo from object store for write"); } } } } - Ok(RepoWriteGuard { - owner_slug, - repo_name: repo_name.to_string(), - local_path, - lock_key, - conn, - object_store: self.object_store.clone(), - }) + Ok(guard) } /// Try to acquire ONLY the per-repo advisory lock, non-blocking, with no @@ -273,7 +281,11 @@ impl RepoStore { if !row.0 { return Ok(None); } - Ok(Some(RepoLockGuard { conn, lock_key })) + Ok(Some(RepoLockGuard { + conn: Some(conn), + lock_key, + repo_name: repo_name.to_string(), + })) } /// Initialize a new bare repo on local disk and upload to Tigris. @@ -454,8 +466,9 @@ pub struct RepoWriteGuard { /// Dedicated pooled connection the advisory lock lives on. Held for the /// guard's lifetime so `release` unlocks on the SAME session that locked, /// and so a concurrent `try_lock_repo` cannot be handed this connection and - /// reentrantly re-grab the lock. - conn: sqlx::pool::PoolConnection, + /// reentrantly re-grab the lock. `Option` so `release` can take it (return + /// it to the pool) while `Drop` closes it when release was never reached. + conn: Option>, object_store: Option>, } @@ -485,12 +498,32 @@ impl RepoWriteGuard { warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo"); } - // Release the advisory lock on the SAME connection that took it, then - // the connection returns to the pool on drop. - let _ = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(self.lock_key) - .execute(&mut *self.conn) - .await; + // Unlock on the SAME connection that took the lock, then TAKE the + // connection so it returns to the pool on drop and the guard's `Drop` + // sees `None` (no close-on-drop). If this future is cancelled before the + // take completes, the connection stays in `self.conn` and `Drop` frees + // the lock by closing it instead. + if let Some(mut conn) = self.conn.take() { + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(self.lock_key) + .execute(&mut *conn) + .await; + } + } +} + +impl Drop for RepoWriteGuard { + fn drop(&mut self) { + // If `release()` was never reached (an early `?`, a panic, or the + // caller's future cancelled on client disconnect), the connection still + // holds the session advisory lock. Close it so Postgres frees the lock + // at session end, rather than returning a lock-holding connection to the + // pool where it would block every future write to this repo. (R1.) + if let Some(mut conn) = self.conn.take() { + warn!(repo = %self.repo_name, + "RepoWriteGuard dropped without release() — closing its connection to free the advisory lock"); + conn.close_on_drop(); + } } } @@ -498,18 +531,34 @@ impl RepoWriteGuard { /// lock (and the dedicated connection it lives on) until `release()`. No Tigris /// I/O — unlike [`RepoWriteGuard`], releasing does NOT upload anything. pub struct RepoLockGuard { - conn: sqlx::pool::PoolConnection, + conn: Option>, lock_key: i64, + repo_name: String, } impl RepoLockGuard { /// Release the advisory lock on the same connection that took it, then return /// the connection to the pool. pub async fn release(mut self) { - let _ = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(self.lock_key) - .execute(&mut *self.conn) - .await; + if let Some(mut conn) = self.conn.take() { + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(self.lock_key) + .execute(&mut *conn) + .await; + } + } +} + +impl Drop for RepoLockGuard { + fn drop(&mut self) { + // Same guarantee as RepoWriteGuard: a lock guard dropped without + // release() closes its connection so Postgres frees the advisory lock, + // rather than returning a lock-holding connection to the pool. (R1.) + if let Some(mut conn) = self.conn.take() { + warn!(repo = %self.repo_name, + "RepoLockGuard dropped without release() — closing its connection to free the advisory lock"); + conn.close_on_drop(); + } } } @@ -773,4 +822,226 @@ mod tests { assert!(after.is_some(), "lock is free after release"); after.unwrap().release().await; } + + // ── Drop-safety of the advisory-lock guards (U1: R1, R2; AE2) ─────────── + + // A minimal ObjectStore double: `exists()` is configurable, `download()` can + // park on a gate until notified (to hold `acquire_write` inside its + // post-lock await for the cancellation test), and `upload()` records its + // calls. Serves U1's reorder test and U3's serialization tests. + struct GatedStore { + exists: bool, + download_gate: Option>, + uploads: std::sync::Arc>>, + } + + impl GatedStore { + fn new(exists: bool) -> Self { + Self { + exists, + download_gate: None, + uploads: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), + } + } + } + + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for GatedStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(self.exists) + } + async fn upload(&self, o: &str, r: &str, _p: &std::path::Path) -> anyhow::Result<()> { + self.uploads + .lock() + .unwrap() + .push((o.to_string(), r.to_string())); + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, _p: &std::path::Path) -> anyhow::Result<()> { + if let Some(gate) = &self.download_gate { + gate.notified().await; + } + Ok(()) + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + // Non-perturbing probe: true iff advisory lock `key` is free (grabs it and + // immediately releases it on the observer's own separate session). + async fn advisory_lock_is_free(conn: &mut sqlx::PgConnection, key: i64) -> bool { + let (got,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *conn) + .await + .unwrap(); + if got { + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *conn) + .await; + } + got + } + + async fn poll_until_free(conn: &mut sqlx::PgConnection, key: i64) -> bool { + for _ in 0..50 { + if advisory_lock_is_free(conn, key).await { + return true; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + false + } + + fn lock_key_for(owner: &str, name: &str) -> i64 { + advisory_lock_key(&owner.replace([':', '/'], "_"), name) + } + + // Build a pool whose idle/lifetime reaping is disabled, so a leaked + // (dropped-without-release) connection is NOT reclaimed by ambient sqlx + // maintenance during the poll window. Without this the default sqlx-test + // pool reaps the connection ~1.8s after drop, freeing the lock on its own + // and masking the leak — the test must observe ONLY the fix's own unlock. + async fn no_reap_pool( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: &sqlx::postgres::PgConnectOptions, + ) -> PgPool { + pool_opts + .max_connections(5) + .min_connections(0) + .idle_timeout(None) + .max_lifetime(None) + .test_before_acquire(false) + .connect_with(connect_opts.clone()) + .await + .unwrap() + } + + // R1/AE2: a RepoWriteGuard dropped WITHOUT release() must free its advisory + // lock — its connection is closed rather than returned to the pool still + // holding the session lock. Pre-fix (no Drop impl) the lock leaks -> RED. + #[sqlx::test] + async fn write_guard_dropped_without_release_frees_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let store = RepoStore::new(tmp.path().to_path_buf(), None, pool); + let owner = "did:key:z6MkDropFreesLockAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "droptest"; + let key = lock_key_for(owner, name); + let mut observer = connect_opts.connect().await.unwrap(); + + let guard = store.acquire_write(owner, name).await.unwrap(); + assert!( + !advisory_lock_is_free(&mut observer, key).await, + "lock must be held while the guard is alive" + ); + + drop(guard); // NO release() + + assert!( + poll_until_free(&mut observer, key).await, + "advisory lock must be freed after a RepoWriteGuard is dropped without release()" + ); + } + + // R1: same guarantee for the purge lock-only guard. + #[sqlx::test] + async fn repo_lock_guard_dropped_without_release_frees_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let store = RepoStore::new(tmp.path().to_path_buf(), None, pool); + let owner = "did:key:z6MkLockGuardDropFreesAAAAAAAAAAAAAAAAAAAAAA"; + let name = "lockdrop"; + let key = lock_key_for(owner, name); + let mut observer = connect_opts.connect().await.unwrap(); + + let guard = store.try_lock_repo(owner, name).await.unwrap().unwrap(); + assert!( + !advisory_lock_is_free(&mut observer, key).await, + "lock must be held while the lock guard is alive" + ); + + drop(guard); // NO release() + + assert!( + poll_until_free(&mut observer, key).await, + "advisory lock must be freed after a RepoLockGuard is dropped without release()" + ); + } + + // R2/KTD-2: cancelling acquire_write DURING its post-lock freshness download + // must free the lock. Pre-reorder the lock is won onto a bare connection + // before the guard exists, so cancellation leaks it -> RED; post-reorder the + // guard wraps the connection first and its Drop frees the lock. + #[sqlx::test] + async fn acquire_write_cancelled_during_download_frees_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + ts.download_gate = Some(std::sync::Arc::new(tokio::sync::Notify::new())); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkCancelDownloadFreesAAAAAAAAAAAAAAAAAAAA"; + let name = "canceldl"; + let key = lock_key_for(owner, name); + let mut observer = connect_opts.connect().await.unwrap(); + + { + let fut = store.acquire_write(owner, name); + tokio::pin!(fut); + tokio::select! { + _ = &mut fut => panic!("acquire_write should be parked in download, not complete"), + _ = tokio::time::sleep(std::time::Duration::from_millis(400)) => {} + } + // `fut` is dropped here — cancels acquire_write mid-download. + } + + assert!( + poll_until_free(&mut observer, key).await, + "advisory lock must be freed when acquire_write is cancelled mid-download" + ); + } + + // R2 negative: the normal release() path must NOT close the connection — it + // returns to the pool, so writes don't churn the pool. On a single-connection + // pool a second acquire_write can only succeed if the first returned its + // connection (unlocked, not closed). + #[sqlx::test] + async fn release_returns_connection_to_pool( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + let pool = pool_opts + .max_connections(1) + .acquire_timeout(std::time::Duration::from_secs(3)) + .connect_with(connect_opts) + .await + .unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + let store = RepoStore::new(tmp.path().to_path_buf(), None, pool); + let owner = "did:key:z6MkReleasePoolsConnAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "releasepool"; + + for _ in 0..3 { + let guard = store.acquire_write(owner, name).await.unwrap(); + guard.release(true).await; + } + } } From 2f679d07a4f75cd7d352814e65b8a7fc7370ffde Mon Sep 17 00:00:00 2001 From: t Date: Thu, 16 Jul 2026 09:53:43 -0500 Subject: [PATCH 22/61] fix(node): complete a fully-received push after client disconnect git_receive_pack now runs the acquire -> receive-pack -> release core in a spawned task the handler awaits but whose cancellation it does not propagate. The pack body is fully buffered before the handler runs, so the task is self-contained: a client that disconnects mid-apply drops the handler future without cancelling the task, and the push still applies, uploads, and releases its lock in order. Drop-safety (the guard's close-on-drop) remains the backstop; this closes the dominant window where a disconnect both bricked the lock and abandoned a received push. The post-push bookkeeping tail stays in the cancellable handler (a disconnect still forgoes it, as today). Settled OQ1 by execution: a guard abandoned inside a detached task when the tokio runtime tears down does not panic in Drop (5/5 runs), so detached-task shutdown tracking stays deferred. Kept the check as a regression guard. Disconnect regression drives a real push with a sleeping pre-receive hook and drops the handler mid-hook: RED on the inline handler (the git group is killed, ref absent), GREEN once detached. --- crates/gitlawb-node/src/api/repos.rs | 211 ++++++++++++++++++++-- crates/gitlawb-node/src/git/repo_store.rs | 46 +++++ 2 files changed, 242 insertions(+), 15 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 3b9d888e..3e4b3256 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -924,24 +924,46 @@ pub async fn git_receive_pack( } tracing::debug!(repo = %name, "acquiring write lock"); - let guard = state - .repo_store - .acquire_write(&record.owner_did, &record.name) - .await - .map_err(|e| { - tracing::error!(repo = %name, err = %e, "acquire_write failed"); - AppError::Git(e.to_string()) - })?; - let disk_path = guard.path().to_path_buf(); - tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack"); let body_len = body.len(); let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let receive_result = smart_http::receive_pack(&disk_path, body, git_timeout).await; - // Always release the advisory lock — even on error — to prevent stale locks - // from blocking subsequent pushes. Only upload to Tigris when the push - // succeeded; uploading a half-applied repo would propagate corruption. - guard.release(receive_result.is_ok()).await; + // Detach the acquire → receive-pack → release core from THIS handler future. + // The pack `body` is already fully buffered, so the spawned task is + // self-contained: a client disconnect drops the handler future but does NOT + // cancel the task, so a fully-received push still completes server-side — + // applies the pack, uploads, and releases the lock in order. What a + // disconnect forgoes is the cancellable post-push bookkeeping below, not the + // push itself. (KTD-3, R3.) + let repo_store = state.repo_store.clone(); + let owner_did = record.owner_did.clone(); + let repo_name = record.name.clone(); + let receive_result = tokio::spawn(async move { + let guard = repo_store + .acquire_write(&owner_did, &repo_name) + .await + .map_err(|e| { + tracing::error!(repo = %repo_name, err = %e, "acquire_write failed"); + AppError::Git(e.to_string()) + })?; + let disk_path = guard.path().to_path_buf(); + tracing::debug!(repo = %repo_name, path = %disk_path.display(), "running git receive-pack"); + let result = smart_http::receive_pack(&disk_path, body, git_timeout).await; + // Always release the advisory lock — even on error — to prevent stale + // locks from blocking subsequent pushes. Only upload to Tigris when the + // push succeeded; uploading a half-applied repo would propagate corruption. + guard.release(result.is_ok()).await; + Ok::<_, AppError>(result) + }) + .await + .map_err(|e| { + tracing::error!(repo = %name, err = %e, "receive-pack task panicked"); + AppError::Internal(anyhow::anyhow!("receive-pack task failed: {e}")) + })??; + + // Recompute the on-disk path for the post-push tail below — the guard that + // exposed it now lives inside the detached task. Identical to the path + // acquire_write resolved (repos_dir/owner_slug/name.git). + let disk_path = store::repo_disk_path(&state.config.repos_dir, &record.owner_did, &record.name); let result = receive_result.map_err(|e| { let app = git_service_app_error(&e); @@ -2579,6 +2601,165 @@ mod tests { ); } + // U2/R3/AE1: a fully-received push must complete server-side even when the + // client disconnects during the apply. The pack is buffered before the + // handler runs, so the acquire→receive→release core is detached from the + // handler future; dropping that future (the disconnect) must NOT cancel the + // push. A sleeping pre-receive hook creates a deterministic mid-apply window; + // dropping the handler during it kills the git group on the inline (pre-fix) + // code but not on the detached code, so the hook completes and the ref lands + // only after the fix. RED pre-fix (marker + ref absent), GREEN after. + #[sqlx::test] + async fn fully_received_push_completes_after_client_disconnect(pool: sqlx::PgPool) { + use axum::extract::{Path as AxPath, State}; + use axum::Extension; + use std::os::unix::fs::PermissionsExt; + use std::process::{Command, Stdio}; + use std::time::Duration; + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + + let owner = "z6u2pushowner"; + let name = "u2push"; + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + + // Helper: run git, asserting success. + fn git(args: &[&str], dir: &std::path::Path) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + // Build a commit in a scratch working repo. + let work = tempfile::TempDir::new().unwrap(); + git(&["init", "-q", "-b", "main", "."], work.path()); + git(&["config", "user.email", "t@t"], work.path()); + git(&["config", "user.name", "t"], work.path()); + std::fs::write(work.path().join("f.txt"), "hi").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c"], work.path()); + let oid = git(&["rev-parse", "HEAD"], work.path()); + + // Server bare repo: has the object (from the clone) but no refs/heads/main, + // so the push is a create satisfiable with an empty pack. + std::fs::create_dir_all(bare.parent().unwrap()).unwrap(); + let out = Command::new("git") + .args([ + "clone", + "--bare", + "-q", + &work.path().to_string_lossy(), + &bare.to_string_lossy(), + ]) + .output() + .unwrap(); + assert!( + out.status.success(), + "clone --bare failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + git( + &[ + "-C", + &bare.to_string_lossy(), + "update-ref", + "-d", + "refs/heads/main", + ], + work.path(), + ); + + // Sleeping pre-receive hook — the mid-apply window; writes a marker at the + // end so we can see the git child ran to completion. + let marker = repos_dir.path().join("hook_ran.marker"); + let hook = bare.join("hooks").join("pre-receive"); + std::fs::create_dir_all(hook.parent().unwrap()).unwrap(); + std::fs::write( + &hook, + format!( + "#!/bin/sh\ncat >/dev/null\nsleep 2\necho done > '{}'\nexit 0\n", + marker.display() + ), + ) + .unwrap(); + std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap(); + + // Receive-pack POST body: create refs/heads/main -> oid, plus an empty pack. + let zero = "0".repeat(40); + let cmd = format!("{zero} {oid} refs/heads/main\0report-status\n"); + let mut body = Vec::new(); + let len = cmd.len() + 4; + body.extend_from_slice(format!("{len:04x}").as_bytes()); + body.extend_from_slice(cmd.as_bytes()); + body.extend_from_slice(b"0000"); + let pack = Command::new("git") + .args([ + "-C", + &bare.to_string_lossy(), + "pack-objects", + "--stdout", + "-q", + ]) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!( + pack.status.success(), + "pack-objects failed: {}", + String::from_utf8_lossy(&pack.stderr) + ); + body.extend_from_slice(&pack.stdout); + let body = Bytes::from(body); + + // Drive the handler, then DROP it mid-hook (the "client disconnect"). + let handler = super::git_receive_pack( + State(state.clone()), + AxPath((owner.to_string(), format!("{name}.git"))), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + body, + ); + let _ = tokio::time::timeout(Duration::from_millis(800), handler).await; + + // Wait past the hook's sleep so a surviving (detached) push can finish. + tokio::time::sleep(Duration::from_millis(3000)).await; + + let ref_present = bare.join("refs/heads/main").exists() + || std::fs::read_to_string(bare.join("packed-refs")) + .unwrap_or_default() + .contains("refs/heads/main"); + assert!( + marker.exists(), + "pre-receive hook must run to completion despite the client disconnect (detached push)" + ); + assert!( + ref_present, + "refs/heads/main must be created after a fully-received push despite client disconnect" + ); + } + /// Repo creation must be throttled by the per-IP creation limiter BEFORE /// signature verification — otherwise a DID farm (one throwaway did:key per /// repo, each carrying a valid but machine-solved iCaptcha proof) walks past diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 04e5dfc3..ad9e7bf8 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -1044,4 +1044,50 @@ mod tests { guard.release(true).await; } } + + // OQ1: settle whether a guard abandoned inside a detached task when the + // tokio runtime tears down panics in Drop. `PoolConnection::drop` spawns a + // task (both the close_on_drop path and the normal return path), and + // `rt::spawn` panics via `missing_rt` if no runtime handle is current. U2 + // makes receive-pack a detached task that graceful shutdown does not drain, + // so at process exit such a task can be dropped mid-flight holding a guard. + // This drops a real runtime with a parked guard-holding task and asserts the + // process survives (a panic-in-Drop during unwind would abort the binary). + #[test] + fn guard_drop_during_runtime_shutdown_does_not_panic() { + let url = match std::env::var("DATABASE_URL") { + Ok(u) => u, + Err(_) => return, // no DB configured — skip (mirrors sqlx::test gating) + }; + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let ready = std::sync::Arc::new(tokio::sync::Notify::new()); + let ready2 = ready.clone(); + rt.block_on(async move { + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(5) + .connect(&url) + .await + .unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + let store = RepoStore::new(tmp.path().to_path_buf(), None, pool); + tokio::spawn(async move { + let _guard = store + .acquire_write("did:key:z6MkOQ1ShutdownAAAAAAAAAAAAAAAAAAAAAAAAA", "oq1") + .await + .unwrap(); + ready2.notify_one(); + // Park forever holding the guard. + std::future::pending::<()>().await; + }); + ready.notified().await; + }); + // Drop the runtime while the detached task is parked holding the guard: + // the task is cancelled, dropping the guard during runtime teardown. Its + // Drop must not panic. Reaching the end of the test proves it didn't. + drop(rt); + } } From c3fedadb145d2fdca2721b9429eb8ce7c021c9cc Mon Sep 17 00:00:00 2001 From: t Date: Thu, 16 Jul 2026 10:06:06 -0500 Subject: [PATCH 23/61] fix(node): serialize archive uploads under the per-repo advisory lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lazy-migration upload in acquire(), init()'s background upload, and release_after_write() (fork's upload) previously PUT to the object store with no lock, so an in-flight upload could resurrect an archive purge-spam had just deleted under the same lock, and init's background upload could clobber a freshly-pushed archive. All three now go through upload_locked(): take the per-repo lock, re-check the local dir exists under it (a purge removes the dir under its lock, so a post-purge uploader finds it gone and skips), upload, then release. The background migration/init uploads skip on contention and self-heal; fork's foreground upload waits (bounded) instead, since a skipped fork upload has no later retry. Tests (recording object-store double): init upload skips under a held lock, fork upload waits then PUTs once after release, a late upload after a purge-style delete does not resurrect the archive, and an uncontended upload PUTs once. Load-bearing per INV-21 — neutralizing the lock+dir-check turns the three contention/resurrect tests RED. --- crates/gitlawb-node/src/git/repo_store.rs | 306 ++++++++++++++++++++-- 1 file changed, 279 insertions(+), 27 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index ad9e7bf8..211830e0 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -71,10 +71,11 @@ impl RepoStore { let key = format!("{owner_slug}/{repo_name}"); let already_migrated = self.migrated.lock().await.contains(&key); if !already_migrated { + let store = self.clone(); let tigris = tigris.clone(); + let did = owner_did.to_string(); let slug = owner_slug.clone(); let name = repo_name.to_string(); - let path = local_path.clone(); let migrated = Arc::clone(&self.migrated); tokio::spawn(async move { // Check if already in Tigris before uploading @@ -84,8 +85,10 @@ impl RepoStore { } Ok(false) => { info!(repo = %name, "migrating local repo to tigris"); - if let Err(e) = tigris.upload(&slug, &name, &path).await { - warn!(repo = %name, err = %e, "lazy migration to tigris failed"); + // Upload under the per-repo lock so it can't race + // a purge. If it skipped (lock contended or dir + // gone), do NOT mark migrated — retry next acquire. + if !store.upload_locked(&did, &name, false).await { return; } info!(repo = %name, "lazy migration to tigris complete"); @@ -290,41 +293,121 @@ impl RepoStore { /// Initialize a new bare repo on local disk and upload to Tigris. pub async fn init(&self, owner_did: &str, repo_name: &str) -> Result { - let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; + let (_owner_slug, local_path) = self.local_path(owner_did, repo_name)?; store::init_bare(&local_path).context("initializing bare repo")?; - // Upload to Tigris in background - if let Some(ref tigris) = self.object_store { - let tigris = tigris.clone(); - let owner_slug = owner_slug.clone(); - let repo_name = repo_name.to_string(); - let path = local_path.clone(); + // Upload to the object store in the background, UNDER the per-repo lock + // so the PUT can't race a purge. Skips on contention; a skipped init + // upload self-heals — the first write's release re-uploads the state. + if self.object_store.is_some() { + let store = self.clone(); + let did = owner_did.to_string(); + let name = repo_name.to_string(); tokio::spawn(async move { - if let Err(e) = tigris.upload(&owner_slug, &repo_name, &path).await { - warn!(repo = %repo_name, err = %e, "failed to upload new repo to tigris"); - } + store.upload_locked(&did, &name, false).await; }); } Ok(local_path) } - /// Upload a repo to Tigris after a write operation (push, merge, fork, etc.). - /// Call this after any operation that modifies the git repo on disk. + /// Upload a repo to Tigris after a write operation (fork, etc.). Call this + /// after any operation that modifies the git repo on disk. Uploads UNDER the + /// per-repo advisory lock so the PUT cannot race a concurrent purge (which + /// deletes the repo under the same lock) and resurrect a deleted archive. + /// Waits (bounded) for the lock — a skipped fork upload would have no later + /// retry; in practice the fork target's lock is uncontended (its DB row is + /// created after this upload). pub async fn release_after_write(&self, owner_did: &str, repo_name: &str) { - if let Some(ref tigris) = self.object_store { - let (owner_slug, local_path) = match self.local_path(owner_did, repo_name) { - Ok(p) => p, + self.upload_locked(owner_did, repo_name, true).await; + } + + /// Upload the local repo to the object store while holding the per-repo + /// advisory lock, then release. The lock serializes the PUT against a + /// concurrent `purge-spam` that deletes the repo (row + dir + archive) under + /// the same lock, so an in-flight upload cannot resurrect a just-deleted + /// archive. Re-checks the local dir exists UNDER the lock — a purge removes + /// the dir under its lock, so a post-purge uploader finds it gone and skips. + /// Returns true iff a PUT was performed. A no-op when no store is configured. + /// + /// `wait`: fork's foreground upload waits (bounded) for the lock; the + /// background migration/init uploads pass `false` and skip on contention — + /// they self-heal (lazy migration retries on the next `acquire`, init's + /// state is re-uploaded by the first write's release). + async fn upload_locked(&self, owner_did: &str, repo_name: &str, wait: bool) -> bool { + let Some(ref store) = self.object_store else { + return false; + }; + let (owner_slug, local_path) = match self.local_path(owner_did, repo_name) { + Ok(p) => p, + Err(e) => { + warn!(repo = %repo_name, err = %e, "rejected unsafe path before object-store upload"); + return false; + } + }; + let guard = if wait { + match self.lock_repo_blocking(owner_did, repo_name).await { + Ok(Some(g)) => g, + Ok(None) => { + warn!(repo = %repo_name, "object-store upload skipped — repo lock still held after retry"); + return false; + } + Err(e) => { + warn!(repo = %repo_name, err = %e, "object-store upload skipped — could not acquire repo lock"); + return false; + } + } + } else { + match self.try_lock_repo(owner_did, repo_name).await { + Ok(Some(g)) => g, + Ok(None) => { + debug!(repo = %repo_name, "object-store upload skipped — repo locked by a live writer"); + return false; + } Err(e) => { - warn!(repo = %repo_name, err = %e, "rejected unsafe path in release_after_write"); - return; + warn!(repo = %repo_name, err = %e, "object-store upload skipped — could not acquire repo lock"); + return false; } - }; - if let Err(e) = tigris.upload(&owner_slug, repo_name, &local_path).await { - warn!(repo = %repo_name, err = %e, "failed to upload repo to tigris after write"); } + }; + // Re-check under the lock: a purge removes the on-disk dir under its lock, + // so a post-purge uploader must find it gone and NOT recreate the archive. + if !local_path.exists() { + warn!(repo = %repo_name, "object-store upload skipped — local repo dir gone under lock (purged?)"); + guard.release().await; + return false; } + let uploaded = match store.upload(&owner_slug, repo_name, &local_path).await { + Ok(()) => true, + Err(e) => { + warn!(repo = %repo_name, err = %e, "failed to upload repo to object store"); + false + } + }; + guard.release().await; + uploaded + } + + /// Bounded-wait lock-only acquire — the fork foreground upload's counterpart + /// to `acquire_write`'s spin, without any object-store I/O. Retries + /// `try_lock_repo` with short backoff. In practice the fork target's lock is + /// uncontended (its DB row is created after the upload), so this returns on + /// the first attempt. + async fn lock_repo_blocking( + &self, + owner_did: &str, + repo_name: &str, + ) -> Result> { + for attempt in 0..30 { + if let Some(g) = self.try_lock_repo(owner_did, repo_name).await? { + return Ok(Some(g)); + } + if attempt < 29 { + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + } + } + Ok(None) } /// Compute the local disk path and owner slug for a repo. @@ -830,17 +913,21 @@ mod tests { // post-lock await for the cancellation test), and `upload()` records its // calls. Serves U1's reorder test and U3's serialization tests. struct GatedStore { - exists: bool, + // `exists` is dynamic so a delete() flips it false and an upload() flips + // it true — lets U3's resurrect test assert a deleted archive stays gone. + exists: std::sync::Arc, download_gate: Option>, uploads: std::sync::Arc>>, + deletes: std::sync::Arc>>, } impl GatedStore { fn new(exists: bool) -> Self { Self { - exists, + exists: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(exists)), download_gate: None, uploads: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), + deletes: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), } } } @@ -848,13 +935,14 @@ mod tests { #[async_trait::async_trait] impl crate::git::tigris::ObjectStore for GatedStore { async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { - Ok(self.exists) + Ok(self.exists.load(std::sync::atomic::Ordering::SeqCst)) } async fn upload(&self, o: &str, r: &str, _p: &std::path::Path) -> anyhow::Result<()> { self.uploads .lock() .unwrap() .push((o.to_string(), r.to_string())); + self.exists.store(true, std::sync::atomic::Ordering::SeqCst); Ok(()) } async fn download(&self, _o: &str, _r: &str, _p: &std::path::Path) -> anyhow::Result<()> { @@ -863,7 +951,13 @@ mod tests { } Ok(()) } - async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + async fn delete(&self, o: &str, r: &str) -> anyhow::Result<()> { + self.deletes + .lock() + .unwrap() + .push((o.to_string(), r.to_string())); + self.exists + .store(false, std::sync::atomic::Ordering::SeqCst); Ok(()) } } @@ -1090,4 +1184,162 @@ mod tests { // Drop must not panic. Reaching the end of the test proves it didn't. drop(rt); } + + // ── U3: archive uploads serialize under the per-repo lock (R7, R8, R9) ─── + + fn repo_dir_of(repos_dir: &std::path::Path, owner: &str, name: &str) -> std::path::PathBuf { + repos_dir + .join(owner.replace([':', '/'], "_")) + .join(format!("{name}.git")) + } + + // R7/R8/AE6: init's background upload must SKIP while a live writer holds the + // repo lock. Pre-fix (unlocked upload) it PUTs regardless -> RED. + #[sqlx::test] + async fn init_upload_skips_while_repo_locked( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + let pool = pool_opts + .max_connections(5) + .connect_with(connect_opts) + .await + .unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + let ts = GatedStore::new(false); + let uploads = ts.uploads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkInitSkipAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "initskip"; + + let held = store.try_lock_repo(owner, name).await.unwrap().unwrap(); + store.init(owner, name).await.unwrap(); + // Let the spawned upload attempt-and-skip. + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + uploads.lock().unwrap().is_empty(), + "init's background upload must skip while the repo is locked by a live writer" + ); + held.release().await; + } + + // R8/AE6: fork's foreground upload (release_after_write) WAITS for the lock + // rather than skipping, then PUTs once after it frees. Pre-fix it PUTs + // immediately while the lock is held -> RED on the "still empty" assert. + #[sqlx::test] + async fn release_after_write_waits_then_uploads_after_lock_frees( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + let pool = pool_opts + .max_connections(5) + .connect_with(connect_opts) + .await + .unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + let ts = GatedStore::new(false); + let uploads = ts.uploads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkForkWaitAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "forkwait"; + let slug = owner.replace([':', '/'], "_"); + std::fs::create_dir_all(repo_dir_of(tmp.path(), owner, name)).unwrap(); + + let held = store.try_lock_repo(owner, name).await.unwrap().unwrap(); + let store2 = store.clone(); + let owner2 = owner.to_string(); + let name2 = name.to_string(); + let h = tokio::spawn(async move { store2.release_after_write(&owner2, &name2).await }); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + uploads.lock().unwrap().is_empty(), + "fork upload must wait (not skip) while the repo lock is held" + ); + + held.release().await; + h.await.unwrap(); + let ups = uploads.lock().unwrap(); + assert_eq!( + ups.len(), + 1, + "fork upload must PUT exactly once after the lock frees" + ); + assert_eq!(ups[0], (slug, name.to_string())); + } + + // R9/AE6: after a purge deletes the archive AND removes the on-disk dir under + // the lock, a late upload (that lost the race) must NOT resurrect the archive + // — it finds the dir gone under the lock and skips. Pre-fix (no dir recheck) + // it re-PUTs and revives the archive -> RED. + #[sqlx::test] + async fn upload_does_not_resurrect_a_purged_archive(pool: sqlx::PgPool) { + use std::sync::atomic::Ordering::SeqCst; + let tmp = tempfile::TempDir::new().unwrap(); + let ts = GatedStore::new(false); + let uploads = ts.uploads.clone(); + let deletes = ts.deletes.clone(); + let exists = ts.exists.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkResurrectAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "resurrect"; + let dir = repo_dir_of(tmp.path(), owner, name); + std::fs::create_dir_all(&dir).unwrap(); + + store.release_after_write(owner, name).await; + assert!(exists.load(SeqCst), "archive exists after the first upload"); + assert_eq!(uploads.lock().unwrap().len(), 1); + + // Simulate a purge: delete the archive and remove the on-disk dir. + store.delete_archive(owner, name).await.unwrap(); + std::fs::remove_dir_all(&dir).unwrap(); + assert_eq!(deletes.lock().unwrap().len(), 1, "archive delete recorded"); + assert!(!exists.load(SeqCst), "archive is deleted by the purge"); + + // A late upload attempt must find the dir gone and skip, not resurrect. + store.release_after_write(owner, name).await; + assert!( + !exists.load(SeqCst), + "a purged archive must stay deleted — the upload found no dir and skipped" + ); + assert_eq!( + uploads.lock().unwrap().len(), + 1, + "no second PUT after the repo dir was purged" + ); + } + + // Negative: an uncontended fork upload PUTs exactly once (the lock is free). + #[sqlx::test] + async fn uncontended_release_after_write_uploads_once(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let ts = GatedStore::new(false); + let uploads = ts.uploads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkUncontendedAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "uncontended"; + std::fs::create_dir_all(repo_dir_of(tmp.path(), owner, name)).unwrap(); + + store.release_after_write(owner, name).await; + assert_eq!( + uploads.lock().unwrap().len(), + 1, + "an uncontended fork upload must PUT exactly once" + ); + } } From f3cf6fc08794c90b4df2ceeed393e2ecc070c9e1 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 16 Jul 2026 10:18:15 -0500 Subject: [PATCH 24/61] fix(node): let purge-spam reach repos that exist only as archives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a multi-machine (object-store) deployment a spam repo can exist only as a Tigris archive with no local copy. Selection filtered on a ref count that returns non-empty for a missing local path, so such a repo was never a candidate and the tool it was hardened for could not reach it. Selection now distinguishes missing-local (Option::None) from a one-ref repo and admits a missing-local row as a *remote-unverified* candidate only when an object store is configured (via RepoStore::has_object_store); partition_for_delete passes those through rather than re-dropping them on the local ref count. The execute loop is unchanged — it already refreshes from the archive and rechecks emptiness under the per-repo lock — so a remote-only empty archive is deleted, one with refs is refreshed and skipped, and missing-both fails closed (refresh no-ops, the local recheck reports non-empty). Storeless deployments keep the old fail-closed behavior. Dry-run marks remote-only candidates distinctly and counts them in a new PurgeSummary field. Tests (recording object-store double): remote-only empty archive deleted, remote-only archive-with-refs skipped, missing-both fail-closed, storeless missing-local not a candidate. Load-bearing per INV-21 — reverting the store-configured admission turns the three remote-reach tests RED. --- crates/gitlawb-node/src/admin.rs | 315 ++++++++++++++++++---- crates/gitlawb-node/src/git/repo_store.rs | 7 + 2 files changed, 273 insertions(+), 49 deletions(-) diff --git a/crates/gitlawb-node/src/admin.rs b/crates/gitlawb-node/src/admin.rs index 35016424..a5c2edf5 100644 --- a/crates/gitlawb-node/src/admin.rs +++ b/crates/gitlawb-node/src/admin.rs @@ -54,6 +54,10 @@ pub struct PurgeSummary { /// survives and could be re-downloaded into a later same-owner/name repo, so /// this is tracked separately and never folded into a clean success. pub archive_failed: usize, + /// Candidates with no local copy, admitted only because an object store is + /// configured (emptiness decided under the lock at execute time). Reported so + /// the dry-run can surface them distinctly from locally-verified candidates. + pub remote_unverified: usize, } /// A repo selected for purge, with the evidence that qualified it. @@ -62,9 +66,14 @@ pub struct Candidate { pub id: String, pub owner_did: String, pub name: String, - /// Number of git refs found on disk. Always 0 for a selected candidate — kept - /// as explicit evidence in the dry-run output rather than an implicit "empty". + /// Number of git refs found on disk. 0 for a locally-verified empty candidate; + /// also 0 for a remote-unverified one whose emptiness is decided under the lock. pub ref_count: usize, + /// True when the repo has no local copy and was admitted only because an object + /// store is configured. Its emptiness has NOT been verified — the execute path + /// must refresh from the archive and recheck UNDER the per-repo lock before any + /// delete; the dry-run lists it distinctly and never touches it. + pub remote_unverified: bool, } /// Whether a DID is on the hard exclusion list. Compared under did:key @@ -82,23 +91,29 @@ fn is_excluded(owner_did: &str) -> bool { /// exclusion + empty logic is directly testable. /// /// `repos` is the raw row set to consider (the caller supplies the target DID's -/// rows). `ref_count_of` returns the number of git refs for a given repo; the CLI -/// wires the real on-disk ref source, tests inject precomputed counts. +/// rows). `local_refs_of` returns `Some(n)` when a local bare repo exists (n +/// refs) and `None` when there is no local copy; the CLI wires the real on-disk +/// source, tests inject precomputed states. `store_configured` gates whether a +/// missing-local repo may be admitted. /// /// A repo qualifies ONLY if, PER REPO: /// 1. its owner is NOT on the exclusion list (gate evaluated FIRST), AND /// 2. its owner is exactly the target burst DID, AND -/// 3. it is verified empty — `ref_count_of` returns 0. +/// 3. EITHER it is locally verified empty (`Some(0)`), OR it has no local copy +/// (`None`) AND an object store is configured — in which case it is admitted +/// as remote-unverified and its emptiness is decided under the lock later. /// -/// The exclusion gate is checked before the empty check so that an empty repo -/// owned by an excluded DID is dropped regardless of its ref signature. +/// The exclusion gate is checked before everything so an empty repo owned by an +/// excluded DID is dropped regardless of its ref signature. A missing-local repo +/// with no object store fails closed (skipped). pub fn select_spam_candidates( repos: &[RepoRecord], target_did: &str, - mut ref_count_of: F, + store_configured: bool, + mut local_refs_of: F, ) -> Vec where - F: FnMut(&RepoRecord) -> usize, + F: FnMut(&RepoRecord) -> Option, { let mut out = Vec::new(); for repo in repos { @@ -113,16 +128,28 @@ where { continue; } - // Per-repo empty check. - let ref_count = ref_count_of(repo); - if ref_count != 0 { - continue; - } + // `local_refs_of` is `Some(n)` when a local bare repo exists and `None` + // when it does not. A local empty repo (Some(0)) is a verified candidate; + // a local non-empty repo is skipped; a missing local copy is a candidate + // ONLY when an object store is configured (its emptiness is then decided + // authoritatively under the lock after refresh), else it fails closed. + let remote_unverified = match local_refs_of(repo) { + Some(0) => false, + Some(_) => continue, + None => { + if store_configured { + true + } else { + continue; + } + } + }; out.push(Candidate { id: repo.id.clone(), owner_did: repo.owner_did.clone(), name: repo.name.clone(), - ref_count, + ref_count: 0, + remote_unverified, }); } out @@ -144,12 +171,40 @@ where /// repo's refs (possibly `0`) and delete a real repo. We defend by requiring the /// bare-repo markers (`HEAD` file + `objects/` dir) before trusting any count; /// anything else fails closed (treated non-empty, skipped). -fn on_disk_ref_count(repos_dir: &Path, repo: &RepoRecord) -> usize { - ref_count_on_disk(repos_dir, &repo.owner_did, &repo.name) +/// Local ref state for selection: `Some(n)` when a local bare repo exists (n +/// refs), `None` when there is no local copy. `None` is what lets selection +/// distinguish a missing-local repo (a remote-unverified candidate when a store +/// is configured) from a one-ref repo — both of which `ref_count_on_disk` +/// collapses to a non-zero count. An unsafe name or an unreadable repo fails +/// closed to `Some(1)` so it is skipped, never admitted as remote-unverified. +fn local_refs_on_disk(repos_dir: &Path, owner_did: &str, name: &str) -> Option { + // Fail closed on an unsafe repo name BEFORE building any on-disk path (a + // peer-mirror row can carry a `../` name). Report it as non-empty so it is + // never a candidate — never as missing (which could admit it remote-unverified). + if let Err(e) = crate::git::repo_store::validate_repo_name(name) { + warn!(name = %name, err = %e, + "purge-spam: unsafe repo name — treating as non-empty (skipped)"); + return Some(1); + } + let path = store::repo_disk_path(repos_dir, owner_did, name); + if !path.join("HEAD").is_file() || !path.join("objects").is_dir() { + // No local bare repo at the expected path. + return None; + } + match store::list_refs(&path) { + Ok(refs) => Some(refs.len()), + Err(e) => { + warn!(path = %path.display(), err = %e, + "purge-spam: could not read refs — treating as non-empty (skipped)"); + Some(1) + } + } } -/// Core of [`on_disk_ref_count`], keyed on owner+name so the execute path can -/// re-verify emptiness right before deleting (using only a [`Candidate`]). +/// Ref count keyed on owner+name (returns 1 for a missing/unsafe/unreadable +/// repo — fail closed), used by the execute path to re-verify emptiness right +/// before deleting (using only a [`Candidate`]) and, after `refresh_from_archive` +/// downloads a remote-unverified candidate, to decide its emptiness under the lock. fn ref_count_on_disk(repos_dir: &Path, owner_did: &str, name: &str) -> usize { // Fail closed on an unsafe repo name BEFORE building any on-disk path. A // peer-mirror row (which skips API name validation) can carry a `../` name; @@ -233,9 +288,16 @@ pub async fn run_purge_spam( .await .context("listing repos for the spam-burst target DID")?; - let candidates = select_spam_candidates(&rows, SPAM_BURST_TARGET_DID, |repo| { - on_disk_ref_count(&repos_dir, repo) - }); + // A repo with no local copy is admitted as a remote-unverified candidate + // only when an object store is configured — its emptiness is then decided + // under the lock after refresh_from_archive. Without a store, missing-local + // fails closed (skipped), preserving the wrong-machine safety rule. + let store_configured = repo_store.has_object_store(); + let candidates = + select_spam_candidates(&rows, SPAM_BURST_TARGET_DID, store_configured, |repo| { + local_refs_on_disk(&repos_dir, &repo.owner_did, &repo.name) + }); + let remote_unverified_count = candidates.iter().filter(|c| c.remote_unverified).count(); info!( target = SPAM_BURST_TARGET_DID, @@ -257,25 +319,39 @@ pub async fn run_purge_spam( if execute { "EXECUTE" } else { "dry-run" } ); for c in &candidates { + let marker = if c.remote_unverified { + " [remote-only, emptiness verified under lock at execute]" + } else { + "" + }; println!( - " {} owner={} name={} refs={}", + " {} owner={} name={} refs={}{marker}", c.id, c.owner_did, c.name, c.ref_count ); } if !execute { println!( - "purge-spam: dry-run — nothing deleted. Re-run with --execute to delete the {} candidate(s).", + "purge-spam: dry-run — nothing deleted ({remote_unverified_count} remote-only, verified under lock only on --execute). Re-run with --execute to delete the {} candidate(s).", candidates.len() ); - return Ok(PurgeSummary::default()); + return Ok(PurgeSummary { + remote_unverified: remote_unverified_count, + ..PurgeSummary::default() + }); } // Re-verify emptiness immediately before deleting: a push may have landed - // between selection and now (TOCTOU). Anything no longer empty is skipped, - // never deleted. + // between selection and now (TOCTOU). A remote-unverified candidate has no + // local copy yet, so `ref_count_on_disk` would report it non-empty and drop + // it here — pass it straight through instead; the authoritative emptiness + // check for it happens under the lock in the execute loop after refresh. let (to_delete, to_skip) = partition_for_delete(&candidates, |c| { - ref_count_on_disk(&repos_dir, &c.owner_did, &c.name) + if c.remote_unverified { + 0 + } else { + ref_count_on_disk(&repos_dir, &c.owner_did, &c.name) + } }); for c in &to_skip { warn!(repo = %c.id, "purge-spam: repo no longer empty at delete time — skipped (TOCTOU)"); @@ -284,6 +360,7 @@ pub async fn run_purge_spam( // Execute: delete per-repo, never a single blanket "delete all of owner X". // A per-repo failure warns and continues rather than aborting the batch. let mut summary = PurgeSummary { + remote_unverified: remote_unverified_count, skipped_not_empty: to_skip.len(), ..PurgeSummary::default() }; @@ -408,12 +485,15 @@ mod tests { } /// Ref counts keyed by repo id; anything absent defaults to 0 (empty). - fn refs_by_id<'a>(map: &'a [(&'a str, usize)]) -> impl Fn(&RepoRecord) -> usize + 'a { + fn refs_by_id<'a>(map: &'a [(&'a str, usize)]) -> impl Fn(&RepoRecord) -> Option + 'a { + // A local bare repo exists for every test row (Some), with `n` refs. move |r: &RepoRecord| { - map.iter() - .find(|(id, _)| *id == r.id) - .map(|(_, n)| *n) - .unwrap_or(0) + Some( + map.iter() + .find(|(id, _)| *id == r.id) + .map(|(_, n)| *n) + .unwrap_or(0), + ) } } @@ -421,7 +501,7 @@ mod tests { #[test] fn empty_target_repo_is_a_candidate() { let repos = vec![repo("t-empty", TARGET, "spam1")]; - let got = select_spam_candidates(&repos, TARGET, refs_by_id(&[])); + let got = select_spam_candidates(&repos, TARGET, false, refs_by_id(&[])); assert_eq!(got.len(), 1, "empty target repo must be selected"); assert_eq!(got[0].id, "t-empty"); assert_eq!(got[0].owner_did, TARGET); @@ -439,7 +519,7 @@ mod tests { repo("t-empty", TARGET, "spam1"), repo("t-nonempty", TARGET, "real"), ]; - let got = select_spam_candidates(&repos, TARGET, refs_by_id(&[("t-nonempty", 3)])); + let got = select_spam_candidates(&repos, TARGET, false, refs_by_id(&[("t-nonempty", 3)])); let ids: Vec<&str> = got.iter().map(|c| c.id.as_str()).collect(); assert!(ids.contains(&"t-empty"), "empty target repo still selected"); assert!( @@ -458,14 +538,14 @@ mod tests { #[test] fn empty_excluded_content_repo_is_absent() { let repos = vec![repo("x-content", EXCLUDED_CONTENT, "anything")]; - let got = select_spam_candidates(&repos, EXCLUDED_CONTENT, refs_by_id(&[])); + let got = select_spam_candidates(&repos, EXCLUDED_CONTENT, false, refs_by_id(&[])); assert!( got.is_empty(), "an empty repo owned by the excluded content DID must be excluded even \ if that DID were the target, got {got:?}" ); // And of course it is also absent when the real burst DID is the target. - let got = select_spam_candidates(&repos, TARGET, refs_by_id(&[])); + let got = select_spam_candidates(&repos, TARGET, false, refs_by_id(&[])); assert!(got.is_empty()); } @@ -475,13 +555,13 @@ mod tests { #[test] fn empty_intern_repo_is_absent() { let repos = vec![repo("x-intern", EXCLUDED_INTERN, "mirror")]; - let got = select_spam_candidates(&repos, EXCLUDED_INTERN, refs_by_id(&[])); + let got = select_spam_candidates(&repos, EXCLUDED_INTERN, false, refs_by_id(&[])); assert!( got.is_empty(), "an empty repo owned by the intern/mirror-bot DID must be excluded even \ if that DID were the target, got {got:?}" ); - let got = select_spam_candidates(&repos, TARGET, refs_by_id(&[])); + let got = select_spam_candidates(&repos, TARGET, false, refs_by_id(&[])); assert!(got.is_empty()); } @@ -490,7 +570,7 @@ mod tests { #[test] fn empty_unrelated_repo_is_absent() { let repos = vec![repo("u-empty", UNRELATED, "whatever")]; - let got = select_spam_candidates(&repos, TARGET, refs_by_id(&[])); + let got = select_spam_candidates(&repos, TARGET, false, refs_by_id(&[])); assert!( got.is_empty(), "an empty repo owned by a non-target DID must not be selected, got {got:?}" @@ -507,7 +587,7 @@ mod tests { repo("x-intern", EXCLUDED_INTERN, "b"), // excluded, empty → out repo("u-empty", UNRELATED, "c"), // wrong owner → out ]; - let got = select_spam_candidates(&repos, TARGET, refs_by_id(&[("t-nonempty", 2)])); + let got = select_spam_candidates(&repos, TARGET, false, refs_by_id(&[("t-nonempty", 2)])); assert_eq!( got.iter().map(|c| c.id.as_str()).collect::>(), vec!["t-empty"], @@ -521,7 +601,7 @@ mod tests { #[test] fn exclusion_gate_precedes_empty_check() { let repos = vec![repo("collision", EXCLUDED_CONTENT, "spam1")]; - let got = select_spam_candidates(&repos, EXCLUDED_CONTENT, refs_by_id(&[])); + let got = select_spam_candidates(&repos, EXCLUDED_CONTENT, false, refs_by_id(&[])); assert!(got.is_empty(), "exclusion must win even on an empty repo"); } @@ -534,6 +614,7 @@ mod tests { owner_did: "o".into(), name: id.into(), ref_count: 0, + remote_unverified: false, }; let cands = vec![cand("still-empty"), cand("now-nonempty")]; // Re-check reports the second repo as no longer empty. @@ -934,19 +1015,30 @@ mod db_tests { let path = store::repo_disk_path(tmp.path(), &repo.owner_did, &repo.name); // (a) Path exists as a plain (non-git) directory under the git ancestor. - // Without the marker guard, git discovery reads the ancestor's 0 refs and - // this repo would be deleted. It must fail closed (>=1, skipped). + // Without the marker guard, git discovery would read the ancestor's 0 + // refs and this repo would be deleted. local_refs_on_disk must report no + // local repo (None) WITHOUT running list_refs — never the ancestor's 0. + // None is skipped unless a store is configured, and the under-lock + // recheck (ref_count_on_disk) fails closed on the same markers regardless. std::fs::create_dir_all(&path).unwrap(); assert_eq!( - on_disk_ref_count(tmp.path(), &repo), + local_refs_on_disk(tmp.path(), &repo.owner_did, &repo.name), + None, + "a non-git dir under a git ancestor must report no local repo, not read the ancestor's refs" + ); + assert_eq!( + ref_count_on_disk(tmp.path(), &repo.owner_did, &repo.name), 1, - "a non-git dir under a git ancestor must fail closed, not read the ancestor's refs" + "the under-lock recheck must also fail closed on a non-git dir" ); - // (b) A real empty bare repo at the same path reads 0 — a genuine candidate. + // (b) A real empty bare repo at the same path reads Some(0) — a candidate. std::fs::remove_dir_all(&path).unwrap(); store::init_bare(&path).unwrap(); - assert_eq!(on_disk_ref_count(tmp.path(), &repo), 0); + assert_eq!( + local_refs_on_disk(tmp.path(), &repo.owner_did, &repo.name), + Some(0) + ); } /// The belt-and-suspenders containment gate (`path_within`) must reject a path @@ -1008,7 +1100,12 @@ mod db_tests { // An empty repo owned by the short-form excluded DID is spared even though // its ref signature (0) otherwise matches the burst. let empty_excluded_short = rec("x-short", short, "spam"); - let cands = select_spam_candidates(&[empty_excluded_short], SPAM_BURST_TARGET_DID, |_| 0); + let cands = select_spam_candidates( + &[empty_excluded_short], + SPAM_BURST_TARGET_DID, + false, + |_| Some(0), + ); assert!( cands.is_empty(), "an empty repo owned by a short-form excluded DID must never be a candidate" @@ -1225,6 +1322,126 @@ mod db_tests { ); } + // AE3/R4: a repo that exists ONLY as an object-store archive (no local copy) + // with an EMPTY archive is reached, refreshed under the lock, and deleted + // (row + dir + archive). Pre-U4 a missing-local row was never a candidate + // (treated non-empty, skipped) -> RED; admitting it remote-unverified -> GREEN. + #[sqlx::test] + async fn purge_deletes_remote_only_empty_archive(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-remote-empty", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + // NO local repo — it exists only as an archive. + let path = store::repo_disk_path(tmp.path(), &target.owner_did, &target.name); + assert!(!path.exists(), "no local copy — remote-only"); + + let (f, deleted) = fake(true, false, false, false); // archive exists, empty + let repo_store = store_backed(tmp.path(), &pool, f); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none(), + "a remote-only empty archive must be reached and deleted" + ); + assert!( + deleted.load(std::sync::atomic::Ordering::SeqCst), + "the archive must be deleted too" + ); + assert_eq!(summary.deleted, 1); + assert_eq!( + summary.remote_unverified, 1, + "the candidate was admitted as remote-unverified" + ); + } + + // AE4/R4: a remote-only archive that turns out to HAVE refs is refreshed under + // the lock and then skipped — never deleted on the missing-local view. + #[sqlx::test] + async fn purge_skips_remote_only_archive_with_refs(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-remote-refs", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &target.owner_did, &target.name); + assert!(!path.exists()); + + let (f, _deleted) = fake(true, true, false, false); // archive exists, HAS refs + let repo_store = store_backed(tmp.path(), &pool, f); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_some(), + "a remote-only archive with refs must be refreshed and skipped, not deleted" + ); + assert_eq!(summary.deleted, 0); + assert_eq!(summary.skipped_not_empty, 1); + } + + // AE5/R5: no local copy AND no archive — the candidate is admitted (a store is + // configured) but the under-lock recheck finds nothing and fails closed. + #[sqlx::test] + async fn purge_skips_remote_unverified_with_no_archive(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-missing-both", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + + let (f, _deleted) = fake(false, false, false, false); // NO archive + let repo_store = store_backed(tmp.path(), &pool, f); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_some(), + "missing local AND no archive must fail closed (not deleted)" + ); + assert_eq!(summary.deleted, 0); + assert_eq!( + summary.skipped_not_empty, 1, + "skipped by the authoritative under-lock recheck" + ); + } + + // R5: with NO object store, a repo with no local copy is not even a candidate + // (the missing-local admission is gated on a configured store). + #[sqlx::test] + async fn purge_storeless_missing_local_is_not_a_candidate(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-nolocal", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + // Storeless RepoStore, no local repo on disk. + let repo_store = test_store(tmp.path(), &pool); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_some(), + "storeless + missing-local must fail closed — never a candidate" + ); + assert_eq!(summary.deleted, 0); + assert_eq!(summary.remote_unverified, 0); + } + // R7: with no object store configured (single-machine), an empty repo is // deleted exactly as before and nothing touches an archive. #[sqlx::test] diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 211830e0..f9a57c3a 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -323,6 +323,13 @@ impl RepoStore { self.upload_locked(owner_did, repo_name, true).await; } + /// Whether an object store is configured. Used by purge-spam to decide + /// whether a repo with no local copy can be a remote-unverified candidate + /// (its emptiness verified under the lock after a refresh) versus fail-closed. + pub fn has_object_store(&self) -> bool { + self.object_store.is_some() + } + /// Upload the local repo to the object store while holding the per-repo /// advisory lock, then release. The lock serializes the PUT against a /// concurrent `purge-spam` that deletes the repo (row + dir + archive) under From 2b10fa0ac7cb5f976596cc7b51c04b3409f58aa6 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 16 Jul 2026 10:26:42 -0500 Subject: [PATCH 25/61] refactor(node): dedup limiter setup + write-brake test driver; fix .env wording - .env.example: the write brake wraps the whole /graphql route, so its docs now say it covers all GraphQL HTTP requests (queries and the playground GET), not only mutations, with WebSocket subscriptions called out as excluded. - main.rs: collapse the identical create/write/push per-IP limiter setup into build_ip_limiter (resolve-env, build bounded, warn-on-zero), preserving the exact limits/defaults and the present-but-unparseable warning in rate_limit_from_env; each limiter now also info-logs its configured value. - api/repos.rs: the nine per-IP write-brake tests share a send_from helper mirroring rate_limit.rs's post_from/post_with, replacing the repeated request-build + ConnectInfo + oneshot + status boilerplate. Behavior identical. --- .env.example | 7 +- crates/gitlawb-node/src/api/repos.rs | 228 ++++++++++++++------------- crates/gitlawb-node/src/main.rs | 48 +++--- 3 files changed, 147 insertions(+), 136 deletions(-) diff --git a/.env.example b/.env.example index e4593025..c0820201 100644 --- a/.env.example +++ b/.env.example @@ -143,8 +143,11 @@ GITLAWB_CREATE_RATE_LIMIT=120 # ── Write rate limiting (non-creation authenticated writes) ─────────────── # Max non-creation write requests per client IP per hour: issue/PR comments, # labels, stars, merges, protect/unprotect, replicas, visibility, tasks, -# bounties, profile, and GraphQL mutations. Its own bucket, separate from the -# creation and push brakes. Uses GITLAWB_TRUSTED_PROXY to resolve the client IP. +# bounties, profile, and all GraphQL HTTP requests. The brake wraps the whole +# /graphql route, so queries and the playground GET consume this bucket too, not +# only mutations (GraphQL WebSocket subscriptions are excluded). Its own bucket, +# separate from the creation and push brakes. Uses GITLAWB_TRUSTED_PROXY to +# resolve the client IP. # NOTE: this is a per-IP aggregate across ALL those write actions, so behind a # shared NAT/egress IP (or with GITLAWB_TRUSTED_PROXY unset) many users collapse # onto one bucket — raise this for automation-heavy or multi-user single-IP diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 3e4b3256..1656a83b 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2807,14 +2807,35 @@ mod tests { ); } + // Shared request driver for the per-IP write-brake tests below: build a + // request with the given method/uri/headers/body, attach the socket peer as + // ConnectInfo (what the IP limiter keys on), send it through the router, and + // return the status. Mirrors post_from/post_with in rate_limit.rs. + async fn send_from( + router: &axum::Router, + method: axum::http::Method, + uri: &str, + headers: &[(&str, &str)], + body: axum::body::Body, + peer: std::net::SocketAddr, + ) -> axum::http::StatusCode { + use tower::ServiceExt; + let mut b = axum::http::Request::builder().method(method).uri(uri); + for (k, v) in headers { + b = b.header(*k, *v); + } + let mut req = b.body(body).unwrap(); + req.extensions_mut() + .insert(axum::extract::ConnectInfo(peer)); + router.clone().oneshot(req).await.unwrap().status() + } + #[sqlx::test] async fn write_route_is_rate_limited_by_ip(pool: sqlx::PgPool) { use axum::body::Body; - use axum::extract::ConnectInfo; - use axum::http::{Method, Request, StatusCode}; + use axum::http::{Method, StatusCode}; use std::net::SocketAddr; use std::time::Duration; - use tower::ServiceExt; let mut state = crate::test_support::test_state(pool).await; // Tiny write bucket, keyed on the socket peer (no trusted proxy). @@ -2828,14 +2849,15 @@ mod tests { let router = crate::server::build_router(state); // A write_routes sink (star). The IP brake is outermost, so the 429 // fires before auth/handler — the path only needs to match. - let mut req = Request::builder() - .method(Method::PUT) - .uri("/api/v1/repos/someowner/somerepo/star") - .body(Body::empty()) - .unwrap(); - req.extensions_mut().insert(ConnectInfo(peer)); - - let status = router.oneshot(req).await.unwrap().status(); + let status = send_from( + &router, + Method::PUT, + "/api/v1/repos/someowner/somerepo/star", + &[], + Body::empty(), + peer, + ) + .await; assert_eq!( status, StatusCode::TOO_MANY_REQUESTS, @@ -2848,11 +2870,9 @@ mod tests { #[sqlx::test] async fn write_flood_does_not_drain_creation_budget(pool: sqlx::PgPool) { use axum::body::Body; - use axum::extract::ConnectInfo; - use axum::http::{Method, Request, StatusCode}; + use axum::http::{Method, StatusCode}; use std::net::SocketAddr; use std::time::Duration; - use tower::ServiceExt; let mut state = crate::test_support::test_state(pool).await; // Exhaust the write bucket for this peer; leave the creation bucket ample. @@ -2869,29 +2889,31 @@ mod tests { // Anchor the test: prove the write bucket is genuinely drained at the // router (a write sink from this peer 429s) so the creation assertion // below cannot pass vacuously on some unrelated non-429 status. - let mut wreq = Request::builder() - .method(Method::PUT) - .uri("/api/v1/repos/someowner/somerepo/star") - .body(Body::empty()) - .unwrap(); - wreq.extensions_mut().insert(ConnectInfo(peer)); assert_eq!( - router.clone().oneshot(wreq).await.unwrap().status(), + send_from( + &router, + Method::PUT, + "/api/v1/repos/someowner/somerepo/star", + &[], + Body::empty(), + peer, + ) + .await, StatusCode::TOO_MANY_REQUESTS, "write bucket must be drained for this peer (test precondition)" ); // Creation from the same peer must NOT be 429 — its bucket is untouched. // (It fails later on missing signature; the point is it is not throttled.) - let mut req = Request::builder() - .method(Method::POST) - .uri("/api/v1/repos") - .header("content-type", "application/json") - .body(Body::from(r#"{"name":"legit","is_public":true}"#)) - .unwrap(); - req.extensions_mut().insert(ConnectInfo(peer)); - - let status = router.oneshot(req).await.unwrap().status(); + let status = send_from( + &router, + Method::POST, + "/api/v1/repos", + &[("content-type", "application/json")], + Body::from(r#"{"name":"legit","is_public":true}"#), + peer, + ) + .await; assert_ne!( status, StatusCode::TOO_MANY_REQUESTS, @@ -2903,11 +2925,9 @@ mod tests { #[sqlx::test] async fn graphql_post_is_rate_limited_by_ip(pool: sqlx::PgPool) { use axum::body::Body; - use axum::extract::ConnectInfo; - use axum::http::{Method, Request, StatusCode}; + use axum::http::{Method, StatusCode}; use std::net::SocketAddr; use std::time::Duration; - use tower::ServiceExt; let mut state = crate::test_support::test_state(pool).await; state.write_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); @@ -2917,15 +2937,15 @@ mod tests { assert!(state.write_rate_limiter.check(&peer.ip().to_string()).await); let router = crate::server::build_router(state); - let mut req = Request::builder() - .method(Method::POST) - .uri("/graphql") - .header("content-type", "application/json") - .body(Body::from(r#"{"query":"{ __typename }"}"#)) - .unwrap(); - req.extensions_mut().insert(ConnectInfo(peer)); - - let status = router.oneshot(req).await.unwrap().status(); + let status = send_from( + &router, + Method::POST, + "/graphql", + &[("content-type", "application/json")], + Body::from(r#"{"query":"{ __typename }"}"#), + peer, + ) + .await; assert_eq!( status, StatusCode::TOO_MANY_REQUESTS, @@ -2938,11 +2958,9 @@ mod tests { #[sqlx::test] async fn issue_comment_is_rate_limited_by_ip(pool: sqlx::PgPool) { use axum::body::Body; - use axum::extract::ConnectInfo; - use axum::http::{Method, Request, StatusCode}; + use axum::http::{Method, StatusCode}; use std::net::SocketAddr; use std::time::Duration; - use tower::ServiceExt; let mut state = crate::test_support::test_state(pool).await; state.write_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); @@ -2952,15 +2970,15 @@ mod tests { assert!(state.write_rate_limiter.check(&peer.ip().to_string()).await); let router = crate::server::build_router(state); - let mut req = Request::builder() - .method(Method::POST) - .uri("/api/v1/repos/someowner/somerepo/issues/1/comments") - .header("content-type", "application/json") - .body(Body::from(r#"{"body":"flood"}"#)) - .unwrap(); - req.extensions_mut().insert(ConnectInfo(peer)); - - let status = router.oneshot(req).await.unwrap().status(); + let status = send_from( + &router, + Method::POST, + "/api/v1/repos/someowner/somerepo/issues/1/comments", + &[("content-type", "application/json")], + Body::from(r#"{"body":"flood"}"#), + peer, + ) + .await; assert_eq!( status, StatusCode::TOO_MANY_REQUESTS, @@ -2974,11 +2992,9 @@ mod tests { #[sqlx::test] async fn under_limit_write_is_not_throttled(pool: sqlx::PgPool) { use axum::body::Body; - use axum::extract::ConnectInfo; - use axum::http::{Method, Request, StatusCode}; + use axum::http::{Method, StatusCode}; use std::net::SocketAddr; use std::time::Duration; - use tower::ServiceExt; let mut state = crate::test_support::test_state(pool).await; // Ample budget; bucket NOT exhausted. @@ -2988,14 +3004,16 @@ mod tests { let peer: SocketAddr = "203.0.113.150:7000".parse().unwrap(); let router = crate::server::build_router(state); - let mut req = Request::builder() - .method(Method::PUT) - .uri("/api/v1/repos/someowner/somerepo/star") - .body(Body::empty()) - .unwrap(); - req.extensions_mut().insert(ConnectInfo(peer)); assert_ne!( - router.oneshot(req).await.unwrap().status(), + send_from( + &router, + Method::PUT, + "/api/v1/repos/someowner/somerepo/star", + &[], + Body::empty(), + peer, + ) + .await, StatusCode::TOO_MANY_REQUESTS, "an under-limit write must pass the brake, not be 429'd" ); @@ -3006,11 +3024,9 @@ mod tests { #[sqlx::test] async fn write_rate_limit_zero_disables_the_brake(pool: sqlx::PgPool) { use axum::body::Body; - use axum::extract::ConnectInfo; - use axum::http::{Method, Request, StatusCode}; + use axum::http::{Method, StatusCode}; use std::net::SocketAddr; use std::time::Duration; - use tower::ServiceExt; let mut state = crate::test_support::test_state(pool).await; state.write_rate_limiter = crate::rate_limit::RateLimiter::new(0, Duration::from_secs(60)); @@ -3019,14 +3035,16 @@ mod tests { let router = crate::server::build_router(state); for _ in 0..5 { - let mut req = Request::builder() - .method(Method::PUT) - .uri("/api/v1/repos/someowner/somerepo/star") - .body(Body::empty()) - .unwrap(); - req.extensions_mut().insert(ConnectInfo(peer)); assert_ne!( - router.clone().oneshot(req).await.unwrap().status(), + send_from( + &router, + Method::PUT, + "/api/v1/repos/someowner/somerepo/star", + &[], + Body::empty(), + peer, + ) + .await, StatusCode::TOO_MANY_REQUESTS, "a 0 write limit must disable the brake" ); @@ -3038,11 +3056,9 @@ mod tests { #[sqlx::test] async fn task_bounty_profile_writes_are_rate_limited(pool: sqlx::PgPool) { use axum::body::Body; - use axum::extract::ConnectInfo; - use axum::http::{Method, Request, StatusCode}; + use axum::http::{Method, StatusCode}; use std::net::SocketAddr; use std::time::Duration; - use tower::ServiceExt; let mut state = crate::test_support::test_state(pool).await; state.write_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); @@ -3056,15 +3072,16 @@ mod tests { (Method::POST, "/api/v1/repos/o/r/bounties"), (Method::PUT, "/api/v1/profile"), ] { - let mut req = Request::builder() - .method(method) - .uri(uri) - .header("content-type", "application/json") - .body(Body::from("{}")) - .unwrap(); - req.extensions_mut().insert(ConnectInfo(peer)); assert_eq!( - router.clone().oneshot(req).await.unwrap().status(), + send_from( + &router, + method, + uri, + &[("content-type", "application/json")], + Body::from("{}"), + peer, + ) + .await, StatusCode::TOO_MANY_REQUESTS, "write group {uri} must be IP-throttled by the write brake" ); @@ -3076,11 +3093,9 @@ mod tests { #[sqlx::test] async fn graphql_ws_is_not_braked(pool: sqlx::PgPool) { use axum::body::Body; - use axum::extract::ConnectInfo; - use axum::http::{Method, Request, StatusCode}; + use axum::http::{Method, StatusCode}; use std::net::SocketAddr; use std::time::Duration; - use tower::ServiceExt; let mut state = crate::test_support::test_state(pool).await; state.write_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); @@ -3089,16 +3104,18 @@ mod tests { assert!(state.write_rate_limiter.check(&peer.ip().to_string()).await); let router = crate::server::build_router(state); - let mut req = Request::builder() - .method(Method::GET) - .uri("/graphql/ws") - .body(Body::empty()) - .unwrap(); - req.extensions_mut().insert(ConnectInfo(peer)); // Not a real ws upgrade, so the subscription service rejects it with some // non-429 status; the point is the write brake never sees it. assert_ne!( - router.oneshot(req).await.unwrap().status(), + send_from( + &router, + Method::GET, + "/graphql/ws", + &[], + Body::empty(), + peer + ) + .await, StatusCode::TOO_MANY_REQUESTS, "/graphql/ws must not be behind the write brake" ); @@ -3110,11 +3127,9 @@ mod tests { #[sqlx::test] async fn every_write_group_passes_under_limit(pool: sqlx::PgPool) { use axum::body::Body; - use axum::extract::ConnectInfo; - use axum::http::{Method, Request, StatusCode}; + use axum::http::{Method, StatusCode}; use std::net::SocketAddr; use std::time::Duration; - use tower::ServiceExt; let mut state = crate::test_support::test_state(pool).await; state.write_rate_limiter = @@ -3131,15 +3146,16 @@ mod tests { (Method::POST, "/api/v1/repos/o/r/bounties"), (Method::PUT, "/api/v1/profile"), ] { - let mut req = Request::builder() - .method(method) - .uri(uri) - .header("content-type", "application/json") - .body(Body::from(r#"{"query":"{ __typename }"}"#)) - .unwrap(); - req.extensions_mut().insert(ConnectInfo(peer)); assert_ne!( - router.clone().oneshot(req).await.unwrap().status(), + send_from( + &router, + method, + uri, + &[("content-type", "application/json")], + Body::from(r#"{"query":"{ __typename }"}"#), + peer, + ) + .await, StatusCode::TOO_MANY_REQUESTS, "under-limit write to {uri} must pass the brake, not 429" ); diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index f85eaded..83656be1 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -303,15 +303,7 @@ async fn main() -> Result<()> { // capped regardless of how many identities it mints. Sized well above any // legitimate per-IP creation rate; GITLAWB_CREATE_RATE_LIMIT overrides, 0 // disables. Bounded key set — the key is a client-influenced IP. - let create_limit = rate_limit_from_env("GITLAWB_CREATE_RATE_LIMIT", 120); - let create_ip_rate_limiter = rate_limit::RateLimiter::new_bounded( - create_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - if create_limit == 0 { - tracing::warn!("GITLAWB_CREATE_RATE_LIMIT=0 — per-IP creation rate limiting disabled"); - } + let create_ip_rate_limiter = build_ip_limiter("GITLAWB_CREATE_RATE_LIMIT", 120, "creation"); // Per-IP brake for the authenticated non-creation write surface (issue/PR // comments, labels, stars, merges, protect, replicas, visibility, tasks, @@ -321,30 +313,14 @@ async fn main() -> Result<()> { // any legitimate per-IP write rate — real agent automation makes many small // writes per hour. GITLAWB_WRITE_RATE_LIMIT overrides; 0 disables. Bounded // key set — the key is a client-influenced IP. - let write_limit = rate_limit_from_env("GITLAWB_WRITE_RATE_LIMIT", 600); - let write_rate_limiter = rate_limit::RateLimiter::new_bounded( - write_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - if write_limit == 0 { - tracing::warn!("GITLAWB_WRITE_RATE_LIMIT=0 — per-IP write rate limiting disabled"); - } + let write_rate_limiter = build_ip_limiter("GITLAWB_WRITE_RATE_LIMIT", 600, "write"); // Push-path flood brake: max git-receive-pack requests per client IP per // hour (counts both the info/refs advertisement and the push POST). Sized // for heavy agent automation while still stopping flood traffic (the June // 2026 attack pushed several times per second per IP). GITLAWB_PUSH_RATE_LIMIT // overrides; 0 disables. Bounded key set — the key is a client-influenced IP. - let push_limit = rate_limit_from_env("GITLAWB_PUSH_RATE_LIMIT", 600); - let push_rate_limiter = rate_limit::RateLimiter::new_bounded( - push_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - if push_limit == 0 { - tracing::warn!("GITLAWB_PUSH_RATE_LIMIT=0 — per-IP push rate limiting disabled"); - } + let push_rate_limiter = build_ip_limiter("GITLAWB_PUSH_RATE_LIMIT", 600, "push"); // Which forwarded header the edge is trusted to set. Default None (trust // nothing, key on the socket peer). Fly nodes set GITLAWB_TRUSTED_PROXY=fly; @@ -352,7 +328,7 @@ async fn main() -> Result<()> { let push_limiter_trust = rate_limit::TrustedProxy::from_env_value( &std::env::var("GITLAWB_TRUSTED_PROXY").unwrap_or_default(), ); - tracing::info!(trust = ?push_limiter_trust, push_limit, "push rate limiter configured"); + tracing::info!(trust = ?push_limiter_trust, "push rate limiter trust configured"); // Peer-sync flood brakes, keyed on the resolved client IP (per-DID is useless // here — a did:key farm self-registers). Two buckets so an unsigned notify @@ -612,6 +588,22 @@ fn rate_limit_from_env(var: &str, default: usize) -> usize { value } +/// Build a bounded per-client-IP rate limiter from an env var: resolve the limit +/// (honoring the present-but-unparseable warning in `rate_limit_from_env`), build +/// the limiter with the shared 1-hour window and 200k-key cap, and warn when the +/// limit is 0 (disabled). Collapses the identical create/write/push setup. +fn build_ip_limiter(var: &str, default: usize, label: &str) -> rate_limit::RateLimiter { + let limit = rate_limit_from_env(var, default); + let limiter = + rate_limit::RateLimiter::new_bounded(limit, std::time::Duration::from_secs(3600), 200_000); + if limit == 0 { + tracing::warn!("{var}=0 — per-IP {label} rate limiting disabled"); + } else { + tracing::info!(%var, limit, "per-IP {label} rate limiter configured"); + } + limiter +} + /// Dispatch an admin subcommand. Connects the database directly (no server, no /// p2p, no metrics) and runs the requested tool to completion, then exits. async fn run_admin_command(command: config::Command, config: &Config) -> Result<()> { From ed7489e807b08c752ce6064439c96c369ebf15c1 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 17:53:33 -0500 Subject: [PATCH 26/61] docs(node): drop the .env.example shutdown-grace entry #196 added (#196) The GITLAWB_SHUTDOWN_GRACE_SECS field, its doc-comment, and main.rs's grace handling pre-date this PR (#22); #196's diff only added the .env.example entry, whose comment promised a 503-on-expiry the node does not implement. Remove just that entry. PR #194 owns the real bounded drain and the config prose in disjoint files. --- .env.example | 3 --- 1 file changed, 3 deletions(-) diff --git a/.env.example b/.env.example index c0820201..9b23cff9 100644 --- a/.env.example +++ b/.env.example @@ -17,9 +17,6 @@ GITLAWB_PORT=7545 # 127.0.0.1:9091). Leave empty (default) to disable. Bind to localhost or a # private interface — the metrics endpoint is unauthenticated. GITLAWB_METRICS_ADDR= -# Max seconds to wait for in-flight requests to drain on shutdown before the -# server returns 503 to anything still in flight and exits. Default: 30. -GITLAWB_SHUTDOWN_GRACE_SECS=30 # ── Storage ─────────────────────────────────────────────────────────────── GITLAWB_REPOS_DIR=/data/repos From de16ed5a0bb5a8ec199ea1600b7c50b5b4b74cec Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 18:25:10 -0500 Subject: [PATCH 27/61] fix(node): give advisory-lock guards a dedicated pool and bound the release upload (#196) Repo-write guards pinned a connection from the shared app pool for the whole receive-pack plus an untimed archive upload, so ~20 concurrent distinct-repo pushes starved unrelated DB handlers. Guards now draw from a dedicated lock_pool (GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS, default 32, sized to peak writers) separate from the app pool that serves handlers and the purge delete, and release()'s Tigris upload is wrapped in a timeout so the connection hold is bounded on every path. The guard still locks and unlocks on its single held connection (no off-session unlock). Db::pool() is now test-only since nothing in the bin reads the raw app pool directly. --- .env.example | 6 + crates/gitlawb-node/src/config.rs | 16 ++ crates/gitlawb-node/src/db/mod.rs | 30 ++- crates/gitlawb-node/src/git/repo_store.rs | 253 ++++++++++++++++++++-- crates/gitlawb-node/src/main.rs | 32 ++- 5 files changed, 314 insertions(+), 23 deletions(-) diff --git a/.env.example b/.env.example index 9b23cff9..74c9c913 100644 --- a/.env.example +++ b/.env.example @@ -33,6 +33,12 @@ DATABASE_URL=postgresql://gitlawb:changeme@localhost:5432/gitlawb # connections open lazily. Size against the DB server's max_connections, # remembering admin tooling opens its own pool. GITLAWB_DB_MAX_CONNECTIONS=20 +# Maximum connections in the dedicated advisory-lock pool (separate from the +# pool above). Each in-flight repo write pins one connection here for its whole +# lifetime, so size it to peak concurrent distinct-repo writers — keeping it +# separate is what stops a push burst from starving request handlers. Keep +# (main pool + lock pool) within the DB server's max_connections. +GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS=32 # Seconds a request waits for a pool connection before failing with 503. GITLAWB_DB_ACQUIRE_TIMEOUT_SECS=5 # Upper bound on each startup connect+migrate attempt, in seconds. Keep it diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 06ff5081..7dd77876 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -221,6 +221,22 @@ pub struct Config { )] pub db_max_connections: u32, + /// Maximum connections in the dedicated advisory-lock pool, separate from + /// the main pool above. Every in-flight repo write (git-receive-pack, fork, + /// merge) pins one connection here for its whole lifetime — the write, its + /// metadata/fan-out tail, and the bounded post-write archive upload — so + /// size it to the expected peak number of concurrent distinct-repo writers, + /// NOT small. Keeping it separate from GITLAWB_DB_MAX_CONNECTIONS is what + /// stops a push burst from starving normal request handlers. Keep + /// (main pool + lock pool) within the database server's max_connections. + #[arg( + long, + env = "GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS", + default_value_t = 32, + value_parser = clap::value_parser!(u32).range(1..) + )] + pub db_lock_pool_max_connections: u32, + /// Maximum time a request waits for a pool connection before failing with /// 503, in seconds. Bounds queueing when the database is slow or down. #[arg( diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index a963f544..def7a394 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -247,7 +247,11 @@ pub struct Db { } impl Db { - /// Access the underlying Postgres connection pool. + /// Access the underlying Postgres connection pool. Test-only: production DB + /// access goes through `Db`'s methods, and the advisory-lock subsystem now + /// uses its own pool (`connect_lock_pool`), so nothing outside tests reaches + /// for the raw app pool. + #[cfg(test)] pub fn pool(&self) -> &PgPool { &self.pool } @@ -292,6 +296,30 @@ impl Db { Ok(db) } + /// Build a standalone Postgres pool for the advisory-lock subsystem, + /// separate from the app pool. No migrations run here (the app pool owns + /// the schema); this pool only serves the session-scoped advisory-lock + /// connections that `RepoStore`'s write guards pin for a write's whole + /// lifetime, so a concurrent-push burst pins connections from here rather + /// than starving the app pool. Sized by GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS. + pub async fn connect_lock_pool( + database_url: &str, + max_connections: u32, + acquire_timeout: Duration, + ) -> Result { + info!( + max_connections, + acquire_timeout_secs = acquire_timeout.as_secs(), + "connecting advisory-lock pool" + ); + PgPoolOptions::new() + .max_connections(max_connections) + .acquire_timeout(acquire_timeout) + .connect(database_url) + .await + .context("connecting advisory-lock pool") + } + /// Cheap liveness probe against the pool, for readiness checks: one /// `SELECT 1` that fails fast when the database is unreachable. pub async fn ping(&self) -> Result<()> { diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index f9a57c3a..0f4e2284 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -20,13 +20,26 @@ use tracing::{debug, info, warn}; use super::store; use super::tigris::ObjectStore; +/// Default bound on `release()`'s post-write archive upload (INV-22). Generous +/// for a large-repo PUT while still guaranteeing the guard's pinned advisory- +/// lock connection returns to the pool within a fixed window even if the upload +/// stalls indefinitely. +const DEFAULT_RELEASE_UPLOAD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300); + /// Centralized repo storage: local disk cache + optional object-store backend. #[derive(Clone)] pub struct RepoStore { repos_dir: PathBuf, object_store: Option>, - /// Shared Postgres pool for advisory locks. - pool: PgPool, + /// Dedicated Postgres pool for advisory-lock guard connections, separate + /// from the app pool that serves normal request handlers. Each write guard + /// pins one connection from here for its whole lifetime, so a concurrent- + /// push burst pins connections from this pool rather than starving the app + /// pool. Sized by GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS to peak writers. + lock_pool: PgPool, + /// Upper bound on the post-write archive upload in `release()`, so a stalled + /// upload cannot pin the guard's lock connection indefinitely. + release_upload_timeout: std::time::Duration, /// Tracks repos already confirmed to exist in the object store — avoids /// redundant HEAD checks and background uploads for repos we've migrated. migrated: Arc>>, @@ -34,11 +47,12 @@ pub struct RepoStore { impl RepoStore { #[cfg(test)] - pub fn for_testing(repos_dir: PathBuf, pool: PgPool) -> Self { + pub fn for_testing(repos_dir: PathBuf, lock_pool: PgPool) -> Self { Self { repos_dir, object_store: None, - pool, + lock_pool, + release_upload_timeout: DEFAULT_RELEASE_UPLOAD_TIMEOUT, migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), } } @@ -46,16 +60,25 @@ impl RepoStore { pub fn new( repos_dir: PathBuf, object_store: Option>, - pool: PgPool, + lock_pool: PgPool, ) -> Self { Self { repos_dir, object_store, - pool, + lock_pool, + release_upload_timeout: DEFAULT_RELEASE_UPLOAD_TIMEOUT, migrated: Arc::new(Mutex::new(HashSet::new())), } } + /// Shrink the `release()` upload timeout — test-only, so a stall test can + /// assert the lock connection is freed within a short bound. + #[cfg(test)] + pub fn with_release_upload_timeout(mut self, timeout: std::time::Duration) -> Self { + self.release_upload_timeout = timeout; + self + } + /// Ensure a repo is available on local disk, downloading from Tigris if needed. /// If the repo exists locally but not yet in Tigris, a background upload is /// spawned to lazily migrate it (on-demand migration for pre-Tigris repos). @@ -174,14 +197,14 @@ impl RepoStore { // // Between FAILED attempts the connection is returned to the pool (the // `drop` below) so a writer spinning on a contended repo does NOT hold a - // pool connection through the retry backoff — holding one per spinner - // would starve the shared pool under a same-repo write burst. Only the - // WINNING attempt keeps its connection (the lock lives on it). + // lock-pool connection through the retry backoff — holding one per + // spinner would starve the lock pool under a same-repo write burst. Only + // the WINNING attempt keeps its connection (the lock lives on it). let conn = { let mut acquired = None; for attempt in 0..60 { let mut c = self - .pool + .lock_pool .acquire() .await .context("acquiring write-lock connection")?; @@ -220,6 +243,7 @@ impl RepoStore { lock_key, conn: Some(conn), object_store: self.object_store.clone(), + release_upload_timeout: self.release_upload_timeout, }; // Always download the latest from the object store before writing. @@ -272,7 +296,7 @@ impl RepoStore { let (owner_slug, _local) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); let mut conn = self - .pool + .lock_pool .acquire() .await .context("acquiring lock connection")?; @@ -560,6 +584,9 @@ pub struct RepoWriteGuard { /// it to the pool) while `Drop` closes it when release was never reached. conn: Option>, object_store: Option>, + /// Upper bound on the `release()` archive upload — copied from the store so + /// a stalled upload cannot pin this connection past the bound. + release_upload_timeout: std::time::Duration, } impl RepoWriteGuard { @@ -574,14 +601,26 @@ impl RepoWriteGuard { /// Tigris (and to every node that later downloads it). The lock is always /// released regardless, to avoid stale locks blocking future writes. pub async fn release(mut self, success: bool) { - // Upload to Tigris only on success. + // Upload to Tigris only on success. Bound the upload so a stalled PUT + // cannot pin the guard's advisory-lock connection indefinitely — on + // timeout we log and fall through to the unlock, returning the + // connection to the pool within the bound. (INV-22.) if success { if let Some(ref tigris) = self.object_store { - if let Err(e) = tigris - .upload(&self.owner_slug, &self.repo_name, &self.local_path) - .await + match tokio::time::timeout( + self.release_upload_timeout, + tigris.upload(&self.owner_slug, &self.repo_name, &self.local_path), + ) + .await { - warn!(repo = %self.repo_name, err = %e, "failed to upload repo to tigris after write"); + Ok(Ok(())) => {} + Ok(Err(e)) => { + warn!(repo = %self.repo_name, err = %e, "failed to upload repo to tigris after write"); + } + Err(_) => { + warn!(repo = %self.repo_name, timeout_secs = self.release_upload_timeout.as_secs(), + "tigris upload timed out after write — releasing the advisory lock without a completed upload"); + } } } } else { @@ -924,6 +963,9 @@ mod tests { // it true — lets U3's resurrect test assert a deleted archive stays gone. exists: std::sync::Arc, download_gate: Option>, + // When set, `upload()` parks on this gate before doing anything — models a + // stalled PUT so the release-upload timeout can be observed (U2). + upload_gate: Option>, uploads: std::sync::Arc>>, deletes: std::sync::Arc>>, } @@ -933,6 +975,7 @@ mod tests { Self { exists: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(exists)), download_gate: None, + upload_gate: None, uploads: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), deletes: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), } @@ -945,6 +988,10 @@ mod tests { Ok(self.exists.load(std::sync::atomic::Ordering::SeqCst)) } async fn upload(&self, o: &str, r: &str, _p: &std::path::Path) -> anyhow::Result<()> { + if let Some(gate) = &self.upload_gate { + // Park indefinitely — the caller's timeout must be what unblocks. + gate.notified().await; + } self.uploads .lock() .unwrap() @@ -1349,4 +1396,178 @@ mod tests { "an uncontended fork upload must PUT exactly once" ); } + + // ── U2: dedicated lock pool + bounded release upload (R2, KTD2) ────────── + + // A no-reap pool sized to `max_connections` with a short acquire timeout, so + // a held guard's connection is not reclaimed mid-assertion (see + // raw-advisory-lock-guard-must-free-on-drop-and-ambient-pool-reap-masks-the-leak.md) + // and an exhausted pool fails fast instead of stalling the whole test. + async fn sized_no_reap_pool( + connect_opts: &sqlx::postgres::PgConnectOptions, + max_connections: u32, + ) -> PgPool { + sqlx::postgres::PgPoolOptions::new() + .max_connections(max_connections) + .acquire_timeout(std::time::Duration::from_secs(2)) + .min_connections(0) + .idle_timeout(None) + .max_lifetime(None) + .test_before_acquire(false) + .connect_with(connect_opts.clone()) + .await + .unwrap() + } + + // R2/KTD2: advisory-lock guards pin connections from the DEDICATED lock pool, + // so N concurrent distinct-repo write guards do NOT consume the app pool — an + // unrelated app-pool handler query still gets a connection. On the pre-split + // base (the store shared the app pool, RepoStore::new(_, _, db.pool())) those + // N guards pinned every app-pool connection and the unrelated query 503'd on + // acquire; the `shared_pool_starves_*` control below pins that RED behavior. + #[sqlx::test] + async fn write_guards_on_lock_pool_do_not_starve_the_app_pool( + _pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + const N: u32 = 3; + let lock_pool = sized_no_reap_pool(&connect_opts, N).await; + let app_pool = sized_no_reap_pool(&connect_opts, N).await; + let tmp = tempfile::TempDir::new().unwrap(); + let store = RepoStore::new(tmp.path().to_path_buf(), None, lock_pool.clone()); + + // Hold N distinct-repo write guards -> N lock-pool connections pinned. + let mut guards = Vec::new(); + for i in 0..N { + let owner = format!("did:key:z6MkStarveApp{i}AAAAAAAAAAAAAAAAAAAAAAAA"); + guards.push(store.acquire_write(&owner, "starve").await.unwrap()); + } + + // The lock pool is now exhausted: a further acquire on it times out — + // proving the guards really pinned all N of its connections (so the app- + // pool assertion below is load-bearing, not vacuously green on idle). + assert!( + lock_pool.acquire().await.is_err(), + "the lock pool must be exhausted by N held write guards" + ); + + // ...but the app pool is a SEPARATE pool, so an unrelated handler query + // still acquires a connection. This is the query that starved pre-split. + assert!( + app_pool.acquire().await.is_ok(), + "the app pool must stay available while N write guards are held on the lock pool" + ); + + for g in guards { + g.release(true).await; + } + } + + // The pre-split hazard, pinned as a characterization test: when the store + // SHARES its guard pool with the app pool (the pre-U2 wiring), N held write + // guards pin every connection and an unrelated query on that SAME pool + // starves (acquire times out). This is the RED the dedicated lock pool + // closes; keeping it green documents WHY the pools must be split. + #[sqlx::test] + async fn shared_pool_starves_the_app_pool_under_held_write_guards( + _pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + const N: u32 = 3; + let shared = sized_no_reap_pool(&connect_opts, N).await; + let tmp = tempfile::TempDir::new().unwrap(); + // Pre-split topology: the store's guard pool IS the app pool. + let store = RepoStore::new(tmp.path().to_path_buf(), None, shared.clone()); + + let mut guards = Vec::new(); + for i in 0..N { + let owner = format!("did:key:z6MkSharedStarve{i}AAAAAAAAAAAAAAAAAAAAAA"); + guards.push(store.acquire_write(&owner, "starve").await.unwrap()); + } + + // An unrelated query on the SAME pool now starves. + assert!( + shared.acquire().await.is_err(), + "a shared pool must starve under N held write guards — the hazard the split closes" + ); + + for g in guards { + g.release(true).await; + } + } + + // R2: the lock pool is sized to peak writers, so N = pool-size concurrent + // distinct-repo writers all acquire and proceed (the starvation is not + // merely relocated from the app pool onto pushers). + #[sqlx::test] + async fn lock_pool_admits_peak_writers_concurrently( + _pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + const N: u32 = 4; + let lock_pool = sized_no_reap_pool(&connect_opts, N).await; + let tmp = tempfile::TempDir::new().unwrap(); + let store = RepoStore::new(tmp.path().to_path_buf(), None, lock_pool); + + let mut guards = Vec::new(); + for i in 0..N { + let owner = format!("did:key:z6MkPeakWriter{i}AAAAAAAAAAAAAAAAAAAAAAAA"); + guards.push( + store + .acquire_write(&owner, "peak") + .await + .expect("every writer up to the lock-pool size must acquire and proceed"), + ); + } + assert_eq!(guards.len(), N as usize); + for g in guards { + g.release(true).await; + } + } + + // R2/INV-22: a stalled release() upload must not pin the guard's advisory- + // lock connection indefinitely. With a short upload bound and an upload that + // never completes, release() still returns and frees the lock within the + // bound. Pre-fix (untimed upload) release() would hang on the stalled PUT + // and never reach the unlock. + #[sqlx::test] + async fn release_upload_stall_is_bounded( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + // The upload parks forever; the release timeout is what must unblock it. + ts.upload_gate = Some(std::sync::Arc::new(tokio::sync::Notify::new())); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ) + .with_release_upload_timeout(std::time::Duration::from_millis(300)); + let owner = "did:key:z6MkReleaseStallBoundAAAAAAAAAAAAAAAAAAAAAA"; + let name = "stall"; + let key = lock_key_for(owner, name); + let mut observer = connect_opts.connect().await.unwrap(); + + let guard = store.acquire_write(owner, name).await.unwrap(); + assert!( + !advisory_lock_is_free(&mut observer, key).await, + "the lock must be held while the guard is alive" + ); + + // release(success) -> the upload stalls, but the bound caps the hold. + let start = std::time::Instant::now(); + guard.release(true).await; + assert!( + start.elapsed() < std::time::Duration::from_secs(5), + "release() must return within the upload bound, not hang on the stalled PUT" + ); + assert!( + poll_until_free(&mut observer, key).await, + "the advisory lock must be freed after the bounded (timed-out) upload" + ); + } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 83656be1..c27ad3d0 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -288,8 +288,20 @@ async fn main() -> Result<()> { let object_store = tigris.map(|t| std::sync::Arc::new(t) as std::sync::Arc); + // Advisory-lock guards get their own pool, separate from the app pool + // (db.pool()) that serves normal handlers, so a concurrent-push burst pins + // lock connections here instead of starving request handlers. Sized to peak + // writers via GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS. + let lock_pool_acquire_timeout = std::time::Duration::from_secs(config.db_acquire_timeout_secs); + let lock_pool = Db::connect_lock_pool( + &config.database_url, + config.db_lock_pool_max_connections, + lock_pool_acquire_timeout, + ) + .await + .context("connecting advisory-lock pool")?; let repo_store = - git::repo_store::RepoStore::new(config.repos_dir.clone(), object_store, db.pool().clone()); + git::repo_store::RepoStore::new(config.repos_dir.clone(), object_store, lock_pool); // Per-DID limiter for the creation endpoints. Keyed on the authenticated // DID (attacker-varied), so bound its key set to cap memory. @@ -641,11 +653,19 @@ async fn run_admin_command(command: config::Command, config: &Config) -> Result< } } }; - let repo_store = git::repo_store::RepoStore::new( - config.repos_dir.clone(), - object_store, - db.pool().clone(), - ); + // Give the purge-spam store a dedicated advisory-lock pool, separate + // from `db`'s app pool. The guard then holds a lock-pool connection + // while `delete_repo_by_id` runs on the app pool, so a purge at + // GITLAWB_DB_MAX_CONNECTIONS=1 does not self-deadlock (KTD3). + let lock_pool = Db::connect_lock_pool( + &config.database_url, + config.db_lock_pool_max_connections, + acquire_timeout, + ) + .await + .context("connecting advisory-lock pool for purge-spam")?; + let repo_store = + git::repo_store::RepoStore::new(config.repos_dir.clone(), object_store, lock_pool); admin::run_purge_spam(&db, &repo_store, &config.repos_dir, execute) .await .map(|_| ()) From bbfdf2bf3ffbfe0c48a5a78cf316631293ba8f1f Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 18:35:34 -0500 Subject: [PATCH 28/61] fix(node): advertise a window-derived Retry-After on rate-limit 429s (#196) Both 429 sites hardcoded Retry-After: 60 while every limiter uses a one-hour window, so a compliant client retrying at 60s kept hitting 429 for nearly another hour. The limiter now returns the delay from its oldest live timestamp (window - elapsed, clamped >= 1); a cap-full rejection with no bucket advertises the full window. The check() -> bool API is preserved as a wrapper so unrelated callers are untouched. --- crates/gitlawb-node/src/rate_limit.rs | 161 ++++++++++++++++++++++++-- 1 file changed, 150 insertions(+), 11 deletions(-) diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index 1d38a0ae..e191cb3e 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -70,10 +70,21 @@ impl RateLimiter { self.window.min(Duration::from_secs(1)) } + /// Returns `true` if the request is allowed. Thin wrapper over + /// [`check_retry`](Self::check_retry) for callers that only need the + /// allow/deny decision, not the rejection's Retry-After delay. pub(crate) async fn check(&self, key: &str) -> bool { + self.check_retry(key).await.is_none() + } + + /// Record a request against `key`. Returns `None` when it is allowed, or + /// `Some(retry_after)` when it is rejected — the delay a client should wait + /// before retrying, so a 429 can advertise a value consistent with the + /// actual window instead of a constant. + pub(crate) async fn check_retry(&self, key: &str) -> Option { // max_requests == 0 means the limiter is disabled, not "block all". if self.max_requests == 0 { - return true; + return None; } let now = Instant::now(); let mut state = self.state.lock().await; @@ -85,10 +96,25 @@ impl RateLimiter { .timestamps .retain(|t| now.duration_since(*t) < self.window); if window.timestamps.len() >= self.max_requests { - return false; + // Insertion order is preserved, so after the retain the first + // element is the oldest live request. It ages out (freeing a + // slot) one window after it landed, so that remaining time is + // the delay to advertise. Clamp to >= 1 so a 429 never tells a + // client to retry immediately. + let retry = window + .timestamps + .first() + .map(|oldest| { + self.window + .as_secs() + .saturating_sub(now.duration_since(*oldest).as_secs()) + }) + .unwrap_or(0) + .max(1); + return Some(Duration::from_secs(retry)); } window.timestamps.push(now); - return true; + return None; } // New key. Enforce the cap BEFORE inserting so a flood of distinct keys @@ -109,7 +135,11 @@ impl RateLimiter { } drop(last_sweep); if state.len() >= self.max_keys { - return false; + // The key is untracked (never inserted), so there is no oldest + // entry to derive a delay from. Advertise the full window as a + // conservative bound: capacity frees as other keys expire, at + // the latest one window from now. + return Some(self.window.max(Duration::from_secs(1))); } } state.insert( @@ -118,7 +148,7 @@ impl RateLimiter { timestamps: vec![now], }, ); - true + None } pub async fn cleanup(&self) { @@ -148,6 +178,23 @@ impl RateLimiter { pub(crate) async fn tracked_keys(&self) -> usize { self.state.lock().await.len() } + + /// Test-only: seed a key's bucket with entries of the given ages (how long + /// ago each request landed), so a test can drive the Retry-After computation + /// off a controlled oldest-entry age without wall-clock sleeps. Ages that + /// would underflow the monotonic clock collapse to `now`. + #[cfg(test)] + pub(crate) async fn seed_aged_entry_for_test(&self, key: &str, ages: &[Duration]) { + let now = Instant::now(); + let timestamps = ages + .iter() + .map(|a| now.checked_sub(*a).unwrap_or(now)) + .collect(); + self.state + .lock() + .await + .insert(key.to_string(), Window { timestamps }); + } } pub async fn rate_limit_by_did(request: Request, next: Next) -> Response { @@ -159,10 +206,10 @@ pub async fn rate_limit_by_did(request: Request, next: Next) -> Response { .map(|a| a.0.clone()); if let (Some(limiter), Some(did)) = (limiter, did) { - if !limiter.check(&did).await { + if let Some(retry_after) = limiter.check_retry(&did).await { return ( StatusCode::TOO_MANY_REQUESTS, - [("retry-after", "60")], + [("retry-after", retry_after.as_secs().to_string())], "rate limit exceeded — try again later", ) .into_response(); @@ -271,10 +318,10 @@ impl axum::extract::FromRequestParts for PeerAddr { /// The shared 429 response for the per-IP flood brakes. Route-agnostic: this /// middleware now serves the push path AND the peer-sync routes, so the message /// stays generic (the offending path is recorded in the warn log below). -pub fn too_many_requests() -> Response { +pub fn too_many_requests(retry_after: Duration) -> Response { ( StatusCode::TOO_MANY_REQUESTS, - [("retry-after", "60")], + [("retry-after", retry_after.as_secs().to_string())], "rate limit exceeded — try again later", ) .into_response() @@ -292,9 +339,9 @@ pub async fn rate_limit_by_ip(request: Request, next: Next) -> Response { if let Some(limiter) = limiter { if let Some(key) = client_key(request.headers(), peer, limiter.trust) { - if !limiter.limiter.check(&key).await { + if let Some(retry_after) = limiter.limiter.check_retry(&key).await { tracing::warn!(key = %key, path = %request.uri().path(), "per-IP rate limit exceeded"); - return too_many_requests(); + return too_many_requests(retry_after); } } } @@ -723,4 +770,96 @@ mod tests { "with no trusted header, requests fall back to the socket peer key (collapse)" ); } + + // ── Retry-After derived from the window, not a constant 60 (U5) ─────── + // A 429 must advertise the delay until the oldest live request ages out of + // the limiter's window, so the value tracks the real window instead of the + // hardcoded 60 that contradicted every limiter's 3600s window. + + async fn resp_from(router: &axum::Router, peer: SocketAddr) -> Response { + use tower::ServiceExt; + let mut req = axum::http::Request::builder() + .method(axum::http::Method::POST) + .uri("/o/r/git-receive-pack") + .body(axum::body::Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + router.clone().oneshot(req).await.unwrap() + } + + fn retry_after_secs(resp: &Response) -> u64 { + resp.headers() + .get("retry-after") + .expect("429 must carry a retry-after header") + .to_str() + .unwrap() + .parse() + .expect("retry-after must be an integer number of seconds") + } + + #[tokio::test] + async fn retry_after_reflects_full_window_for_freshly_filled_bucket() { + // A just-filled bucket's oldest entry is ~now, so the advertised delay + // must be close to the whole window (100s), NOT the old constant 60. + let router = ip_limited_router(IpRateLimiter { + limiter: RateLimiter::new(1, Duration::from_secs(100)), + trust: TrustedProxy::None, + }); + let peer: SocketAddr = "203.0.113.50:6000".parse().unwrap(); + assert_eq!(resp_from(&router, peer).await.status(), StatusCode::OK); + let resp = resp_from(&router, peer).await; + assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS); + let retry = retry_after_secs(&resp); + assert!( + (95..=100).contains(&retry), + "freshly-filled bucket must advertise ~window (95..=100s), got {retry} (constant-60 bug if 60)" + ); + } + + #[tokio::test] + async fn retry_after_shrinks_as_oldest_entry_ages() { + // Seed the sole slot with an entry already 95s into a 100s window: the + // next request is rejected and must advertise only the ~5s remaining, + // not 60. + let limiter = RateLimiter::new(1, Duration::from_secs(100)); + let peer: SocketAddr = "203.0.113.51:6000".parse().unwrap(); + limiter + .seed_aged_entry_for_test(&peer.ip().to_string(), &[Duration::from_secs(95)]) + .await; + let router = ip_limited_router(IpRateLimiter { + limiter, + trust: TrustedProxy::None, + }); + let resp = resp_from(&router, peer).await; + assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS); + let retry = retry_after_secs(&resp); + assert!( + (1..=10).contains(&retry), + "an aged bucket must advertise the small remaining delay (1..=10s), got {retry}" + ); + } + + #[tokio::test] + async fn retry_after_clamps_to_at_least_one_second() { + // Oldest entry is 99s into a 100s window: the raw remainder rounds toward + // 1s and must never be advertised as 0 (a 0/blank retry lets a rejected + // client hammer immediately). + let limiter = RateLimiter::new(1, Duration::from_secs(100)); + let peer: SocketAddr = "203.0.113.52:6000".parse().unwrap(); + limiter + .seed_aged_entry_for_test(&peer.ip().to_string(), &[Duration::from_secs(99)]) + .await; + let router = ip_limited_router(IpRateLimiter { + limiter, + trust: TrustedProxy::None, + }); + let resp = resp_from(&router, peer).await; + assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS); + let retry = retry_after_secs(&resp); + assert!(retry >= 1, "retry-after must clamp to >= 1, got {retry}"); + assert!( + retry <= 2, + "with 99s of a 100s window elapsed, the remaining delay is ~1s, got {retry}" + ); + } } From 773cd7597996c7c588fa5d4522cdc1af50c64b6d Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 18:50:07 -0500 Subject: [PATCH 29/61] fix(node): skip slug-scoped purge cascade when a mirror row shares the slug (#196) delete_repo_by_id cascaded branch_cids/sync_queue/received_ref_updates/ arweave_anchors by the normalized owner/name slug, which is non-unique across repos rows (INV-9): a did:key:zX row and a bare zX mirror of the same name share the slug. Purging an empty did:key row therefore wiped a surviving non-empty mirror's branch->CID index and Arweave anchors. The cascade now runs only when no other repos row resolves to the same slug (checked with the same OWNER_KEY_CASE_SQL the slug is built from, on the delete's own transaction); the physical row and id-keyed children are always removed. The concurrent colliding-sibling-create window stays an accepted operator-scoped residual. --- crates/gitlawb-node/src/db/mod.rs | 132 +++++++++++++++++++++++++++--- 1 file changed, 122 insertions(+), 10 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index def7a394..10a0624a 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1363,16 +1363,36 @@ impl Db { } // Children keyed on the derived `owner_short/name` slug rather than the id. - for table in [ - "branch_cids", - "sync_queue", - "received_ref_updates", - "arweave_anchors", - ] { - sqlx::query(&format!("DELETE FROM {table} WHERE repo = $1")) - .bind(&slug) - .execute(&mut *tx) - .await?; + // INV-9 same-method slug collision: a canonical `did:key:zX` row and a bare + // `zX` mirror both normalize (via OWNER_KEY_CASE_SQL) to slug `zX/name`, so + // these tables cannot tell whose rows they are. Purging one row must not wipe + // a surviving sibling's records. Gate the slug-scoped deletes on the absence + // of any OTHER repos row resolving to the same slug, using the same + // OWNER_KEY_CASE_SQL the slug was built from so the check is byte-identical to + // the cascade key. (Residual: this is a point-in-time check inside one READ + // COMMITTED tx, not full serialization against a concurrent colliding create; + // see the plan's KTD4. Purge is the sole caller, so the window is + // operator-scoped.) + let sibling_exists: bool = sqlx::query_scalar(&format!( + "SELECT EXISTS(SELECT 1 FROM repos WHERE id <> $1 AND {key} || '/' || name = $2)", + key = OWNER_KEY_CASE_SQL + )) + .bind(id) + .bind(&slug) + .fetch_one(&mut *tx) + .await?; + if !sibling_exists { + for table in [ + "branch_cids", + "sync_queue", + "received_ref_updates", + "arweave_anchors", + ] { + sqlx::query(&format!("DELETE FROM {table} WHERE repo = $1")) + .bind(&slug) + .execute(&mut *tx) + .await?; + } } // NOTE: `bounties` (financial: amount/wallet/tx_hash) and `issue_comments` // (no issues table to map issue_id → repo) are deliberately NOT cascaded @@ -3751,6 +3771,98 @@ mod dedup_db_tests { ); } + /// U4 gate-raise (INV-9 same-method slug collision): a canonical `did:key:zX` + /// row and a bare `zX` mirror of the same name both normalize to slug `zX/name`, + /// which is the key the slug-scoped child tables use. Purging the EMPTY canonical + /// row must NOT run the slug-scoped cascade, because that would wipe the surviving + /// non-empty mirror's branch_cids/arweave_anchors (keyed on the shared slug). The + /// physical `did:key:` row still goes; the sibling's slug-scoped records stay. + /// RED on base: the cascade fires and both survivor rows are deleted. + #[sqlx::test] + async fn delete_repo_keeps_sibling_slug_records_on_collision(pool: PgPool) { + let db = db(pool).await; + // Two rows collapsing to slug `z6MkSiblingCollisionFixture/victim`: an empty + // canonical `did:key:` row (deletion target) and a non-empty bare mirror. + let canonical = rec( + "rid-canonical-empty", + "did:key:z6MkSiblingCollisionFixture", + "victim", + "empty canonical", + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:00Z", + ); + let mirror = rec( + "z6MkSiblingCollisionFixture/victim", + "z6MkSiblingCollisionFixture", + "victim", + "non-empty mirror", + "2026-02-01T00:00:00Z", + "2026-02-01T00:00:00Z", + ); + db.create_repo(&canonical).await.unwrap(); + db.create_repo(&mirror).await.unwrap(); + + // Both rows resolve to this slug; the child rows belong to the mirror. + let slug = format!( + "{}/victim", + crate::db::normalize_owner_key("did:key:z6MkSiblingCollisionFixture") + ); + sqlx::query( + "INSERT INTO branch_cids (repo, ref_name, sha, cid, node_did, updated_at) + VALUES ($1, 'refs/heads/main', '1', 'cid', 'n', '2026-02-01T00:00:00Z')", + ) + .bind(&slug) + .execute(db.pool()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO arweave_anchors (id, repo, owner_did, ref_name, old_sha, new_sha, cid, irys_tx_id, arweave_url, node_did, anchored_at) + VALUES ('anch1', $1, 'z6MkSiblingCollisionFixture', 'refs/heads/main', '0', '1', 'cid', 'irys', 'https://ar/1', 'n', '2026-02-01T00:00:00Z')", + ) + .bind(&slug) + .execute(db.pool()) + .await + .unwrap(); + + // Purge the empty canonical row. + let removed = db.delete_repo_by_id("rid-canonical-empty").await.unwrap(); + assert_eq!(removed, 1, "the empty canonical row is deleted"); + + async fn count(db: &Db, sql: &str, arg: &str) -> i64 { + sqlx::query_scalar::<_, i64>(sql) + .bind(arg) + .fetch_one(db.pool()) + .await + .unwrap() + } + // The surviving mirror keeps its slug-scoped records (the collision guard + // skips the slug-scoped cascade because a sibling row shares the slug). + assert!( + count(&db, "SELECT COUNT(*) FROM branch_cids WHERE repo=$1", &slug).await > 0, + "sibling's branch_cids must survive the collision purge" + ); + assert!( + count( + &db, + "SELECT COUNT(*) FROM arweave_anchors WHERE repo=$1", + &slug + ) + .await + > 0, + "sibling's arweave_anchors must survive the collision purge" + ); + assert_eq!( + count( + &db, + "SELECT COUNT(*) FROM repos WHERE id=$1", + "z6MkSiblingCollisionFixture/victim" + ) + .await, + 1, + "the surviving mirror repos row is untouched" + ); + } + /// The canonical `did:key:` row and the short-owner mirror row of one logical /// repo collapse to a single deduped entry: the canonical row wins and inherits /// the group's most recent `updated_at`. From 4860e5268fd3509f8b8343b3cb822ff2bea14a8b Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 19:17:02 -0500 Subject: [PATCH 30/61] fix(node): fail closed classifying the purge target directory (#196) local_refs_on_disk returned None for a missing path, an existing non-bare directory, AND an unreadable path alike; with a store configured, None promotes the candidate to remote_unverified and the archive refresh's remove_dir_all replaces whatever sits at the target. It now returns None only for a verified NotFound (symlink_metadata + ErrorKind::NotFound, so a dangling symlink counts as present); every other state fails closed as present so a non-repo directory at the target is never overwritten. --- crates/gitlawb-node/src/admin.rs | 112 +++++++++++++++++++++++++++---- 1 file changed, 98 insertions(+), 14 deletions(-) diff --git a/crates/gitlawb-node/src/admin.rs b/crates/gitlawb-node/src/admin.rs index a5c2edf5..ecc59966 100644 --- a/crates/gitlawb-node/src/admin.rs +++ b/crates/gitlawb-node/src/admin.rs @@ -172,11 +172,15 @@ where /// bare-repo markers (`HEAD` file + `objects/` dir) before trusting any count; /// anything else fails closed (treated non-empty, skipped). /// Local ref state for selection: `Some(n)` when a local bare repo exists (n -/// refs), `None` when there is no local copy. `None` is what lets selection -/// distinguish a missing-local repo (a remote-unverified candidate when a store -/// is configured) from a one-ref repo — both of which `ref_count_on_disk` -/// collapses to a non-zero count. An unsafe name or an unreadable repo fails -/// closed to `Some(1)` so it is skipped, never admitted as remote-unverified. +/// refs), `None` ONLY when the path is verifiably absent (a `NotFound` from +/// `symlink_metadata`). `None` is what lets selection distinguish a truly +/// missing-local repo (a remote-unverified candidate when a store is configured) +/// from a one-ref repo — both of which `ref_count_on_disk` collapses to a +/// non-zero count. Everything else fails closed to `Some(1)` so it is skipped and +/// never admitted as remote-unverified: an unsafe name, an unreadable/unstat-able +/// path, a dangling symlink, or an existing directory that is not a bare git repo +/// (returning `None` for the latter would promote it, and the remote_unverified +/// refresh's `remove_dir_all` would overwrite that non-repository directory). fn local_refs_on_disk(repos_dir: &Path, owner_did: &str, name: &str) -> Option { // Fail closed on an unsafe repo name BEFORE building any on-disk path (a // peer-mirror row can carry a `../` name). Report it as non-empty so it is @@ -187,9 +191,27 @@ fn local_refs_on_disk(repos_dir: &Path, owner_did: &str, name: &str) -> Option return None, + Err(e) => { + warn!(path = %path.display(), err = %e, + "purge-spam: could not stat path — treating as non-empty (skipped)"); + return Some(1); + } + Ok(_) => {} + } if !path.join("HEAD").is_file() || !path.join("objects").is_dir() { - // No local bare repo at the expected path. - return None; + // The path exists but is not a bare repo — fail closed as present so it is + // never promoted to remote_unverified and overwritten by a refresh. + warn!(path = %path.display(), + "purge-spam: path exists but is not a bare git repo — treating as non-empty (skipped)"); + return Some(1); } match store::list_refs(&path) { Ok(refs) => Some(refs.len()), @@ -1014,17 +1036,26 @@ mod db_tests { let repo = rec("t-x", SPAM_BURST_TARGET_DID, "spam1"); let path = store::repo_disk_path(tmp.path(), &repo.owner_did, &repo.name); + // (a0) A genuinely MISSING path is a verified NotFound — the only case that + // returns None (eligible for a remote-unverified archive refresh). + assert!(!path.exists()); + assert_eq!( + local_refs_on_disk(tmp.path(), &repo.owner_did, &repo.name), + None, + "a verifiably-absent path must return None (eligible for archive refresh)" + ); + // (a) Path exists as a plain (non-git) directory under the git ancestor. - // Without the marker guard, git discovery would read the ancestor's 0 - // refs and this repo would be deleted. local_refs_on_disk must report no - // local repo (None) WITHOUT running list_refs — never the ancestor's 0. - // None is skipped unless a store is configured, and the under-lock - // recheck (ref_count_on_disk) fails closed on the same markers regardless. + // Without the marker guard, git discovery would read the ancestor's 0 refs + // and this repo would be deleted. It must fail CLOSED to Some(1) — NOT + // None: returning None would promote it to remote_unverified, and the + // refresh's remove_dir_all would overwrite this existing directory. And it + // must never run list_refs to read the ancestor's 0. std::fs::create_dir_all(&path).unwrap(); assert_eq!( local_refs_on_disk(tmp.path(), &repo.owner_did, &repo.name), - None, - "a non-git dir under a git ancestor must report no local repo, not read the ancestor's refs" + Some(1), + "an existing non-git dir must fail closed to Some(1), never None (which would overwrite it)" ); assert_eq!( ref_count_on_disk(tmp.path(), &repo.owner_did, &repo.name), @@ -1388,6 +1419,59 @@ mod db_tests { assert_eq!(summary.skipped_not_empty, 1); } + // U7/R7: an existing NON-repository directory (holding operator data) sits at + // the exact target path. It is not a bare git repo, so it must fail CLOSED — + // never be admitted remote-unverified, because that promotion drives + // refresh_from_archive, whose remove_dir_all + extract would OVERWRITE the + // directory. Load-bearing: on base, local_refs_on_disk returns None for a + // non-bare path, the candidate is promoted, refresh wipes the dir and the row + // is deleted (RED). After the fix, Some(1) keeps it out entirely (GREEN). + #[sqlx::test] + async fn purge_does_not_overwrite_non_repo_dir_at_target(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-nonrepo", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + + // A plain directory with a file — deliberately NOT a bare git repo. + let path = store::repo_disk_path(tmp.path(), &target.owner_did, &target.name); + std::fs::create_dir_all(&path).unwrap(); + let sentinel = path.join("keep.txt"); + std::fs::write(&sentinel, b"operator data").unwrap(); + + let (f, deleted) = fake(true, false, false, false); // archive exists, empty + let repo_store = store_backed(tmp.path(), &pool, f); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + sentinel.exists(), + "an existing non-repo dir must not be overwritten by the purge refresh" + ); + assert_eq!( + std::fs::read(&sentinel).unwrap(), + b"operator data", + "the non-repo dir's contents must be intact" + ); + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_some(), + "a repo whose disk path is a non-bare dir must not be purged" + ); + assert_eq!(summary.deleted, 0); + assert_eq!( + summary.remote_unverified, 0, + "a non-bare existing dir must never be admitted remote-unverified" + ); + assert!( + !deleted.load(std::sync::atomic::Ordering::SeqCst), + "the archive must not be deleted" + ); + } + // AE5/R5: no local copy AND no archive — the candidate is admitted (a store is // configured) but the under-lock recheck finds nothing and fails closed. #[sqlx::test] From 2ceec1239772743ea79038ecdd0fe02041c00f22 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 19:35:18 -0500 Subject: [PATCH 31/61] fix(node): run the push completion tail in the disconnect-surviving task (#196) The receive-pack refactor detached only acquire/apply/release; the whole success tail (push+trust records, ref certificates, webhooks, replication/pinning, ref-update broadcast, peer-notify) ran inline in the cancellable request future. A client disconnect mid-apply committed and uploaded the git state but dropped that tail, leaving accepted git history with none of the metadata or fan-out every other push produces. The tail now runs inside the detached task, gated on receive-pack Ok; report-status still returns to a connected client early over a oneshot (never awaited on the JoinHandle), and a dropped receiver no longer aborts the tail. guard.release runs on every path. --- crates/gitlawb-node/src/api/repos.rs | 1130 ++++++++++++++++---------- 1 file changed, 720 insertions(+), 410 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 1656a83b..d32ad541 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -927,470 +927,504 @@ pub async fn git_receive_pack( let body_len = body.len(); let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - // Detach the acquire → receive-pack → release core from THIS handler future. - // The pack `body` is already fully buffered, so the spawned task is - // self-contained: a client disconnect drops the handler future but does NOT - // cancel the task, so a fully-received push still completes server-side — - // applies the pack, uploads, and releases the lock in order. What a - // disconnect forgoes is the cancellable post-push bookkeeping below, not the - // push itself. (KTD-3, R3.) - let repo_store = state.repo_store.clone(); - let owner_did = record.owner_did.clone(); - let repo_name = record.name.clone(); - let receive_result = tokio::spawn(async move { - let guard = repo_store - .acquire_write(&owner_did, &repo_name) + // Detach the WHOLE push — acquire → receive-pack → release AND the success + // tail (metadata + fan-out) — from THIS handler future. The pack `body` is + // already fully buffered, so the spawned task is self-contained: a client + // disconnect drops the handler future but does NOT cancel the task, so a + // fully-received push still applies the pack, releases the lock, AND runs its + // full metadata/fan-out tail server-side. Folding the tail INTO the task (vs + // leaving it in the cancellable request future) is what prevents a split-brain + // on disconnect: committed git with no push record / certs / broadcast. (KTD1, + // R1.) + // + // report-status is handed to the (possibly still-connected) client over a + // oneshot the request future awaits — NOT by awaiting the JoinHandle, which + // would re-couple the client response to the full tail completing (a latency + // regression). The task sends the receive-pack result, then continues the tail + // independently; if the client has disconnected the receiver is dropped and the + // send fails, which the task ignores and runs the tail regardless. + let (report_tx, report_rx) = tokio::sync::oneshot::channel::>(); + tokio::spawn(async move { + let guard = match state + .repo_store + .acquire_write(&record.owner_did, &record.name) .await - .map_err(|e| { - tracing::error!(repo = %repo_name, err = %e, "acquire_write failed"); - AppError::Git(e.to_string()) - })?; + { + Ok(g) => g, + Err(e) => { + tracing::error!(repo = %record.name, err = %e, "acquire_write failed"); + let _ = report_tx.send(Err(AppError::Git(e.to_string()))); + return; + } + }; let disk_path = guard.path().to_path_buf(); - tracing::debug!(repo = %repo_name, path = %disk_path.display(), "running git receive-pack"); + tracing::debug!(repo = %record.name, path = %disk_path.display(), "running git receive-pack"); let result = smart_http::receive_pack(&disk_path, body, git_timeout).await; - // Always release the advisory lock — even on error — to prevent stale - // locks from blocking subsequent pushes. Only upload to Tigris when the - // push succeeded; uploading a half-applied repo would propagate corruption. + // Always release the advisory lock — even on error, and BEFORE the tail — + // to prevent stale locks and to avoid holding the write lock across the + // fan-out. Only upload to Tigris when the push succeeded; uploading a + // half-applied repo would propagate corruption. guard.release(result.is_ok()).await; - Ok::<_, AppError>(result) - }) - .await - .map_err(|e| { - tracing::error!(repo = %name, err = %e, "receive-pack task panicked"); - AppError::Internal(anyhow::anyhow!("receive-pack task failed: {e}")) - })??; - // Recompute the on-disk path for the post-push tail below — the guard that - // exposed it now lives inside the detached task. Identical to the path - // acquire_write resolved (repos_dir/owner_slug/name.git). - let disk_path = store::repo_disk_path(&state.config.repos_dir, &record.owner_did, &record.name); - - let result = receive_result.map_err(|e| { - let app = git_service_app_error(&e); - match &app { - AppError::Timeout(_) => tracing::warn!(repo = %name, "git receive-pack timed out"), - AppError::BadRequest(msg) => { - tracing::warn!(repo = %name, err = %msg, "git receive-pack: bad client request") + // On receive-pack error, deliver the classified error to the client and run + // NO tail. On success, hand report-status to the client now, then continue + // the tail regardless of whether the client is still listening. + let response = match result { + Ok(resp) => resp, + Err(e) => { + let app = git_service_app_error(&e); + match &app { + AppError::Timeout(_) => { + tracing::warn!(repo = %record.name, "git receive-pack timed out") + } + AppError::BadRequest(msg) => { + tracing::warn!(repo = %record.name, err = %msg, "git receive-pack: bad client request") + } + _ => tracing::error!(repo = %record.name, err = %e, "git receive-pack failed"), + } + let _ = report_tx.send(Err(app)); + return; } - _ => tracing::error!(repo = %name, err = %e, "git receive-pack failed"), - } - app - })?; + }; + let _ = report_tx.send(Ok(response)); - // Update the repo's updated_at timestamp after a successful push - let _ = state.db.touch_repo(&record.id).await; + // Update the repo's updated_at timestamp after a successful push + let _ = state.db.touch_repo(&record.id).await; - // Record the successful push for metrics. The body has already been - // consumed by smart_http::receive_pack so we observe size up front. - crate::metrics::record_push(&record.id); - crate::metrics::observe_pack_size(body_len as f64); + // Record the successful push for metrics. The body has already been + // consumed by smart_http::receive_pack so we observe size up front. + crate::metrics::record_push(&record.id); + crate::metrics::observe_pack_size(body_len as f64); - // Record push event for trust score and issue a signed ref certificate. - // The route is behind `require_signature`, so the verified pusher identity is - // always present; use it directly rather than re-parsing the headers. - let did = auth.0.as_str(); - { - // Use the first new commit hash we parsed, fall back to timestamp - let commit_hash = ref_updates - .first() - .map(|u| u.new_sha.clone()) - .unwrap_or_else(|| Utc::now().timestamp().to_string()); - - let _ = state.db.record_push(did, &record.id, &commit_hash, 0).await; - if let Ok(push_count) = state.db.get_push_count(did).await { - // 0.05 base (from registration) + 0.05 per push, capped at 1.0 - // 1 push → 0.10, 5 pushes → 0.30, 19 pushes → 1.0 - let new_score = (push_count as f64 * 0.05 + 0.05).min(1.0); - let _ = state.db.update_trust_score(did, new_score).await; - } + // Record push event for trust score and issue a signed ref certificate. + // The route is behind `require_signature`, so the verified pusher identity is + // always present; use it directly rather than re-parsing the headers. + let did = auth.0.as_str(); + { + // Use the first new commit hash we parsed, fall back to timestamp + let commit_hash = ref_updates + .first() + .map(|u| u.new_sha.clone()) + .unwrap_or_else(|| Utc::now().timestamp().to_string()); + + let _ = state.db.record_push(did, &record.id, &commit_hash, 0).await; + if let Ok(push_count) = state.db.get_push_count(did).await { + // 0.05 base (from registration) + 0.05 per push, capped at 1.0 + // 1 push → 0.10, 5 pushes → 0.30, 19 pushes → 1.0 + let new_score = (push_count as f64 * 0.05 + 0.05).min(1.0); + let _ = state.db.update_trust_score(did, new_score).await; + } - // Issue a signed certificate for every ref this push advanced, each - // carrying that ref's real old→new transition. A multi-ref push must - // not collapse to a single cert covering only the first ref. - for update in &ref_updates { - match cert::issue_ref_certificate( - &state, - &record.id, - &update.ref_name, - &update.old_sha, - &update.new_sha, - did, - ) - .await - { - Ok(c) => { - tracing::info!(cert_id = %c.id, repo = %record.name, ref_name = %update.ref_name, pusher = %did, "issued ref certificate") - } - Err(e) => { - tracing::warn!(err = %e, ref_name = %update.ref_name, "failed to issue ref certificate") + // Issue a signed certificate for every ref this push advanced, each + // carrying that ref's real old→new transition. A multi-ref push must + // not collapse to a single cert covering only the first ref. + for update in &ref_updates { + match cert::issue_ref_certificate( + &state, + &record.id, + &update.ref_name, + &update.old_sha, + &update.new_sha, + did, + ) + .await + { + Ok(c) => { + tracing::info!(cert_id = %c.id, repo = %record.name, ref_name = %update.ref_name, pusher = %did, "issued ref certificate") + } + Err(e) => { + tracing::warn!(err = %e, ref_name = %update.ref_name, "failed to issue ref certificate") + } } } } - } - // Fire push webhooks — one per ref update - if !ref_updates.is_empty() { - let base_url = state - .config - .public_url - .as_deref() - .unwrap_or("http://127.0.0.1:7545") - .trim_end_matches('/'); - let owner_short = crate::db::normalize_owner_key(&record.owner_did); - let clone_url = format!("{}/{}/{}.git", base_url, owner_short, record.name); - - for update in &ref_updates { - let payload = serde_json::json!({ - "ref": update.ref_name, - "before": update.old_sha, - "after": update.new_sha, - "created": update.old_sha == ZERO_SHA, - "forced": false, - "pusher": { - "did": did, - }, - "repository": { - "id": record.id, - "name": record.name, - "owner_did": record.owner_did, - "clone_url": clone_url, - }, - }); - webhooks::fire_event( - state.db.clone(), - state.http_client.clone(), - &record.id, - "push", - payload, - ); + // Fire push webhooks — one per ref update + if !ref_updates.is_empty() { + let base_url = state + .config + .public_url + .as_deref() + .unwrap_or("http://127.0.0.1:7545") + .trim_end_matches('/'); + let owner_short = crate::db::normalize_owner_key(&record.owner_did); + let clone_url = format!("{}/{}/{}.git", base_url, owner_short, record.name); + + for update in &ref_updates { + let payload = serde_json::json!({ + "ref": update.ref_name, + "before": update.old_sha, + "after": update.new_sha, + "created": update.old_sha == ZERO_SHA, + "forced": false, + "pusher": { + "did": did, + }, + "repository": { + "id": record.id, + "name": record.name, + "owner_did": record.owner_did, + "clone_url": clone_url, + }, + }); + webhooks::fire_event( + state.db.clone(), + state.http_client.clone(), + &record.id, + "push", + payload, + ); + } } - } - // Replication enforcement (Phase 2): decide once per push whether the public - // may read this repo at all and, if so, which blob OIDs must not leave the - // node. `withheld == None` means replicate nothing (private / mode A / - // undetermined): skip every pin so even commit and tree objects (which - // withheld_blob_oids never lists) stay local. `announce` gates the - // network-facing announcements. Fail closed: a private or undetermined repo - // never leaks. - let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); - let (announce, withheld) = replication_withheld_set( - rules_opt.clone(), - &record.owner_did, - record.is_public, - disk_path.clone(), - ) - .await; - - // Resolve the per-push pin candidate set once, off the async worker, then - // filter to what may actually replicate. Delta path: the reachable-only - // `withheld` set suffices (delta objects are reachable). Full-scan path: the - // candidate set can include dangling blobs the withheld set never classified, - // so fail closed — replicate a blob only if it is reachable AND - // visibility-allowed (#99). Only computed when something will actually - // replicate; every degraded path logs rather than failing silently. - let object_list: Vec = if let Some(withheld_set) = withheld.clone() { - let new_tips: Vec = ref_updates - .iter() - .map(|u| u.new_sha.clone()) - .filter(|s| s != ZERO_SHA) - .collect(); - let old_tips: Vec = ref_updates - .iter() - .map(|u| u.old_sha.clone()) - .filter(|s| s != ZERO_SHA) - .collect(); - let pin_set = crate::git::push_delta::resolve_candidates_for_push( + // Replication enforcement (Phase 2): decide once per push whether the public + // may read this repo at all and, if so, which blob OIDs must not leave the + // node. `withheld == None` means replicate nothing (private / mode A / + // undetermined): skip every pin so even commit and tree objects (which + // withheld_blob_oids never lists) stay local. `announce` gates the + // network-facing announcements. Fail closed: a private or undetermined repo + // never leaks. + let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); + let (announce, withheld) = replication_withheld_set( + rules_opt.clone(), + &record.owner_did, + record.is_public, disk_path.clone(), - new_tips, - old_tips, ) .await; - if pin_set.full_scan { - fail_closed_full_scan_objects( - disk_path.clone(), - rules_opt.clone().unwrap_or_default(), - record.is_public, - record.owner_did.clone(), - pin_set.candidates, - ) - .await - } else { - crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) - } - } else { - Vec::new() - }; - // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). - // Skipped entirely when the public cannot read the repo (withheld == None). - if withheld.is_some() { - let object_list_ipfs = object_list.clone(); - let ipfs_api = state.config.ipfs_api.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let rules_for_enc = rules_opt.clone(); - let repo_id = record.id.clone(); - let owner_did = record.owner_did.clone(); - let is_public = record.is_public; - let irys_url = state.config.irys_url.clone(); - let http_client = std::sync::Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let node_seed = state.node_keypair.to_seed(); - let repo_name = record.name.clone(); - tokio::spawn(async move { - let pinned = crate::ipfs_pin::pin_new_objects( - &ipfs_api, - &repo_path_clone, - object_list_ipfs, - &db_clone, + // Resolve the per-push pin candidate set once, off the async worker, then + // filter to what may actually replicate. Delta path: the reachable-only + // `withheld` set suffices (delta objects are reachable). Full-scan path: the + // candidate set can include dangling blobs the withheld set never classified, + // so fail closed — replicate a blob only if it is reachable AND + // visibility-allowed (#99). Only computed when something will actually + // replicate; every degraded path logs rather than failing silently. + let object_list: Vec = if let Some(withheld_set) = withheld.clone() { + let new_tips: Vec = ref_updates + .iter() + .map(|u| u.new_sha.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = ref_updates + .iter() + .map(|u| u.old_sha.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let pin_set = crate::git::push_delta::resolve_candidates_for_push( + disk_path.clone(), + new_tips, + old_tips, ) .await; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); - for (sha, cid) in &pinned { - tracing::info!(sha = %sha, %cid, "pinned"); - } + if pin_set.full_scan { + fail_closed_full_scan_objects( + disk_path.clone(), + rules_opt.clone().unwrap_or_default(), + record.is_public, + record.owner_did.clone(), + pin_set.candidates, + ) + .await + } else { + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) } + } else { + Vec::new() + }; - // Option B1: encrypt-then-pin the withheld blobs so authorized - // readers can recover them when the origin cannot serve them. - // No path-scoped rule can withhold a blob, so withheld_blob_recipients - // would return an empty map after a full per-ref walk; skip it. Mirrors - // the has_path_scoped_rule gate on the other two withheld-walk sites. - if let Some(rules) = rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) - { - let p = repo_path_clone.clone(); - let owner = owner_did.clone(); - let recip = tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_recipients( - &p, &rules, is_public, &owner, - ) - }) + // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). + // Skipped entirely when the public cannot read the repo (withheld == None). + if withheld.is_some() { + let object_list_ipfs = object_list.clone(); + let ipfs_api = state.config.ipfs_api.clone(); + let repo_path_clone = disk_path.clone(); + let db_clone = state.db.clone(); + let rules_for_enc = rules_opt.clone(); + let repo_id = record.id.clone(); + let owner_did = record.owner_did.clone(); + let is_public = record.is_public; + let irys_url = state.config.irys_url.clone(); + let http_client = std::sync::Arc::clone(&state.http_client); + let node_did_str = state.node_did.to_string(); + let node_seed = state.node_keypair.to_seed(); + let repo_name = record.name.clone(); + tokio::spawn(async move { + let pinned = crate::ipfs_pin::pin_new_objects( + &ipfs_api, + &repo_path_clone, + object_list_ipfs, + &db_clone, + ) .await; - if let Ok(Ok(recipients)) = recip { - let delta = crate::encrypted_pin::encrypt_and_pin( - &ipfs_api, - &repo_path_clone, - &db_clone, - &repo_id, - &node_seed, - &recipients, - ) - .await; + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); + for (sha, cid) in &pinned { + tracing::info!(sha = %sha, %cid, "pinned"); + } + } - // Option B3: anchor a per-push manifest of the blobs sealed - // this push to Arweave, so the oid->cid index survives total - // node loss. Best-effort; never fails the push. - if !delta.is_empty() && !irys_url.is_empty() { - let owner_short = crate::db::normalize_owner_key(&owner_did); - let repo_slug = format!("{owner_short}/{repo_name}"); - let ts = chrono::Utc::now().to_rfc3339(); - let manifest = crate::arweave::EncryptedManifest { - repo: &repo_slug, - owner_did: &owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &delta, - }; - match crate::arweave::anchor_encrypted_manifest( - &http_client, - &irys_url, - &manifest, + // Option B1: encrypt-then-pin the withheld blobs so authorized + // readers can recover them when the origin cannot serve them. + // No path-scoped rule can withhold a blob, so withheld_blob_recipients + // would return an empty map after a full per-ref walk; skip it. Mirrors + // the has_path_scoped_rule gate on the other two withheld-walk sites. + if let Some(rules) = + rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) + { + let p = repo_path_clone.clone(); + let owner = owner_did.clone(); + let recip = tokio::task::spawn_blocking(move || { + crate::git::visibility_pack::withheld_blob_recipients( + &p, &rules, is_public, &owner, ) - .await - { - Ok(tx) if !tx.is_empty() => tracing::info!( - repo = %repo_slug, - tx_id = %tx, - "anchored encrypted manifest to Arweave" - ), - Ok(_) => {} - Err(e) => tracing::warn!( - repo = %repo_slug, - err = %e, - "encrypted manifest anchor failed" - ), + }) + .await; + if let Ok(Ok(recipients)) = recip { + let delta = crate::encrypted_pin::encrypt_and_pin( + &ipfs_api, + &repo_path_clone, + &db_clone, + &repo_id, + &node_seed, + &recipients, + ) + .await; + + // Option B3: anchor a per-push manifest of the blobs sealed + // this push to Arweave, so the oid->cid index survives total + // node loss. Best-effort; never fails the push. + if !delta.is_empty() && !irys_url.is_empty() { + let owner_short = crate::db::normalize_owner_key(&owner_did); + let repo_slug = format!("{owner_short}/{repo_name}"); + let ts = chrono::Utc::now().to_rfc3339(); + let manifest = crate::arweave::EncryptedManifest { + repo: &repo_slug, + owner_did: &owner_did, + node_did: &node_did_str, + timestamp: &ts, + blobs: &delta, + }; + match crate::arweave::anchor_encrypted_manifest( + &http_client, + &irys_url, + &manifest, + ) + .await + { + Ok(tx) if !tx.is_empty() => tracing::info!( + repo = %repo_slug, + tx_id = %tx, + "anchored encrypted manifest to Arweave" + ), + Ok(_) => {} + Err(e) => tracing::warn!( + repo = %repo_slug, + err = %e, + "encrypted manifest anchor failed" + ), + } } } } - } - }); - } + }); + } - // Pin new git objects to Pinata, then record branch→CID and gossip - { - let pinata_jwt = state.config.pinata_jwt.clone(); - let pinata_upload_url = state.config.pinata_upload_url.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let http_client = Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let repo_slug = format!( - "{}/{}", - crate::db::normalize_owner_key(&record.owner_did), - record.name - ); - let ref_updates_clone = ref_updates - .iter() - .map(|u| (u.ref_name.clone(), u.old_sha.clone(), u.new_sha.clone())) - .collect::>(); - let p2p_handle = state.p2p.clone(); - let pusher_did_clone = did.to_string(); - let db_for_peers = state.db.clone(); - let ref_update_tx = state.ref_update_tx.clone(); - let irys_url = state.config.irys_url.clone(); - let owner_did_for_arweave = record.owner_did.clone(); - let self_public_url = state.config.public_url.clone(); - let node_keypair = Arc::clone(&state.node_keypair); - let object_list_pinata = object_list; - let do_pinata_replication = withheld.is_some(); - tokio::spawn(async move { - let pinned = if do_pinata_replication { - crate::pinata::pin_new_objects( - &http_client, - &pinata_upload_url, - &pinata_jwt, - &repo_path_clone, - object_list_pinata, - &db_clone, - ) - .await - } else { - Vec::new() - }; + // Pin new git objects to Pinata, then record branch→CID and gossip + { + let pinata_jwt = state.config.pinata_jwt.clone(); + let pinata_upload_url = state.config.pinata_upload_url.clone(); + let repo_path_clone = disk_path.clone(); + let db_clone = state.db.clone(); + let http_client = Arc::clone(&state.http_client); + let node_did_str = state.node_did.to_string(); + let repo_slug = format!( + "{}/{}", + crate::db::normalize_owner_key(&record.owner_did), + record.name + ); + let ref_updates_clone = ref_updates + .iter() + .map(|u| (u.ref_name.clone(), u.old_sha.clone(), u.new_sha.clone())) + .collect::>(); + let p2p_handle = state.p2p.clone(); + let pusher_did_clone = did.to_string(); + let db_for_peers = state.db.clone(); + let ref_update_tx = state.ref_update_tx.clone(); + let irys_url = state.config.irys_url.clone(); + let owner_did_for_arweave = record.owner_did.clone(); + let self_public_url = state.config.public_url.clone(); + let node_keypair = Arc::clone(&state.node_keypair); + let object_list_pinata = object_list; + let do_pinata_replication = withheld.is_some(); + tokio::spawn(async move { + let pinned = if do_pinata_replication { + crate::pinata::pin_new_objects( + &http_client, + &pinata_upload_url, + &pinata_jwt, + &repo_path_clone, + object_list_pinata, + &db_clone, + ) + .await + } else { + Vec::new() + }; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to Pinata"); - } + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to Pinata"); + } - // Build sha→cid map from pinned objects - let cid_map: std::collections::HashMap = pinned.into_iter().collect(); + // Build sha→cid map from pinned objects + let cid_map: std::collections::HashMap = + pinned.into_iter().collect(); - // Record branch→CID for each ref update and publish gossip - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let cid = cid_map.get(new_sha).map(|s| s.as_str()); + // Record branch→CID for each ref update and publish gossip + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let cid = cid_map.get(new_sha).map(|s| s.as_str()); + + if let Some(cid_str) = cid { + let _ = db_clone + .upsert_branch_cid( + &repo_slug, + ref_name, + new_sha, + cid_str, + &node_did_str, + ) + .await; + } - if let Some(cid_str) = cid { - let _ = db_clone - .upsert_branch_cid(&repo_slug, ref_name, new_sha, cid_str, &node_did_str) - .await; + if announce { + if let Some(p2p) = &p2p_handle { + p2p.publish_ref_update(crate::p2p::RefUpdateEvent { + node_did: node_did_str.clone(), + pusher_did: pusher_did_clone.clone(), + repo: repo_slug.clone(), + ref_name: ref_name.clone(), + old_sha: old_sha.clone(), + new_sha: new_sha.clone(), + timestamp: chrono::Utc::now().to_rfc3339(), + cert_id: None, + cid: cid.map(|s| s.to_string()), + }) + .await; + } + } } + // Broadcast ref update to GraphQL subscription listeners — one per ref. + // Gated on `announce`: /graphql/ws is unauthenticated (mounted after + // the optional_signature layer), and the subscription resolver has no + // caller to gate against, so only publicly-readable ref updates may + // reach anonymous subscribers. Mirrors the gossip (above) and Arweave + // (below) sends, which are already `announce`-gated. Without this a + // private-repo push would leak live ref metadata over the socket — + // the subscription analog of #112/#114. + let now_ts = chrono::Utc::now().to_rfc3339(); if announce { - if let Some(p2p) = &p2p_handle { - p2p.publish_ref_update(crate::p2p::RefUpdateEvent { - node_did: node_did_str.clone(), - pusher_did: pusher_did_clone.clone(), + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let _ = ref_update_tx.send(crate::state::RefUpdateBroadcast { repo: repo_slug.clone(), ref_name: ref_name.clone(), old_sha: old_sha.clone(), new_sha: new_sha.clone(), - timestamp: chrono::Utc::now().to_rfc3339(), - cert_id: None, - cid: cid.map(|s| s.to_string()), - }) - .await; + pusher_did: pusher_did_clone.clone(), + node_did: node_did_str.clone(), + timestamp: now_ts.clone(), + }); } } - } - - // Broadcast ref update to GraphQL subscription listeners — one per ref. - // Gated on `announce`: /graphql/ws is unauthenticated (mounted after - // the optional_signature layer), and the subscription resolver has no - // caller to gate against, so only publicly-readable ref updates may - // reach anonymous subscribers. Mirrors the gossip (above) and Arweave - // (below) sends, which are already `announce`-gated. Without this a - // private-repo push would leak live ref metadata over the socket — - // the subscription analog of #112/#114. - let now_ts = chrono::Utc::now().to_rfc3339(); - if announce { - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let _ = ref_update_tx.send(crate::state::RefUpdateBroadcast { - repo: repo_slug.clone(), - ref_name: ref_name.clone(), - old_sha: old_sha.clone(), - new_sha: new_sha.clone(), - pusher_did: pusher_did_clone.clone(), - node_did: node_did_str.clone(), - timestamp: now_ts.clone(), - }); - } - } - // Arweave permanent anchoring — fire for each ref update. - // Suppressed for repos the public cannot read (public permanent ledger). - if announce && !irys_url.is_empty() { - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let cid = cid_map.get(new_sha).cloned(); - let anchor = crate::arweave::RefAnchor { - repo: repo_slug.clone(), - owner_did: owner_did_for_arweave.clone(), - ref_name: ref_name.clone(), - old_sha: old_sha.clone(), - new_sha: new_sha.clone(), - cid: cid.clone(), - timestamp: now_ts.clone(), - node_did: node_did_str.clone(), - }; - match crate::arweave::anchor_ref_update(&http_client, &irys_url, &anchor).await - { - Ok(tx_id) if !tx_id.is_empty() => { - let arweave_url = crate::arweave::arweave_url(&tx_id); - let _ = db_clone - .record_arweave_anchor(&crate::db::RecordAnchorInput { - repo: &repo_slug, - owner_did: &owner_did_for_arweave, - ref_name, - old_sha, - new_sha, - cid: cid.as_deref(), - irys_tx_id: &tx_id, - arweave_url: &arweave_url, - node_did: &node_did_str, - }) - .await; + // Arweave permanent anchoring — fire for each ref update. + // Suppressed for repos the public cannot read (public permanent ledger). + if announce && !irys_url.is_empty() { + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let cid = cid_map.get(new_sha).cloned(); + let anchor = crate::arweave::RefAnchor { + repo: repo_slug.clone(), + owner_did: owner_did_for_arweave.clone(), + ref_name: ref_name.clone(), + old_sha: old_sha.clone(), + new_sha: new_sha.clone(), + cid: cid.clone(), + timestamp: now_ts.clone(), + node_did: node_did_str.clone(), + }; + match crate::arweave::anchor_ref_update(&http_client, &irys_url, &anchor) + .await + { + Ok(tx_id) if !tx_id.is_empty() => { + let arweave_url = crate::arweave::arweave_url(&tx_id); + let _ = db_clone + .record_arweave_anchor(&crate::db::RecordAnchorInput { + repo: &repo_slug, + owner_did: &owner_did_for_arweave, + ref_name, + old_sha, + new_sha, + cid: cid.as_deref(), + irys_tx_id: &tx_id, + arweave_url: &arweave_url, + node_did: &node_did_str, + }) + .await; + } + Ok(_) => {} + Err(e) => { + tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed") + } } - Ok(_) => {} - Err(e) => tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed"), } } - } - // HTTP peer notification — notify all known peers to pull from us. - // This is the reliable fallback when Gossipsub p2p is not yet connected. - // Suppressed for repos the public cannot read. Runs last so a slow or - // unreachable peer cannot delay the local GraphQL broadcast or Arweave - // anchoring above; this is the lowest-priority best-effort step. - if announce { - if let Ok(peers) = db_for_peers.list_peers().await { - for peer in peers { - if peer.http_url.is_empty() { - continue; - } - let peer_url = peer.http_url.trim_end_matches('/'); - if let Some(self_url) = self_public_url.as_deref() { - if peer_url == self_url.trim_end_matches('/') { + // HTTP peer notification — notify all known peers to pull from us. + // This is the reliable fallback when Gossipsub p2p is not yet connected. + // Suppressed for repos the public cannot read. Runs last so a slow or + // unreachable peer cannot delay the local GraphQL broadcast or Arweave + // anchoring above; this is the lowest-priority best-effort step. + if announce { + if let Ok(peers) = db_for_peers.list_peers().await { + for peer in peers { + if peer.http_url.is_empty() { continue; } + let peer_url = peer.http_url.trim_end_matches('/'); + if let Some(self_url) = self_public_url.as_deref() { + if peer_url == self_url.trim_end_matches('/') { + continue; + } + } + let notify_url = format!("{peer_url}{SYNC_NOTIFY_PATH}"); + notify_peer_of_refs( + &http_client, + node_keypair.as_ref(), + &peer.did, + ¬ify_url, + &repo_slug, + &ref_updates_clone, + &node_did_str, + &pusher_did_clone, + ) + .await; } - let notify_url = format!("{peer_url}{SYNC_NOTIFY_PATH}"); - notify_peer_of_refs( - &http_client, - node_keypair.as_ref(), - &peer.did, - ¬ify_url, - &repo_slug, - &ref_updates_clone, - &node_did_str, - &pusher_did_clone, - ) - .await; } } - } - }); - } + }); + } + }); + + // Await report-status from the detached task WITHOUT blocking on the tail. If + // the client disconnected, this future is already dropped and `report_rx` with + // it — the task then runs the tail regardless. A send-before-report task panic + // drops the sender, surfaced here as an internal error (matching the prior + // JoinError → 500 mapping). + let result = report_rx.await.map_err(|_| { + AppError::Internal(anyhow::anyhow!( + "receive-pack task failed before reporting report-status" + )) + })??; Ok(result) } @@ -2760,6 +2794,282 @@ mod tests { ); } + // U1/R1: a fully-received push must produce its metadata + fan-out TAIL even + // when the client disconnects mid-apply. The pre-fix handler detaches only + // acquire→receive→release; the whole success tail (touch_repo, record_push, + // certs, webhooks, replication, ref broadcast) runs inline in the cancellable + // request future, so a disconnect after receive-pack succeeds commits the git + // refs (in the surviving task) but drops the tail — a split-brain: committed + // git, no push record. Here we drop the handler mid-apply (sleeping hook), let + // the surviving task finish, and assert the push record landed. RED pre-fix + // (push_events row absent), GREEN after (tail folded into the detached task). + #[sqlx::test] + async fn push_metadata_tail_completes_after_client_disconnect(pool: sqlx::PgPool) { + use axum::extract::{Path as AxPath, State}; + use axum::Extension; + use std::os::unix::fs::PermissionsExt; + use std::process::{Command, Stdio}; + use std::time::Duration; + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + + let owner = "z6u1tailowner"; + let name = "u1tail"; + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + + fn git(args: &[&str], dir: &std::path::Path) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + // Build a commit in a scratch working repo. + let work = tempfile::TempDir::new().unwrap(); + git(&["init", "-q", "-b", "main", "."], work.path()); + git(&["config", "user.email", "t@t"], work.path()); + git(&["config", "user.name", "t"], work.path()); + std::fs::write(work.path().join("f.txt"), "hi").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c"], work.path()); + let oid = git(&["rev-parse", "HEAD"], work.path()); + + std::fs::create_dir_all(bare.parent().unwrap()).unwrap(); + let out = Command::new("git") + .args([ + "clone", + "--bare", + "-q", + &work.path().to_string_lossy(), + &bare.to_string_lossy(), + ]) + .output() + .unwrap(); + assert!( + out.status.success(), + "clone --bare failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + git( + &[ + "-C", + &bare.to_string_lossy(), + "update-ref", + "-d", + "refs/heads/main", + ], + work.path(), + ); + + // Sleeping pre-receive hook: the deterministic mid-apply window we drop in. + let hook = bare.join("hooks").join("pre-receive"); + std::fs::create_dir_all(hook.parent().unwrap()).unwrap(); + std::fs::write(&hook, "#!/bin/sh\ncat >/dev/null\nsleep 2\nexit 0\n").unwrap(); + std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap(); + + // Receive-pack POST body: create refs/heads/main -> oid, plus a real pack. + let zero = "0".repeat(40); + let cmd = format!("{zero} {oid} refs/heads/main\0report-status\n"); + let mut body = Vec::new(); + let len = cmd.len() + 4; + body.extend_from_slice(format!("{len:04x}").as_bytes()); + body.extend_from_slice(cmd.as_bytes()); + body.extend_from_slice(b"0000"); + let pack = Command::new("git") + .args([ + "-C", + &bare.to_string_lossy(), + "pack-objects", + "--stdout", + "-q", + ]) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!( + pack.status.success(), + "pack-objects failed: {}", + String::from_utf8_lossy(&pack.stderr) + ); + body.extend_from_slice(&pack.stdout); + let body = Bytes::from(body); + + // Drive the handler, then DROP it mid-hook (the "client disconnect"). + let handler = super::git_receive_pack( + State(state.clone()), + AxPath((owner.to_string(), format!("{name}.git"))), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + body, + ); + let _ = tokio::time::timeout(Duration::from_millis(800), handler).await; + + // Poll for the tail's push record past the hook's sleep. The pusher DID is + // the repo owner, so the push_events count for `owner` becomes 1 once the + // surviving task runs the tail. + let mut pushes = 0i64; + for _ in 0..40 { + pushes = state.db.get_push_count(owner).await.unwrap(); + if pushes >= 1 { + break; + } + tokio::time::sleep(Duration::from_millis(150)).await; + } + assert_eq!( + pushes, 1, + "the push-metadata tail (record_push) must land after a fully-received \ + push despite the client disconnect — it must run in the detached task, \ + not the cancelled request future" + ); + } + + // U1/R1 no-regression: a CONNECTED push (handler awaited to completion) must + // still get report-status back over the oneshot AND run the full tail. Guards + // the two ways the refactor could regress a connected client: the response + // coupling to tail completion (a latency regression), and the oneshot never + // delivering (client would see a spurious 500). + #[sqlx::test] + async fn connected_push_returns_report_status_and_runs_tail(pool: sqlx::PgPool) { + use axum::extract::{Path as AxPath, State}; + use axum::Extension; + use std::process::{Command, Stdio}; + use std::time::Duration; + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + + let owner = "z6u1connowner"; + let name = "u1conn"; + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + + fn git(args: &[&str], dir: &std::path::Path) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + let work = tempfile::TempDir::new().unwrap(); + git(&["init", "-q", "-b", "main", "."], work.path()); + git(&["config", "user.email", "t@t"], work.path()); + git(&["config", "user.name", "t"], work.path()); + std::fs::write(work.path().join("f.txt"), "hi").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c"], work.path()); + let oid = git(&["rev-parse", "HEAD"], work.path()); + + std::fs::create_dir_all(bare.parent().unwrap()).unwrap(); + let out = Command::new("git") + .args([ + "clone", + "--bare", + "-q", + &work.path().to_string_lossy(), + &bare.to_string_lossy(), + ]) + .output() + .unwrap(); + assert!(out.status.success(), "clone --bare failed"); + git( + &[ + "-C", + &bare.to_string_lossy(), + "update-ref", + "-d", + "refs/heads/main", + ], + work.path(), + ); + + // No hook: the push applies immediately, so the handler returns promptly. + let zero = "0".repeat(40); + let cmd = format!("{zero} {oid} refs/heads/main\0report-status\n"); + let mut body = Vec::new(); + let len = cmd.len() + 4; + body.extend_from_slice(format!("{len:04x}").as_bytes()); + body.extend_from_slice(cmd.as_bytes()); + body.extend_from_slice(b"0000"); + let pack = Command::new("git") + .args([ + "-C", + &bare.to_string_lossy(), + "pack-objects", + "--stdout", + "-q", + ]) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(pack.status.success(), "pack-objects failed"); + body.extend_from_slice(&pack.stdout); + let body = Bytes::from(body); + + // Await the handler to completion (connected client) and assert it returned + // report-status (200) over the oneshot — not a 500 from a dropped sender. + let resp = super::git_receive_pack( + State(state.clone()), + AxPath((owner.to_string(), format!("{name}.git"))), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + body, + ) + .await + .expect("connected push must return report-status"); + assert_eq!(resp.status(), axum::http::StatusCode::OK); + + // And the tail runs (detached task): the push record lands. + let mut pushes = 0i64; + for _ in 0..40 { + pushes = state.db.get_push_count(owner).await.unwrap(); + if pushes >= 1 { + break; + } + tokio::time::sleep(Duration::from_millis(150)).await; + } + assert_eq!(pushes, 1, "the tail must run on a connected push too"); + } + /// Repo creation must be throttled by the per-IP creation limiter BEFORE /// signature verification — otherwise a DID farm (one throwaway did:key per /// repo, each carrying a valid but machine-solved iCaptcha proof) walks past From 308530f56658fc39d889a4e2b700976ce3411668 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 19:48:47 -0500 Subject: [PATCH 32/61] fix(node): tombstone bounties when their repo is purged (#196) Bounties are keyed only on (repo_owner, repo_name) with no repo id, and purge left them intact, so recreating the same name re-attached the old open bounty and claim_bounty succeeded against it. Purge now transitions the repo's non-terminal bounties (open/claimed/submitted) to a dead 'purged' status on the delete's own transaction, preserving the financial record. 'purged' is inert against every mutating bounty endpoint, including dispute (which would otherwise reset claimed/ submitted back to open). --- crates/gitlawb-node/src/db/mod.rs | 163 +++++++++++++++++++++++++++++- 1 file changed, 162 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 10a0624a..6cbbe576 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1398,6 +1398,27 @@ impl Db { // (no issues table to map issue_id → repo) are deliberately NOT cascaded // here — dropping money records or unmappable rows on a repo delete would // be wrong. They are left intact by design. + // + // But bounties key on (repo_owner, repo_name), not an immutable repo id, so + // an untouched OPEN bounty would silently re-attach to a same-name repo the + // owner recreates and become claimable again (finding F). Tombstone the + // repo's non-terminal bounties to a dead `purged` status instead of deleting + // them: this preserves the financial record while making them unclaimable + // (claim/submit/approve/cancel/dispute all gate on specific non-`purged` + // source statuses, so `purged` is inert and no transition reopens it). + // Scope by (repo_owner, repo_name) as the surviving purge does; the already + // terminal `completed`/`cancelled` bounties are left as-is. (Residual: a + // tombstoned bounty stays readable/listable under the same owner/name after + // recreation — a view, not a claim/payout re-attach; see the plan's KTD8.) + sqlx::query( + "UPDATE bounties SET status = 'purged' + WHERE repo_owner = $1 AND repo_name = $2 + AND status IN ('open', 'claimed', 'submitted')", + ) + .bind(&owner_did) + .bind(&name) + .execute(&mut *tx) + .await?; let result = sqlx::query("DELETE FROM repos WHERE id = $1") .bind(id) @@ -3646,7 +3667,7 @@ mod agent_discovery_tests { #[cfg(test)] mod dedup_db_tests { - use super::{Db, RepoRecord}; + use super::{BountyRecord, Db, RepoRecord}; use chrono::{DateTime, Utc}; use sqlx::PgPool; @@ -3863,6 +3884,146 @@ mod dedup_db_tests { ); } + /// U8 (F): bounties key on (repo_owner, repo_name), not an immutable repo id, + /// and `delete_repo_by_id` deliberately does not delete them (financial record). + /// Without tombstoning, a purged repo's OPEN bounty stays claimable and silently + /// re-attaches to a same-name repo recreated by the (spam) owner. The purge must + /// transition the affected non-terminal bounties to the terminal `purged` status + /// (preserving amount/wallet/tx_hash) so no claim/dispute path can re-expose them. + /// A bounty on an unrelated surviving repo must be left untouched. + /// RED on base: the bounty stays `open` and `claim_bounty` succeeds after recreate. + #[sqlx::test] + async fn delete_repo_by_id_tombstones_bounties_against_recreation(pool: PgPool) { + let db = db(pool).await; + let owner = "did:key:z6MkBountyPurgeFixtureForTombstone"; + let victim = rec( + "rid-bounty-victim", + owner, + "victim", + "spam repo", + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:00Z", + ); + db.create_repo(&victim).await.unwrap(); + + // An unrelated surviving repo with its own open bounty. + let survivor_owner = "did:key:z6MkBountySurvivorFixtureUntouched"; + let survivor = rec( + "rid-bounty-survivor", + survivor_owner, + "keeper", + "legit repo", + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:00Z", + ); + db.create_repo(&survivor).await.unwrap(); + + let bounty = |id: &str, r_owner: &str, r_name: &str, status: &str| BountyRecord { + id: id.to_string(), + repo_owner: r_owner.to_string(), + repo_name: r_name.to_string(), + issue_id: None, + title: "fix the bug".to_string(), + amount: 5_000, + creator_did: "did:key:z6MkCreator".to_string(), + claimant_did: None, + claimant_wallet: None, + pr_id: None, + status: status.to_string(), + created_at: "2026-01-01T00:00:00Z".to_string(), + claimed_at: None, + submitted_at: None, + completed_at: None, + deadline_secs: 604_800, + tx_hash: None, + }; + db.create_bounty(&bounty("bnt-victim", owner, "victim", "open")) + .await + .unwrap(); + // A mid-flight `claimed` bounty must also be tombstoned: otherwise a later + // dispute (which resets `claimed`/`submitted` back to `open`) would re-expose + // it on the recreated repo. + db.create_bounty(&bounty("bnt-victim-claimed", owner, "victim", "claimed")) + .await + .unwrap(); + // An already-terminal `completed` bounty is outside the tombstone set. + db.create_bounty(&bounty( + "bnt-victim-completed", + owner, + "victim", + "completed", + )) + .await + .unwrap(); + db.create_bounty(&bounty("bnt-survivor", survivor_owner, "keeper", "open")) + .await + .unwrap(); + + // Purge the spam repo, then let the owner recreate the same owner/name. + let removed = db.delete_repo_by_id("rid-bounty-victim").await.unwrap(); + assert_eq!(removed, 1, "the purged repo row is deleted"); + let recreated = rec( + "rid-bounty-recreated", + owner, + "victim", + "recreated repo", + "2026-03-01T00:00:00Z", + "2026-03-01T00:00:00Z", + ); + db.create_repo(&recreated).await.unwrap(); + + // The purged repo's bounty is tombstoned, so a claim is a no-op: the + // `UPDATE ... WHERE status='open'` matches nothing and it stays `purged`. + db.claim_bounty( + "bnt-victim", + "did:key:z6MkAttacker", + Some("0xattacker"), + "2026-03-02T00:00:00Z", + ) + .await + .unwrap(); + let victim_bounty = db.get_bounty("bnt-victim").await.unwrap().unwrap(); + assert_eq!( + victim_bounty.status, "purged", + "the purged repo's bounty must be tombstoned, not re-claimable" + ); + assert!( + victim_bounty.claimant_did.is_none(), + "the tombstoned bounty must not accept a new claimant" + ); + // The financial record is preserved (tombstoned, not deleted). + assert_eq!(victim_bounty.amount, 5_000, "amount preserved on tombstone"); + assert_eq!( + victim_bounty.creator_did, "did:key:z6MkCreator", + "creator_did preserved on tombstone" + ); + + // A mid-flight claimed bounty on the same repo is tombstoned too (so a + // dispute cannot reopen it against the recreated repo). + let claimed_bounty = db.get_bounty("bnt-victim-claimed").await.unwrap().unwrap(); + assert_eq!( + claimed_bounty.status, "purged", + "a claimed bounty on the purged repo must also be tombstoned" + ); + // An already-terminal completed bounty is left as-is (not in the tombstone set). + let completed_bounty = db + .get_bounty("bnt-victim-completed") + .await + .unwrap() + .unwrap(); + assert_eq!( + completed_bounty.status, "completed", + "a terminal completed bounty must be left untouched" + ); + + // The unrelated survivor's bounty is untouched and still claimable. + let survivor_bounty = db.get_bounty("bnt-survivor").await.unwrap().unwrap(); + assert_eq!( + survivor_bounty.status, "open", + "an unrelated repo's bounty must stay open" + ); + } + /// The canonical `did:key:` row and the short-owner mirror row of one logical /// repo collapse to a single deduped entry: the canonical row wins and inherits /// the group's most recent `updated_at`. From 97bf91a643ce359c28132134fa13e758da8db490 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 20:00:13 -0500 Subject: [PATCH 33/61] fix(node): serialize create_repo against the purge advisory lock (#196) Purge holds a per-owner/name advisory lock across delete-row + remove-dir, but create_repo took no synchronous lock, so a concurrent same-key create slipped into the row-deleted-but-dir-not-removed window and left a repos row pointing at storage the purge then removed. create_repo now acquires the same lock (lock_repo_blocking -> the guard the purge holds, keyed identically) around its existence-check, init, and insert, releasing on every path. Serializes a same-key create against a purge; the INV-9 colliding-sibling case stays the accepted KTD4 residual. --- crates/gitlawb-node/src/api/repos.rs | 184 ++++++++++++++++++++++ crates/gitlawb-node/src/git/repo_store.rs | 6 +- 2 files changed, 188 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index d32ad541..4bffaace 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -199,6 +199,27 @@ pub async fn create_repo( // Owner is the authenticated agent's DID let owner_did = auth.0; + // Serialize against a concurrent purge of the SAME owner/name. The purge holds + // this per-repo advisory lock across its delete-row + remove-dir, so without + // taking the same lock here a create could slip into the row-deleted-but-dir- + // not-yet-removed window and land a repos row pointing at a directory the purge + // then removes (a dangling row). Held across the existence-check -> init -> + // create_repo sequence so the two operations cannot interleave; released + // explicitly on the success path, and its guard's Drop frees the lock on every + // error path below. Bounded-wait (no object-store I/O), so the uncontended hot + // path returns on the first attempt. (KTD6, R6.) + let repo_lock = state + .repo_store + .lock_repo_blocking(&owner_did, &req.name) + .await + .map_err(|e| AppError::Git(e.to_string()))? + .ok_or_else(|| { + AppError::Git(format!( + "could not acquire repo lock for {owner_did}/{} — held by a live writer or purge", + req.name + )) + })?; + // Check it doesn't already exist if state.db.get_repo(&owner_did, &req.name).await?.is_some() { return Err(AppError::RepoExists(req.name)); @@ -235,6 +256,10 @@ pub async fn create_repo( state.db.create_repo(&record).await?; + // Row + on-disk dir now both exist consistently; the race window is closed, so + // release the per-repo lock before the (best-effort) proof recording below. + repo_lock.release().await; + // Persist the proof so it can travel with the repo and a mirroring peer can // re-verify it (enforce-mode origins only; off/shadow yield no proof here). if let Some(p) = verified_proof { @@ -3471,4 +3496,163 @@ mod tests { ); } } + + // ── U6/R6: create_repo serializes against a same-key purge ─────────────── + + // create_repo must take the SAME per-owner/name advisory lock the purge holds + // (try_lock_repo / RepoLockGuard), so a create cannot slip into the purge's + // delete-row -> [window] -> remove-dir gap and land a repos row pointing at a + // directory the purge then removes (a dangling row). Deterministic form of the + // race: hold the purge lock, then prove create BLOCKS on it rather than + // proceeding into the window — it must not insert its row while the lock is + // held, and must complete cleanly (row + on-disk dir both present) once the + // lock frees. RED on base (create takes no lock): the row lands while the lock + // is held. GREEN after (create serializes on the same key). + #[sqlx::test] + async fn create_repo_serializes_against_purge_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use axum::extract::State; + use axum::Extension; + use std::time::Duration; + + // Multi-connection pool: while the purge guard pins one connection for the + // held lock, create's own try_lock_repo attempt must get a DIFFERENT + // connection and observe the advisory lock held (Ok(None) -> it retries), + // not merely block on pool exhaustion. + let pool = pool_opts + .max_connections(5) + .connect_with(connect_opts) + .await + .unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + + let owner = "did:key:z6MkCreatePurgeRaceAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "racerepo"; + + // The purge path holds this exact per-repo advisory lock across its + // delete-row + remove-dir. Take it via the same helper the purge uses. + let purge_guard = state + .repo_store + .try_lock_repo(owner, name) + .await + .unwrap() + .expect("lock is free before the create"); + + // Kick off a create for the SAME owner/name while the purge holds the lock. + let state2 = state.clone(); + let owner2 = owner.to_string(); + let handle = tokio::spawn(async move { + super::create_repo( + State(state2), + Extension(crate::auth::AuthenticatedDid(owner2)), + axum::http::HeaderMap::new(), + Json(CreateRepoRequest { + name: name.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + }), + ) + .await + }); + + // While the purge lock is held, create must NOT complete its init+insert. + tokio::time::sleep(Duration::from_millis(600)).await; + assert!( + state.db.get_repo(owner, name).await.unwrap().is_none(), + "create_repo must not insert a repos row while a purge holds the same-key \ + lock (RED on base: create takes no lock and the row lands in the window)" + ); + assert!( + !handle.is_finished(), + "create must be blocked on the purge lock, not have completed" + ); + + // Purge releases; create can now proceed and create cleanly. + purge_guard.release().await; + + let created = tokio::time::timeout(Duration::from_secs(8), handle) + .await + .expect("create should finish once the lock frees") + .expect("create task join") + .expect("create_repo returns Ok once it wins the lock"); + assert_eq!(created.0, StatusCode::CREATED); + + // End state is consistent: the row AND its on-disk dir are both present — + // never a row pointing at a removed directory. + assert!( + state.db.get_repo(owner, name).await.unwrap().is_some(), + "repos row must be present after the create wins the lock" + ); + let dir = repos_dir + .path() + .join(owner.replace([':', '/'], "_")) + .join(format!("{name}.git")); + assert!( + dir.exists(), + "the created repo's on-disk dir must be present — no dangling row" + ); + } + + // U6 no-regression: with no concurrent purge, create_repo still succeeds and + // leaves a consistent row + on-disk dir. Guards against the lock acquisition + // wedging the uncontended hot path. + #[sqlx::test] + async fn create_repo_succeeds_without_lock_contention( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use axum::extract::State; + use axum::Extension; + + let pool = pool_opts + .max_connections(5) + .connect_with(connect_opts) + .await + .unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + + let owner = "did:key:z6MkCreateNoContendBBBBBBBBBBBBBBBBBBBBBBBB"; + let name = "solo"; + + let created = super::create_repo( + State(state.clone()), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + axum::http::HeaderMap::new(), + Json(CreateRepoRequest { + name: name.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + }), + ) + .await + .expect("uncontended create_repo must succeed"); + assert_eq!(created.0, StatusCode::CREATED); + + assert!( + state.db.get_repo(owner, name).await.unwrap().is_some(), + "row present after an uncontended create" + ); + let dir = repos_dir + .path() + .join(owner.replace([':', '/'], "_")) + .join(format!("{name}.git")); + assert!( + dir.exists(), + "on-disk dir present after an uncontended create" + ); + } } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 0f4e2284..48a513e0 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -424,8 +424,10 @@ impl RepoStore { /// to `acquire_write`'s spin, without any object-store I/O. Retries /// `try_lock_repo` with short backoff. In practice the fork target's lock is /// uncontended (its DB row is created after the upload), so this returns on - /// the first attempt. - async fn lock_repo_blocking( + /// the first attempt. Also used by `create_repo` to serialize its + /// existence-check -> init -> insert against a concurrent same-key purge + /// (which holds this same lock across delete-row + remove-dir). + pub async fn lock_repo_blocking( &self, owner_did: &str, repo_name: &str, From 02343b4820f140b7cd1b659f8302f74243a447c4 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 20:14:19 -0500 Subject: [PATCH 34/61] test(node): prove purge deletes at MAX_CONNECTIONS=1 with the split pools (#196) The split pool from the earlier fix already resolves the one-connection self-deadlock (the guard holds a lock_pool connection while the delete runs on the app pool), so this locks that wiring in: a purge at app-pool MAX_CONNECTIONS=1 deletes the target row. The test is load-bearing for the wiring - a shared-pool store (guard and delete on one pool) deadlocks the delete's begin() on the held connection and deletes nothing. No production change; U2 did the wiring. --- crates/gitlawb-node/src/admin.rs | 76 ++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/crates/gitlawb-node/src/admin.rs b/crates/gitlawb-node/src/admin.rs index ecc59966..3b8bd83a 100644 --- a/crates/gitlawb-node/src/admin.rs +++ b/crates/gitlawb-node/src/admin.rs @@ -672,6 +672,27 @@ mod db_tests { crate::git::repo_store::RepoStore::for_testing(repos_dir.to_path_buf(), pool.clone()) } + /// A Postgres pool of a fixed size with a short acquire timeout and no ambient + /// reaping, so a purge exercised at GITLAWB_DB_MAX_CONNECTIONS=1 fails fast + /// (rather than stalling the whole test) if the single connection is pinned. + /// min/idle/lifetime pinned so a held connection is never reclaimed + /// mid-assertion. + async fn sized_no_reap_pool( + connect_opts: &sqlx::postgres::PgConnectOptions, + max_connections: u32, + ) -> PgPool { + sqlx::postgres::PgPoolOptions::new() + .max_connections(max_connections) + .acquire_timeout(std::time::Duration::from_secs(2)) + .min_connections(0) + .idle_timeout(None) + .max_lifetime(None) + .test_before_acquire(false) + .connect_with(connect_opts.clone()) + .await + .unwrap() + } + fn rec(id: &str, owner: &str, name: &str) -> RepoRecord { RepoRecord { id: id.to_string(), @@ -829,6 +850,61 @@ mod db_tests { assert!(!path.exists(), "on-disk bare repo dir must be removed too"); } + // U3 (R3/KTD3): a purge at GITLAWB_DB_MAX_CONNECTIONS=1 (app pool size 1) must + // still delete. U2 split the pools: the advisory-lock guard pins a connection + // from the dedicated lock_pool while delete_repo_by_id runs on the app pool, so + // the single app connection is free for the delete's begin() even while the guard + // holds the repo lock. Load-bearing for the admin wiring: if the purge store were + // (mis)wired to share ONE pool for the guard and the delete, the held guard + // connection would leave begin() nothing to acquire, delete would time out + // (PoolTimedOut), and the row would survive — deleted stays 0. The split store + // built here is what keeps the purge correct at one connection. + #[sqlx::test] + async fn purge_at_one_app_connection_deletes_with_split_pools( + _pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + // Migrate + seed on a normal multi-connection pool: migrate() pins one + // connection for its advisory lock while running migration queries on + // others, so it cannot run on a size-1 pool. + let setup_db = Db::for_testing(sized_no_reap_pool(&connect_opts, 5).await); + setup_db.run_migrations().await.unwrap(); + let empty = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + setup_db.create_repo(&empty).await.unwrap(); + + let tmp = tempfile::TempDir::new().unwrap(); + let path = store::repo_disk_path(tmp.path(), &empty.owner_did, &empty.name); + store::init_bare(&path).unwrap(); + assert_eq!( + store::list_refs(&path).unwrap().len(), + 0, + "precondition: the candidate is a real empty bare repo" + ); + + // The purge runs at ONE app connection (finding E's MAX_CONNECTIONS=1) with a + // SEPARATE lock pool. Split store: the guard draws from lock_pool, the delete + // runs on the app pool via `db` — the production wiring from main.rs's + // purge-spam path. + let app_pool = sized_no_reap_pool(&connect_opts, 1).await; + let lock_pool = sized_no_reap_pool(&connect_opts, 1).await; + let db = Db::for_testing(app_pool); + let store = + crate::git::repo_store::RepoStore::new(tmp.path().to_path_buf(), None, lock_pool); + let summary = run_purge_spam(&db, &store, tmp.path(), true).await.unwrap(); + + assert_eq!( + summary.deleted, 1, + "a split-pool purge at one app connection must delete the candidate row" + ); + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none(), + "the target row must be gone after a size-1 purge" + ); + } + // U4 (M4): a repo whose per-repo advisory lock is held by a live writer must be // SKIPPED by purge, not deleted out from under the push. Holds the lock via the // same RepoStore (a separate pooled connection), runs execute, and asserts the From 15435e3d6d14c381291a3052d34f282b3dbcb396 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 20:33:20 -0500 Subject: [PATCH 35/61] fix(node): fail the push when the durable upload times out, don't 200 (#196) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bounded release() upload (from the lock-pool fix) abandoned a slow PUT, freed the lock, and returned no failure signal, so receive-pack reported 200 while the archive never reached object storage — and the next acquire_write downloaded the stale pre-push archive over local disk, reverting the pushed refs. A timed-out or errored durable upload now returns upload_ok=false; the handler sends a 5xx and skips the success tail so the client re-pushes idempotently instead of trusting a lost commit. The lock still frees on every path and the connection-hold bound is unchanged. --- crates/gitlawb-node/src/api/repos.rs | 184 +++++++++++++++++++++- crates/gitlawb-node/src/git/repo_store.rs | 18 ++- 2 files changed, 199 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 4bffaace..77198225 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -989,7 +989,7 @@ pub async fn git_receive_pack( // to prevent stale locks and to avoid holding the write lock across the // fan-out. Only upload to Tigris when the push succeeded; uploading a // half-applied repo would propagate corruption. - guard.release(result.is_ok()).await; + let upload_ok = guard.release(result.is_ok()).await; // On receive-pack error, deliver the classified error to the client and run // NO tail. On success, hand report-status to the client now, then continue @@ -1011,6 +1011,24 @@ pub async fn git_receive_pack( return; } }; + + // The refs applied locally, but the durable upload to object storage + // FAILED or timed out. Report the push as failed (5xx) instead of 200 so + // the idempotent client re-pushes and re-uploads: otherwise the next + // acquire_write downloads the STALE pre-push archive over local disk and + // reverts these refs, silently losing the commit the client believes it + // landed. Skip the whole success tail — a push whose durable copy never + // landed must not be recorded/broadcast as accepted. (P1 data-loss fix.) + if !upload_ok { + tracing::error!(repo = %record.name, + "durable upload failed after receive-pack — reporting push failure so the client retries"); + let _ = report_tx.send(Err(AppError::Git(format!( + "durable storage upload failed for {}", + record.name + )))); + return; + } + let _ = report_tx.send(Ok(response)); // Update the repo's updated_at timestamp after a successful push @@ -3095,6 +3113,170 @@ mod tests { assert_eq!(pushes, 1, "the tail must run on a connected push too"); } + // P1 data-loss regression: when the durable (Tigris) upload in release() + // times out, the refs are applied on local disk but never persisted to + // object storage. The pre-fix handler still reported 200 to the client AND + // ran the success tail, so the client trusted a push that the NEXT + // acquire_write would revert from the stale pre-push archive — silent data + // loss. The fix surfaces a failed/timed-out upload as a FAILED push (5xx) so + // the idempotent client re-pushes, and skips the whole success tail. Here we + // stall the upload past a tiny release timeout, drive a fully-applied push, + // and assert (a) the client gets a non-2xx and (b) the tail did NOT run + // (no push record). RED pre-fix: 200 + push record present. + #[sqlx::test] + async fn durable_upload_timeout_fails_push_and_skips_tail(pool: sqlx::PgPool) { + use axum::extract::{Path as AxPath, State}; + use axum::response::IntoResponse; + use axum::Extension; + use std::path::Path as StdPath; + use std::process::{Command, Stdio}; + use std::time::Duration; + + // Object store whose upload() parks forever: the release() timeout is the + // only thing that unblocks it. `exists()` is false so acquire_write never + // downloads over the fresh local bare repo. + struct StallStore; + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for StallStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(false) + } + async fn upload(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + std::future::pending::<()>().await; + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(StallStore)), + pool.clone(), + ) + .with_release_upload_timeout(Duration::from_millis(200)); + + let owner = "z6durablefailowner"; + let name = "durablefail"; + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + + fn git(args: &[&str], dir: &std::path::Path) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + let work = tempfile::TempDir::new().unwrap(); + git(&["init", "-q", "-b", "main", "."], work.path()); + git(&["config", "user.email", "t@t"], work.path()); + git(&["config", "user.name", "t"], work.path()); + std::fs::write(work.path().join("f.txt"), "hi").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c"], work.path()); + let oid = git(&["rev-parse", "HEAD"], work.path()); + + std::fs::create_dir_all(bare.parent().unwrap()).unwrap(); + let out = Command::new("git") + .args([ + "clone", + "--bare", + "-q", + &work.path().to_string_lossy(), + &bare.to_string_lossy(), + ]) + .output() + .unwrap(); + assert!(out.status.success(), "clone --bare failed"); + git( + &[ + "-C", + &bare.to_string_lossy(), + "update-ref", + "-d", + "refs/heads/main", + ], + work.path(), + ); + + // No hook: the push applies immediately; the stall is entirely in release(). + let zero = "0".repeat(40); + let cmd = format!("{zero} {oid} refs/heads/main\0report-status\n"); + let mut body = Vec::new(); + let len = cmd.len() + 4; + body.extend_from_slice(format!("{len:04x}").as_bytes()); + body.extend_from_slice(cmd.as_bytes()); + body.extend_from_slice(b"0000"); + let pack = Command::new("git") + .args([ + "-C", + &bare.to_string_lossy(), + "pack-objects", + "--stdout", + "-q", + ]) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(pack.status.success(), "pack-objects failed"); + body.extend_from_slice(&pack.stdout); + let body = Bytes::from(body); + + // Await the handler: acquire → receive-pack (applies) → release (upload + // stalls, timing out after 200ms). The client MUST see a failure, not 200. + let resp = super::git_receive_pack( + State(state.clone()), + AxPath((owner.to_string(), format!("{name}.git"))), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + body, + ) + .await; + let status = match resp { + Ok(r) => r.status(), + Err(e) => e.into_response().status(), + }; + assert!( + status.is_server_error(), + "a timed-out durable upload must fail the push (5xx) so the client \ + retries — got {status}, which the client trusts as a landed push \ + that a later acquire_write would silently revert" + ); + + // And the success tail must NOT have run: no push record for a push whose + // durable copy never landed. Give the (detached) tail time to run if it + // were going to, then assert it did not. + tokio::time::sleep(Duration::from_millis(400)).await; + let pushes = state.db.get_push_count(owner).await.unwrap(); + assert_eq!( + pushes, 0, + "the success tail (record_push) must NOT run when the durable upload \ + failed — the push was not durably accepted" + ); + } + /// Repo creation must be throttled by the per-IP creation limiter BEFORE /// signature verification — otherwise a DID farm (one throwaway did:key per /// repo, each carrying a valid but machine-solved iCaptcha proof) walks past diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 48a513e0..a23edc24 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -602,11 +602,21 @@ impl RepoWriteGuard { /// half-applied or otherwise inconsistent repo would propagate corruption to /// Tigris (and to every node that later downloads it). The lock is always /// released regardless, to avoid stale locks blocking future writes. - pub async fn release(mut self, success: bool) { + pub async fn release(mut self, success: bool) -> bool { + // Whether the durable copy is safe. Stays `true` when there is nothing to + // upload (no object store, or `success == false` where no upload is + // attempted) — those are not upload failures. Only an upload that was + // ATTEMPTED and then errored or timed out flips it `false`, which the + // caller surfaces as a FAILED push so the client re-pushes rather than + // trusting a commit that never reached durable storage. (P1 data-loss.) + let mut upload_ok = true; + // Upload to Tigris only on success. Bound the upload so a stalled PUT // cannot pin the guard's advisory-lock connection indefinitely — on // timeout we log and fall through to the unlock, returning the - // connection to the pool within the bound. (INV-22.) + // connection to the pool within the bound. (INV-22.) The lock is freed on + // EVERY arm below regardless of `upload_ok`; only the return value tells + // the caller whether the durable write landed. if success { if let Some(ref tigris) = self.object_store { match tokio::time::timeout( @@ -617,9 +627,11 @@ impl RepoWriteGuard { { Ok(Ok(())) => {} Ok(Err(e)) => { + upload_ok = false; warn!(repo = %self.repo_name, err = %e, "failed to upload repo to tigris after write"); } Err(_) => { + upload_ok = false; warn!(repo = %self.repo_name, timeout_secs = self.release_upload_timeout.as_secs(), "tigris upload timed out after write — releasing the advisory lock without a completed upload"); } @@ -640,6 +652,8 @@ impl RepoWriteGuard { .execute(&mut *conn) .await; } + + upload_ok } } From 9e75c5a2e6b3252f7f9bd9f40893cc183fc05924 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 20:48:20 -0500 Subject: [PATCH 36/61] fix(node): tombstone bounties under both owner forms on purge (#196) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tombstone matched only the repo's full did:key owner, but create_bounty stores repo_owner verbatim from the URL path — commonly the bare short form. A short-owner-form bounty therefore stayed open and re-attached to a recreated repo, the exact hole the tombstone closes for the full form. The UPDATE now matches repo_owner IN (full, normalized short) so both forms are tombstoned. Adds short-form + submitted-status fixtures and asserts dispute cannot reopen a purged bounty. --- crates/gitlawb-node/src/db/mod.rs | 69 ++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 6cbbe576..53c0a86e 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1410,12 +1410,20 @@ impl Db { // terminal `completed`/`cancelled` bounties are left as-is. (Residual: a // tombstoned bounty stays readable/listable under the same owner/name after // recreation — a view, not a claim/payout re-attach; see the plan's KTD8.) + // + // Match BOTH owner representations. `create_bounty` stores `repo_owner` + // verbatim from the URL Path param, which is commonly the bare SHORT form + // (`z6Mk...`), while `repos.owner_did` here is the full `did:key:z6Mk...`. + // Binding only the full form leaves the common short-form bounty `open`, so it + // re-attaches to the recreated repo — the exact hole the tombstone closes. + // `normalize_owner_key` gives the bare short form the URL uses. sqlx::query( "UPDATE bounties SET status = 'purged' - WHERE repo_owner = $1 AND repo_name = $2 + WHERE repo_owner IN ($1, $2) AND repo_name = $3 AND status IN ('open', 'claimed', 'submitted')", ) .bind(&owner_did) + .bind(normalize_owner_key(&owner_did)) .bind(&name) .execute(&mut *tx) .await?; @@ -3667,7 +3675,7 @@ mod agent_discovery_tests { #[cfg(test)] mod dedup_db_tests { - use super::{BountyRecord, Db, RepoRecord}; + use super::{normalize_owner_key, BountyRecord, Db, RepoRecord}; use chrono::{DateTime, Utc}; use sqlx::PgPool; @@ -3940,12 +3948,29 @@ mod dedup_db_tests { db.create_bounty(&bounty("bnt-victim", owner, "victim", "open")) .await .unwrap(); + // The common `create_bounty` path stores `repo_owner` verbatim from the URL + // Path param, which is the bare SHORT owner form (`z6Mk...`), not the full + // `did:key:z6Mk...`. The tombstone must match this form too or a short-form + // bounty stays `open` and re-attaches to the recreated repo. + let short_owner = normalize_owner_key(owner); + db.create_bounty(&bounty("bnt-victim-short", short_owner, "victim", "open")) + .await + .unwrap(); // A mid-flight `claimed` bounty must also be tombstoned: otherwise a later // dispute (which resets `claimed`/`submitted` back to `open`) would re-expose // it on the recreated repo. db.create_bounty(&bounty("bnt-victim-claimed", owner, "victim", "claimed")) .await .unwrap(); + // A mid-flight `submitted` bounty is in the tombstone set too. + db.create_bounty(&bounty( + "bnt-victim-submitted", + owner, + "victim", + "submitted", + )) + .await + .unwrap(); // An already-terminal `completed` bounty is outside the tombstone set. db.create_bounty(&bounty( "bnt-victim-completed", @@ -3998,6 +4023,26 @@ mod dedup_db_tests { "creator_did preserved on tombstone" ); + // The short-owner-form bounty (the common create_bounty path) is tombstoned + // too, so it cannot re-attach to the recreated repo and become claimable. + db.claim_bounty( + "bnt-victim-short", + "did:key:z6MkAttacker", + Some("0xattacker"), + "2026-03-02T00:00:00Z", + ) + .await + .unwrap(); + let short_bounty = db.get_bounty("bnt-victim-short").await.unwrap().unwrap(); + assert_eq!( + short_bounty.status, "purged", + "a short-owner-form bounty on the purged repo must also be tombstoned" + ); + assert!( + short_bounty.claimant_did.is_none(), + "the tombstoned short-form bounty must not accept a new claimant" + ); + // A mid-flight claimed bounty on the same repo is tombstoned too (so a // dispute cannot reopen it against the recreated repo). let claimed_bounty = db.get_bounty("bnt-victim-claimed").await.unwrap().unwrap(); @@ -4005,6 +4050,26 @@ mod dedup_db_tests { claimed_bounty.status, "purged", "a claimed bounty on the purged repo must also be tombstoned" ); + // Dispute gates on status IN ('claimed','submitted'); a tombstoned bounty is + // `purged`, so dispute is inert and cannot reopen it back to `open`. + db.dispute_bounty("bnt-victim-claimed").await.unwrap(); + let disputed_bounty = db.get_bounty("bnt-victim-claimed").await.unwrap().unwrap(); + assert_eq!( + disputed_bounty.status, "purged", + "dispute must not reopen a tombstoned bounty" + ); + + // A mid-flight submitted bounty on the same repo is tombstoned too. + let submitted_bounty = db + .get_bounty("bnt-victim-submitted") + .await + .unwrap() + .unwrap(); + assert_eq!( + submitted_bounty.status, "purged", + "a submitted bounty on the purged repo must also be tombstoned" + ); + // An already-terminal completed bounty is left as-is (not in the tombstone set). let completed_bounty = db .get_bounty("bnt-victim-completed") From 2a3d9e8d593b145df0534b79261eaae2a436b107 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 20:55:34 -0500 Subject: [PATCH 37/61] fix(node): fail closed on a symlink at the purge target (#196) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit local_refs_on_disk used symlink_metadata only to detect NotFound, then the HEAD/objects marker checks followed a symlink — so a symlink to a real empty bare repo outside repos_dir was admitted as an empty purgeable candidate, and the DB row was deleted before path_within refused the dir removal. The symlink is now classified present/skipped before the marker checks. Adds outside-empty-bare and dangling-symlink tests. --- crates/gitlawb-node/src/admin.rs | 65 ++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/crates/gitlawb-node/src/admin.rs b/crates/gitlawb-node/src/admin.rs index 3b8bd83a..2e84d774 100644 --- a/crates/gitlawb-node/src/admin.rs +++ b/crates/gitlawb-node/src/admin.rs @@ -204,6 +204,16 @@ fn local_refs_on_disk(repos_dir: &Path, owner_did: &str, name: &str) -> Option { + warn!(path = %path.display(), + "purge-spam: path is a symlink — treating as non-empty (skipped)"); + return Some(1); + } Ok(_) => {} } if !path.join("HEAD").is_file() || !path.join("objects").is_dir() { @@ -1148,6 +1158,61 @@ mod db_tests { ); } + /// A symlink at a burst repo's expected on-disk path is never a legitimate + /// bare repo we may treat as empty-and-purgeable, even when its target is a + /// real EMPTY bare repo living OUTSIDE repos_dir. `symlink_metadata` only + /// distinguishes NotFound; the marker checks (`HEAD` file, `objects` dir) + /// FOLLOW the link, so the outside repo's 0 refs would be admitted — the DB + /// row is deleted (the `path_within` gate spares the outside dir, but only + /// AFTER delete_repo_by_id). A symlink at the target must fail CLOSED. + #[test] + fn symlink_to_outside_empty_bare_fails_closed() { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + + // A real empty bare repo OUTSIDE repos_dir (HEAD + objects/ => looks bare). + let outside = tmp.path().join("outside.git"); + store::init_bare(&outside).unwrap(); + assert!(outside.join("HEAD").is_file() && outside.join("objects").is_dir()); + + let repo = rec("t-sym", SPAM_BURST_TARGET_DID, "spam1"); + let target = store::repo_disk_path(&repos_dir, &repo.owner_did, &repo.name); + std::fs::create_dir_all(target.parent().unwrap()).unwrap(); + std::os::unix::fs::symlink(&outside, &target).unwrap(); + + // Following the link the empty bare repo reads 0 refs (admitted as empty). + // A symlink at the target must fail closed to Some(1), never Some(0)/None. + assert_eq!( + local_refs_on_disk(&repos_dir, &repo.owner_did, &repo.name), + Some(1), + "a symlink at the target (even to a real empty bare repo outside repos_dir) must fail closed to Some(1)" + ); + } + + /// The DANGLING-symlink case (target does not exist) is why the stat uses + /// `symlink_metadata`, not `metadata`: `metadata` would surface NotFound and + /// wrongly return None (promoting the repo to remote-unverified, whose + /// refresh does a remove_dir_all). `symlink_metadata` succeeds, so the path + /// is present, and the symlink classification fails it closed to Some(1). + #[test] + fn dangling_symlink_fails_closed() { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + + let repo = rec("t-dangle", SPAM_BURST_TARGET_DID, "spam1"); + let target = store::repo_disk_path(&repos_dir, &repo.owner_did, &repo.name); + std::fs::create_dir_all(target.parent().unwrap()).unwrap(); + std::os::unix::fs::symlink("/nonexistent/target", &target).unwrap(); + + assert_eq!( + local_refs_on_disk(&repos_dir, &repo.owner_did, &repo.name), + Some(1), + "a dangling symlink is present (not NotFound) and must fail closed to Some(1)" + ); + } + /// The belt-and-suspenders containment gate (`path_within`) must reject a path /// that resolves outside repos_dir even when the *name* itself is innocuous — /// e.g. the owner slug dir is a symlink pointing elsewhere. Layer 1 (name From 90048b17fe5bc415607a7079539b2d6968babb21 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 21:12:45 -0500 Subject: [PATCH 38/61] fix(node): retry-after on the push-advert 429, and create_repo locks on the app pool (#196) Two review follow-ups: - The unauthenticated info/refs?service=git-receive-pack limiter still returned a bare 429 with no Retry-After while the other limiters advertise the window-derived delay; it now uses check_retry and the shared too_many_requests helper. Removed the orphaned AppError::TooManyRequests variant and gated the now-test-only check(). - create_repo's transient serialization lock drew from the write lock_pool, so a burst that pins lock_pool starved repo creation. Postgres advisory locks are database-global, so create now takes its lock from the app pool (via lock_repo_blocking_on) and still serializes against a purge holding the same key from lock_pool. --- crates/gitlawb-node/src/api/repos.rs | 165 +++++++++++++++++++++- crates/gitlawb-node/src/db/mod.rs | 13 +- crates/gitlawb-node/src/error.rs | 6 - crates/gitlawb-node/src/git/repo_store.rs | 40 +++++- crates/gitlawb-node/src/rate_limit.rs | 4 +- 5 files changed, 204 insertions(+), 24 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 77198225..70caebe1 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -208,9 +208,16 @@ pub async fn create_repo( // explicitly on the success path, and its guard's Drop frees the lock on every // error path below. Bounded-wait (no object-store I/O), so the uncontended hot // path returns on the first attempt. (KTD6, R6.) + // + // Take this transient lock on the APP pool, not the write lock_pool. lock_pool + // is sized for long-held receive-pack write guards; a push burst that pins it + // would otherwise starve create's acquire (-> 500). The advisory lock is + // DB-global, so locking from the app pool still serializes against a same-key + // purge holding the lock from lock_pool. (INV-15 sibling: don't couple a cheap + // path to the expensive path's capacity.) let repo_lock = state .repo_store - .lock_repo_blocking(&owner_did, &req.name) + .lock_repo_blocking_on(state.db.pool(), &owner_did, &req.name) .await .map_err(|e| AppError::Git(e.to_string()))? .ok_or_else(|| { @@ -579,11 +586,12 @@ pub async fn git_info_refs( // trusted-proxy policy as the POST middleware (shared buckets). if service == "git-receive-pack" { if let Some(key) = crate::rate_limit::client_key(&headers, peer, state.push_limiter_trust) { - if !state.push_rate_limiter.check(&key).await { + // Use check_retry (like U5's other 429 sites) so the rejection carries a + // window-derived Retry-After, built via the shared 429 response helper + // that the per-IP middleware also uses. + if let Some(retry_after) = state.push_rate_limiter.check_retry(&key).await { tracing::warn!(repo = %name, key = %key, "receive-pack advertisement rate limited"); - return Err(AppError::TooManyRequests( - "push rate limit exceeded — try again later".into(), - )); + return Ok(crate::rate_limit::too_many_requests(retry_after)); } } } @@ -2678,6 +2686,60 @@ mod tests { ); } + /// The receive-pack advertisement 429 must carry a window-derived Retry-After, + /// consistent with the other push 429 sites (U5). Before the fix this site + /// returned a bare 429 with no Retry-After header at all — a client had nothing + /// to back off on. A freshly-filled bucket's oldest + /// entry is ~now, so the advertised delay must be close to the whole window + /// (100s), never missing and never the old constant 60. + #[sqlx::test] + async fn receive_pack_advertisement_429_carries_window_derived_retry_after(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Budget 1, 100s window, keyed on the socket peer (no trusted proxy). + state.push_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(100)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6advretry", "adv", "/tmp/advretry", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.56:6000".parse().unwrap(); + // Fill the single-request budget so the handler's own check rejects. + assert!(state.push_rate_limiter.check(&peer.ip().to_string()).await); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6advretry/adv/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let resp = router.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS); + let retry: u64 = resp + .headers() + .get("retry-after") + .expect("advertisement 429 must carry a retry-after header") + .to_str() + .unwrap() + .parse() + .expect("retry-after must be an integer number of seconds"); + assert!( + (95..=100).contains(&retry), + "freshly-filled bucket must advertise ~window (95..=100s), got {retry} \ + (missing header or constant-60 bug otherwise)" + ); + } + // U2/R3/AE1: a fully-received push must complete server-side even when the // client disconnects during the apply. The pack is buffered before the // handler runs, so the acquire→receive→release core is detached from the @@ -3837,4 +3899,97 @@ mod tests { "on-disk dir present after an uncontended create" ); } + + // create_repo's transient serialization lock must NOT draw from the write + // lock_pool. lock_pool is sized for LONG-HELD receive-pack write guards; a + // concurrent-push burst that pins every lock_pool connection would otherwise + // starve legitimate repo creation (create's acquire times out -> 500). The fix + // takes create's transient lock on the APP pool instead, which the write burst + // never touches. RED on base (create locks on lock_pool): with the lock_pool + // saturated by held write guards, create errors. GREEN after (create locks on + // the app pool): create still succeeds. + #[sqlx::test] + async fn create_repo_survives_a_saturated_lock_pool( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use axum::extract::State; + use axum::Extension; + use std::time::Duration; + + // App pool: serves state.db AND (after the fix) create's transient lock. + let app_pool = pool_opts + .max_connections(5) + .connect_with(connect_opts.clone()) + .await + .unwrap(); + let mut state = crate::test_support::test_state(app_pool.clone()).await; + + // A SEPARATE, small write lock pool that fails fast on exhaustion, so a + // create wired to it errors quickly rather than blocking the whole test. + const N: u32 = 2; + let lock_pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(N) + .acquire_timeout(Duration::from_secs(2)) + .min_connections(0) + .idle_timeout(None) + .max_lifetime(None) + .test_before_acquire(false) + .connect_with(connect_opts.clone()) + .await + .unwrap(); + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + lock_pool.clone(), + ); + + // Saturate the lock pool with N held write guards on distinct repos — the + // 32-writer burst, scaled down. Each guard pins one lock_pool connection. + let mut guards = Vec::new(); + for i in 0..N { + let owner = format!("did:key:z6MkSaturateLock{i}AAAAAAAAAAAAAAAAAAAAAA"); + guards.push( + state + .repo_store + .acquire_write(&owner, "held") + .await + .unwrap(), + ); + } + // The lock pool is exhausted, so the create assertion below is load-bearing + // (not vacuously green because the pool happened to be idle). + assert!( + lock_pool.acquire().await.is_err(), + "the lock pool must be exhausted by N held write guards" + ); + + // create for an UNCONTENDED key. Its advisory lock is free (the write + // guards hold different keys), so the only thing that can stop it is the + // saturated lock_pool — which it must not touch. + let owner = "did:key:z6MkCreateAppPoolCCCCCCCCCCCCCCCCCCCCCCCC"; + let name = "created"; + let created = super::create_repo( + State(state.clone()), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + axum::http::HeaderMap::new(), + Json(CreateRepoRequest { + name: name.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + }), + ) + .await + .expect("create must survive a saturated lock pool by locking on the app pool"); + assert_eq!(created.0, StatusCode::CREATED); + assert!( + state.db.get_repo(owner, name).await.unwrap().is_some(), + "row present after a create under a saturated lock pool" + ); + + for g in guards { + g.release(true).await; + } + } } diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 53c0a86e..b89ee5c3 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -247,12 +247,13 @@ pub struct Db { } impl Db { - /// Access the underlying Postgres connection pool. Test-only: production DB - /// access goes through `Db`'s methods, and the advisory-lock subsystem now - /// uses its own pool (`connect_lock_pool`), so nothing outside tests reaches - /// for the raw app pool. - #[cfg(test)] - pub fn pool(&self) -> &PgPool { + /// Access the underlying Postgres connection pool. Most DB access goes through + /// `Db`'s methods; the write advisory-lock subsystem uses its own pool + /// (`connect_lock_pool`). `create_repo` is the one production caller: its + /// transient serialization lock is taken on THIS app pool rather than the + /// write lock pool, so a write burst that pins the lock pool cannot starve + /// repo creation. + pub(crate) fn pool(&self) -> &PgPool { &self.pool } diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index bdd5aec9..b6e1ab5a 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -35,9 +35,6 @@ pub enum AppError { #[error("invalid request: {0}")] BadRequest(String), - #[error("too many requests: {0}")] - TooManyRequests(String), - #[error("incomplete: {0}")] Incomplete(String), @@ -132,9 +129,6 @@ impl IntoResponse for AppError { // IcaptchaProofRequired is handled above (it carries extra headers/fields). AppError::IcaptchaProofRequired { .. } => unreachable!("handled before this match"), AppError::BadRequest(msg) => (StatusCode::BAD_REQUEST, "bad_request", msg.clone()), - AppError::TooManyRequests(msg) => { - (StatusCode::TOO_MANY_REQUESTS, "rate_limited", msg.clone()) - } AppError::Incomplete(msg) => { (StatusCode::UNPROCESSABLE_ENTITY, "incomplete", msg.clone()) } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index a23edc24..a413467a 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -292,14 +292,25 @@ impl RepoStore { &self, owner_did: &str, repo_name: &str, + ) -> Result> { + self.try_lock_repo_on(&self.lock_pool, owner_did, repo_name) + .await + } + + /// [`try_lock_repo`](Self::try_lock_repo) against a caller-chosen pool. The + /// Postgres advisory lock is DATABASE-global, so the lock still mutually + /// excludes holders of the same key from any other pool — only the connection + /// the guard pins comes from `pool`. `create_repo` uses this with the app pool + /// so its transient lock never draws from the write `lock_pool`. + async fn try_lock_repo_on( + &self, + pool: &PgPool, + owner_did: &str, + repo_name: &str, ) -> Result> { let (owner_slug, _local) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); - let mut conn = self - .lock_pool - .acquire() - .await - .context("acquiring lock connection")?; + let mut conn = pool.acquire().await.context("acquiring lock connection")?; let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") .bind(lock_key) .fetch_one(&mut *conn) @@ -431,9 +442,26 @@ impl RepoStore { &self, owner_did: &str, repo_name: &str, + ) -> Result> { + self.lock_repo_blocking_on(&self.lock_pool, owner_did, repo_name) + .await + } + + /// [`lock_repo_blocking`](Self::lock_repo_blocking) against a caller-chosen + /// pool. `create_repo` passes the app pool so its transient serialization lock + /// (check-exists -> init -> insert) does not draw a connection from the write + /// `lock_pool` — a receive-pack burst that pins every lock-pool connection + /// must not starve repo creation. The advisory lock is DB-global, so this + /// still serializes create against a same-key purge that holds the lock from + /// the lock pool. + pub async fn lock_repo_blocking_on( + &self, + pool: &PgPool, + owner_did: &str, + repo_name: &str, ) -> Result> { for attempt in 0..30 { - if let Some(g) = self.try_lock_repo(owner_did, repo_name).await? { + if let Some(g) = self.try_lock_repo_on(pool, owner_did, repo_name).await? { return Ok(Some(g)); } if attempt < 29 { diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index e191cb3e..316e3880 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -72,7 +72,9 @@ impl RateLimiter { /// Returns `true` if the request is allowed. Thin wrapper over /// [`check_retry`](Self::check_retry) for callers that only need the - /// allow/deny decision, not the rejection's Retry-After delay. + /// allow/deny decision, not the rejection's Retry-After delay. Test-only now: + /// production 429 sites use `check_retry` so they can advertise Retry-After. + #[cfg(test)] pub(crate) async fn check(&self, key: &str) -> bool { self.check_retry(key).await.is_none() } From 2ca205c0dd94962998db8a7705dd09044564f6a2 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 21:46:18 -0500 Subject: [PATCH 39/61] fix(node): correct the create-pool and bounty-tombstone remediations (#196) A re-run adversarial panel caught three issues in the prior round's own fixes: - create_repo took its serialization lock from the app pool and then did its DB work on the same pool, so at MAX_CONNECTIONS=1 it self-deadlocked (a config finding E requires to work). Reverted to locking on the write lock_pool: lock and create-work use separate pools, and the DB-global advisory lock still serializes against a purge. The extreme-burst lock_pool wait is an accepted retryable-503 residual. Db::pool() is test-only again. - The bounty tombstone matched only one owner form and ran unconditionally. It now normalizes BOTH sides (covering did:key-row + short-bounty AND bare-row + full-bounty) and is gated on the same no-surviving-sibling check as the slug cascade, so it never tombstones a bounty a surviving mirror row still serves. Tests: create-at-1-app-connection (no self-deadlock), bare-row full-form tombstone, and sibling-survives-keeps-bounty-open, each RED before / GREEN after. --- crates/gitlawb-node/src/api/repos.rs | 103 +++++++-------- crates/gitlawb-node/src/db/mod.rs | 145 ++++++++++++++++++---- crates/gitlawb-node/src/git/repo_store.rs | 12 +- 3 files changed, 169 insertions(+), 91 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 70caebe1..54b40c39 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -209,15 +209,18 @@ pub async fn create_repo( // error path below. Bounded-wait (no object-store I/O), so the uncontended hot // path returns on the first attempt. (KTD6, R6.) // - // Take this transient lock on the APP pool, not the write lock_pool. lock_pool - // is sized for long-held receive-pack write guards; a push burst that pins it - // would otherwise starve create's acquire (-> 500). The advisory lock is - // DB-global, so locking from the app pool still serializes against a same-key - // purge holding the lock from lock_pool. (INV-15 sibling: don't couple a cheap - // path to the expensive path's capacity.) + // Lock on the write lock_pool (via `lock_repo_blocking`), NOT the app pool. The + // create's own work (get_repo, proof.consume, db.create_repo) runs on the app + // pool via state.db, so drawing the lock guard from a SEPARATE pool keeps it from + // pinning an app connection that work then needs. At GITLAWB_DB_MAX_CONNECTIONS=1 + // an app-pool lock self-deadlocks (get_repo waits for the one connection the + // guard holds -> PoolTimedOut -> 500). The advisory lock is DB-global, so it + // still serializes against a same-key purge holding it from lock_pool. Accepted + // residual: a sustained 32-writer burst that pins lock_pool can make this acquire + // wait -> a retryable 503, far less bad than breaking create at MAX_CONNECTIONS=1. let repo_lock = state .repo_store - .lock_repo_blocking_on(state.db.pool(), &owner_did, &req.name) + .lock_repo_blocking(&owner_did, &req.name) .await .map_err(|e| AppError::Git(e.to_string()))? .ok_or_else(|| { @@ -3900,16 +3903,17 @@ mod tests { ); } - // create_repo's transient serialization lock must NOT draw from the write - // lock_pool. lock_pool is sized for LONG-HELD receive-pack write guards; a - // concurrent-push burst that pins every lock_pool connection would otherwise - // starve legitimate repo creation (create's acquire times out -> 500). The fix - // takes create's transient lock on the APP pool instead, which the write burst - // never touches. RED on base (create locks on lock_pool): with the lock_pool - // saturated by held write guards, create errors. GREEN after (create locks on - // the app pool): create still succeeds. + // create_repo must NOT take its serialization lock from the APP pool. Doing so + // pins an app-pool connection in the lock guard while the create's own work + // (get_repo / proof.consume / db.create_repo, all on state.db = the app pool) + // still needs an app connection. At GITLAWB_DB_MAX_CONNECTIONS=1 (a supported + // config) the guard holds the only app connection and get_repo self-deadlocks + // (PoolTimedOut -> 500). Locking on the separate write lock_pool instead keeps + // the lock and the work on different pools, so a size-1 app pool never wedges. + // RED on the app-pool-lock code: create times out -> Err. GREEN after reverting + // create to lock on lock_pool. #[sqlx::test] - async fn create_repo_survives_a_saturated_lock_pool( + async fn create_repo_does_not_self_deadlock_at_one_app_connection( pool_opts: sqlx::postgres::PgPoolOptions, connect_opts: sqlx::postgres::PgConnectOptions, ) { @@ -3917,24 +3921,30 @@ mod tests { use axum::Extension; use std::time::Duration; - // App pool: serves state.db AND (after the fix) create's transient lock. - let app_pool = pool_opts - .max_connections(5) + // Migrate the per-test DB on a normal pool: `migrate()` pins one connection for + // its cross-process advisory lock while applying statements on a second, so it + // needs >1 connection and cannot itself run on the size-1 pool under test. + let migrate_pool = pool_opts.connect_with(connect_opts.clone()).await.unwrap(); + let mut state = crate::test_support::test_state(migrate_pool).await; + + // Now point state.db at an APP pool of MAX size 1 on the SAME database, the + // minimal supported config (GITLAWB_DB_MAX_CONNECTIONS=1). The schema is + // already applied, so this pool only serves create's work (get_repo -> insert). + // A short acquire timeout makes the RED case (the lock guard pinning this one + // connection while get_repo waits for another) fail fast instead of hanging. + let app_pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(Duration::from_secs(2)) .connect_with(connect_opts.clone()) .await .unwrap(); - let mut state = crate::test_support::test_state(app_pool.clone()).await; + state.db = std::sync::Arc::new(crate::db::Db::for_testing(app_pool.clone())); - // A SEPARATE, small write lock pool that fails fast on exhaustion, so a - // create wired to it errors quickly rather than blocking the whole test. - const N: u32 = 2; + // A SEPARATE write lock pool (where create's serialization lock belongs). Its + // being separate is the whole point: the lock guard must not draw from the app + // pool the create work runs on. let lock_pool = sqlx::postgres::PgPoolOptions::new() - .max_connections(N) - .acquire_timeout(Duration::from_secs(2)) - .min_connections(0) - .idle_timeout(None) - .max_lifetime(None) - .test_before_acquire(false) + .max_connections(5) .connect_with(connect_opts.clone()) .await .unwrap(); @@ -3944,31 +3954,8 @@ mod tests { lock_pool.clone(), ); - // Saturate the lock pool with N held write guards on distinct repos — the - // 32-writer burst, scaled down. Each guard pins one lock_pool connection. - let mut guards = Vec::new(); - for i in 0..N { - let owner = format!("did:key:z6MkSaturateLock{i}AAAAAAAAAAAAAAAAAAAAAA"); - guards.push( - state - .repo_store - .acquire_write(&owner, "held") - .await - .unwrap(), - ); - } - // The lock pool is exhausted, so the create assertion below is load-bearing - // (not vacuously green because the pool happened to be idle). - assert!( - lock_pool.acquire().await.is_err(), - "the lock pool must be exhausted by N held write guards" - ); - - // create for an UNCONTENDED key. Its advisory lock is free (the write - // guards hold different keys), so the only thing that can stop it is the - // saturated lock_pool — which it must not touch. - let owner = "did:key:z6MkCreateAppPoolCCCCCCCCCCCCCCCCCCCCCCCC"; - let name = "created"; + let owner = "did:key:z6MkOneAppConnNoDeadlockDDDDDDDDDDDDDDDDDDDD"; + let name = "solo"; let created = super::create_repo( State(state.clone()), Extension(crate::auth::AuthenticatedDid(owner.to_string())), @@ -3981,15 +3968,11 @@ mod tests { }), ) .await - .expect("create must survive a saturated lock pool by locking on the app pool"); + .expect("create must not self-deadlock at app-pool size 1 (lock on lock_pool)"); assert_eq!(created.0, StatusCode::CREATED); assert!( state.db.get_repo(owner, name).await.unwrap().is_some(), - "row present after a create under a saturated lock pool" + "row present after a create with a size-1 app pool" ); - - for g in guards { - g.release(true).await; - } } } diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index b89ee5c3..dcde7c29 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -247,13 +247,12 @@ pub struct Db { } impl Db { - /// Access the underlying Postgres connection pool. Most DB access goes through - /// `Db`'s methods; the write advisory-lock subsystem uses its own pool - /// (`connect_lock_pool`). `create_repo` is the one production caller: its - /// transient serialization lock is taken on THIS app pool rather than the - /// write lock pool, so a write burst that pins the lock pool cannot starve - /// repo creation. - pub(crate) fn pool(&self) -> &PgPool { + /// Access the underlying Postgres connection pool. Test-only: all production DB + /// access goes through `Db`'s methods, and the write advisory-lock subsystem uses + /// its own pool (`connect_lock_pool`). Tests reach for the raw pool to seed and + /// assert directly. + #[cfg(test)] + pub fn pool(&self) -> &PgPool { &self.pool } @@ -917,6 +916,18 @@ pub(crate) fn normalize_owner_key(did: &str) -> &str { /// drift apart. If you change `normalize_owner_key`, update this const too. const OWNER_KEY_CASE_SQL: &str = "CASE WHEN owner_did LIKE 'did:key:%' AND position(':' in substr(owner_did, 9)) = 0 THEN substr(owner_did, 9) ELSE owner_did END"; +/// `OWNER_KEY_CASE_SQL` over an arbitrary owner column. The const above hardcodes +/// `owner_did` (the `repos` column the `idx_repos_owner_key_name` index is built +/// on); the bounty tombstone needs the same normalizer over `bounties.repo_owner`, +/// which is not index-backed. Byte-identical to `normalize_owner_key`; update this +/// alongside the const and the fn if the rule changes. +fn owner_key_case_sql(col: &str) -> String { + format!( + "CASE WHEN {col} LIKE 'did:key:%' AND position(':' in substr({col}, 9)) = 0 \ + THEN substr({col}, 9) ELSE {col} END" + ) +} + #[cfg(test)] mod normalize_owner_key_tests { use super::normalize_owner_key; @@ -1412,22 +1423,34 @@ impl Db { // tombstoned bounty stays readable/listable under the same owner/name after // recreation — a view, not a claim/payout re-attach; see the plan's KTD8.) // - // Match BOTH owner representations. `create_bounty` stores `repo_owner` - // verbatim from the URL Path param, which is commonly the bare SHORT form - // (`z6Mk...`), while `repos.owner_did` here is the full `did:key:z6Mk...`. - // Binding only the full form leaves the common short-form bounty `open`, so it - // re-attaches to the recreated repo — the exact hole the tombstone closes. - // `normalize_owner_key` gives the bare short form the URL uses. - sqlx::query( - "UPDATE bounties SET status = 'purged' - WHERE repo_owner IN ($1, $2) AND repo_name = $3 - AND status IN ('open', 'claimed', 'submitted')", - ) - .bind(&owner_did) - .bind(normalize_owner_key(&owner_did)) - .bind(&name) - .execute(&mut *tx) - .await?; + // Gate the tombstone on the SAME sibling check as the cascade: only when no + // other repos row resolves to this slug. If a mirror row survives, the logical + // repo still exists (and still serves via the mirror), so its bounty is still + // valid — tombstoning it would kill a live repo's bounty. Only when the last + // row for the slug is gone can a recreate re-attach an open bounty. + // + // When it does fire, match EVERY owner representation by normalizing BOTH sides. + // `create_bounty` stores `repo_owner` verbatim from the URL Path param, commonly + // the bare SHORT form (`z6Mk...`), but a full-DID URL that resolves to a bare + // mirror row yields the FULL `did:key:z6Mk...` form. The repo row here is likewise + // either form (`delete_repo_by_id` is reachable on bare mirror rows). Comparing the + // SQL-normalized stored `repo_owner` against the normalized target covers all four + // combinations (did:key-row + short-bounty AND bare-row + full-bounty); binding + // only one side left the mirror form `open`, so it re-attached to the recreated + // repo. It's a rare purge, so a seq scan on `bounties` (the normalizer is not + // index-backed) is fine. + if !sibling_exists { + sqlx::query(&format!( + "UPDATE bounties SET status = 'purged' + WHERE {key} = $1 AND repo_name = $2 + AND status IN ('open', 'claimed', 'submitted')", + key = owner_key_case_sql("repo_owner") + )) + .bind(normalize_owner_key(&owner_did)) + .bind(&name) + .execute(&mut *tx) + .await?; + } let result = sqlx::query("DELETE FROM repos WHERE id = $1") .bind(id) @@ -3854,6 +3877,30 @@ mod dedup_db_tests { .await .unwrap(); + // An OPEN bounty on the shared owner/name — it belongs to the logical repo the + // surviving mirror still serves, so purging the empty sibling must NOT tombstone it. + db.create_bounty(&BountyRecord { + id: "bnt-collision".to_string(), + repo_owner: "z6MkSiblingCollisionFixture".to_string(), + repo_name: "victim".to_string(), + issue_id: None, + title: "sibling bounty".to_string(), + amount: 5_000, + creator_did: "did:key:z6MkCreator".to_string(), + claimant_did: None, + claimant_wallet: None, + pr_id: None, + status: "open".to_string(), + created_at: "2026-02-01T00:00:00Z".to_string(), + claimed_at: None, + submitted_at: None, + completed_at: None, + deadline_secs: 604_800, + tx_hash: None, + }) + .await + .unwrap(); + // Purge the empty canonical row. let removed = db.delete_repo_by_id("rid-canonical-empty").await.unwrap(); assert_eq!(removed, 1, "the empty canonical row is deleted"); @@ -3891,6 +3938,17 @@ mod dedup_db_tests { 1, "the surviving mirror repos row is untouched" ); + // The bounty is NOT tombstoned: the mirror still serves the logical repo, so + // the tombstone is gated on no sibling surviving. + assert_eq!( + db.get_bounty("bnt-collision") + .await + .unwrap() + .unwrap() + .status, + "open", + "a bounty on a repo whose mirror survives the purge must stay open, not be tombstoned" + ); } /// U8 (F): bounties key on (repo_owner, repo_name), not an immutable repo id, @@ -4088,6 +4146,47 @@ mod dedup_db_tests { survivor_bounty.status, "open", "an unrelated repo's bounty must stay open" ); + + // Symmetric owner-form case (the mirror of `bnt-victim-short`). Here the repo + // row is the BARE short-owner mirror (`owner_did = z6X`), while the bounty was + // stored in the FULL `did:key:z6X` form, the shape a full-DID URL that + // resolves to the bare mirror produces. The tombstone must normalize BOTH the + // stored `repo_owner` and the target, or this full-form bounty stays `open` + // when the bare row is purged and re-attaches to a recreated repo, the same + // re-attach hole in the opposite direction. `delete_repo_by_id` is reachable on + // bare mirror rows (`list_repos_by_owner_did` matches under normalization). + // RED before the symmetric fix (only the target side was normalized): stays + // `open`. GREEN after: `purged`. + let bare_owner = "z6MkBareOwnerFixtureForSymmetricTombstone"; + let bare = rec( + "rid-bounty-bare-victim", + bare_owner, + "victim", + "bare spam repo", + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:00Z", + ); + db.create_repo(&bare).await.unwrap(); + db.create_bounty(&bounty( + "bnt-bare-full", + &format!("did:key:{bare_owner}"), + "victim", + "open", + )) + .await + .unwrap(); + let removed_bare = db + .delete_repo_by_id("rid-bounty-bare-victim") + .await + .unwrap(); + assert_eq!(removed_bare, 1, "the bare-owner repo row is deleted"); + let bare_full_bounty = db.get_bounty("bnt-bare-full").await.unwrap().unwrap(); + assert_eq!( + bare_full_bounty.status, "purged", + "a FULL-form bounty on a BARE-owner repo row must be tombstoned too \ + (symmetric normalization); RED before: only the target side was \ + normalized, so the full-form bounty stayed open" + ); } /// The canonical `did:key:` row and the short-owner mirror row of one logical diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index a413467a..9e2d6faa 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -300,8 +300,7 @@ impl RepoStore { /// [`try_lock_repo`](Self::try_lock_repo) against a caller-chosen pool. The /// Postgres advisory lock is DATABASE-global, so the lock still mutually /// excludes holders of the same key from any other pool — only the connection - /// the guard pins comes from `pool`. `create_repo` uses this with the app pool - /// so its transient lock never draws from the write `lock_pool`. + /// the guard pins comes from `pool`. Backs the `lock_pool`-based delegators. async fn try_lock_repo_on( &self, pool: &PgPool, @@ -448,12 +447,9 @@ impl RepoStore { } /// [`lock_repo_blocking`](Self::lock_repo_blocking) against a caller-chosen - /// pool. `create_repo` passes the app pool so its transient serialization lock - /// (check-exists -> init -> insert) does not draw a connection from the write - /// `lock_pool` — a receive-pack burst that pins every lock-pool connection - /// must not starve repo creation. The advisory lock is DB-global, so this - /// still serializes create against a same-key purge that holds the lock from - /// the lock pool. + /// pool. The advisory lock is DB-global, so the guard mutually excludes the same + /// key across pools; only the pinned connection comes from `pool`. Backs the + /// `lock_pool`-based [`lock_repo_blocking`](Self::lock_repo_blocking) delegator. pub async fn lock_repo_blocking_on( &self, pool: &PgPool, From eef62d7c0d725d7bc3470498ef5d67e64e4981d8 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 22:16:53 -0500 Subject: [PATCH 40/61] fix(node): close the durable-upload data-loss class on every write path (#196) The earlier fix that fails a push when its Tigris upload times out was wired only into git_receive_pack; the merge, issue-create, and issue-close write paths still ignored release()'s result and returned 2xx after a failed upload, so the next acquire_write reverted their change from the stale archive. release() is now #[must_use], which surfaces every caller: the production upload paths fail with a 5xx on a false return (so the client retries), and the internal/test callers are explicitly justified. create_repo's lock-acquire failure now returns a retryable 503 (new AppError::Unavailable) instead of a 500, matching its own contract. --- crates/gitlawb-node/src/api/issues.rs | 156 +++++++++++++++++++++- crates/gitlawb-node/src/api/pulls.rs | 19 ++- crates/gitlawb-node/src/api/repos.rs | 9 +- crates/gitlawb-node/src/error.rs | 27 ++++ crates/gitlawb-node/src/git/repo_store.rs | 27 ++-- 5 files changed, 218 insertions(+), 20 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 17acf9af..27b164c0 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -70,11 +70,25 @@ pub async fn create_issue( let create_result = git_issues::create_issue(&disk_path, &issue_id, &json_str); - // Always release the advisory lock — even on error; upload to Tigris only on success. - guard.release(create_result.is_ok()).await; + // Always release the advisory lock, even on error; upload to Tigris only on + // success. A false return means the durable upload failed/timed out: the + // issue ref applied to local disk but never reached object storage, so the + // next acquire_write would revert it from the stale archive. Report 5xx so + // the client retries rather than trusting an issue that never landed durably, + // and skip the trust bump below. (Same P1 data-loss guard as receive-pack.) + let upload_ok = guard.release(create_result.is_ok()).await; create_result.map_err(|e| AppError::Git(e.to_string()))?; + if !upload_ok { + tracing::error!(repo = %record.name, + "durable upload failed after issue create; reporting failure so the client retries"); + return Err(AppError::Git(format!( + "durable storage upload failed for {}", + record.name + ))); + } + // Bump trust score for the issue author — increment current score by 0.05 // (avoids the push_count=0 stuck-at-0.05 bug for agents who only file issues) if let Some(ref author_did) = issue.author { @@ -244,11 +258,14 @@ pub async fn close_issue( .ok() .and_then(|i| i.author), Ok(None) => { - guard.release(false).await; + // release(false): no upload attempted, so the bool is always true + // (nothing to fail on). Ignore it on this pre-write error path. + let _ = guard.release(false).await; return Err(AppError::NotFound(format!("issue {issue_id} not found"))); } Err(e) => { - guard.release(false).await; + // release(false): no upload attempted, nothing to fail on. + let _ = guard.release(false).await; return Err(AppError::Git(e.to_string())); } }; @@ -257,7 +274,8 @@ pub async fn close_issue( .as_deref() .is_some_and(|a| crate::api::did_matches(&auth.0, a)); if !is_owner && !is_author { - guard.release(false).await; + // release(false): no upload attempted, nothing to fail on. + let _ = guard.release(false).await; return Err(AppError::Forbidden( "only the repo owner or the issue author can close this issue".into(), )); @@ -265,13 +283,25 @@ pub async fn close_issue( let close_result = git_issues::close_issue(&disk_path, &issue_id); - // Always release the advisory lock — even on error; upload to Tigris only on success. - guard.release(close_result.is_ok()).await; + // Always release the advisory lock, even on error; upload to Tigris only on + // success. A false return means the durable upload failed: the close applied + // to local disk but never reached object storage, so a later acquire_write + // reverts it. Report 5xx so the client retries. (P1 data-loss guard.) + let upload_ok = guard.release(close_result.is_ok()).await; let updated = close_result .map_err(|e| AppError::Git(e.to_string()))? .ok_or_else(|| AppError::RepoNotFound(format!("issue {issue_id} not found")))?; + if !upload_ok { + tracing::error!(repo = %repo, + "durable upload failed after issue close; reporting failure so the client retries"); + return Err(AppError::Git(format!( + "durable storage upload failed for {}", + record.name + ))); + } + let issue: serde_json::Value = serde_json::from_str(&updated) .map_err(|e| AppError::BadRequest(format!("invalid issue data: {e}")))?; @@ -279,3 +309,115 @@ pub async fn close_issue( Ok(Json(issue)) } + +#[cfg(test)] +mod tests { + use super::*; + use axum::extract::{Path as AxPath, State}; + use axum::Extension; + use std::path::Path as StdPath; + use std::time::Duration; + + // Object store whose upload() parks forever, so release()'s bounded timeout + // is the only thing that completes it, modeling a stalled durable PUT. exists() + // is false so acquire_write never downloads over the fresh local bare repo. + struct StallStore; + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for StallStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(false) + } + async fn upload(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + std::future::pending::<()>().await; + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + // P1-class data-loss regression, issue-create path. create_issue writes the + // issue ref to local disk, then release(true) uploads the repo to durable + // storage. When that upload times out the local write is NOT persisted, so + // the next acquire_write reverts it from the stale archive. The pre-fix + // handler ignored release()'s bool and still returned 201 AND bumped the + // author's trust score, so the client trusted an issue that never landed + // durably. The fix surfaces a failed/timed-out upload as a 5xx and skips the + // success tail (the trust bump). Stall the upload past a tiny release timeout + // and assert (a) the handler returns 5xx and (b) the trust bump did not run. + // RED pre-fix: 201 + trust score bumped to 0.05. + #[sqlx::test] + async fn issue_create_durable_upload_timeout_fails_and_skips_side_effects(pool: sqlx::PgPool) { + use axum::response::IntoResponse; + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(StallStore)), + pool.clone(), + ) + .with_release_upload_timeout(Duration::from_millis(200)); + + let owner = "z6issueownerdurablefail"; + let name = "issuedurablefail"; + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + // Public mirror repo so the owner clears authorize_repo_read, and a real + // agent row so the post-write trust bump is an observable durable side + // effect (register_agent seeds trust_score = 0.0). + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + state.db.register_agent(owner, &[]).await.unwrap(); + + std::fs::create_dir_all(&bare).unwrap(); + let out = std::process::Command::new("git") + .args(["init", "--bare", "-q", &bare.to_string_lossy()]) + .output() + .unwrap(); + assert!(out.status.success(), "git init --bare failed"); + + // acquire_write → git_issues::create_issue (writes the ref) → release + // (upload stalls, times out after 200ms). The client MUST see a failure. + let resp = super::create_issue( + State(state.clone()), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + AxPath((owner.to_string(), name.to_string())), + axum::Json(CreateIssueRequest { + title: "t".into(), + body: None, + signed_payload: None, + }), + ) + .await; + let status = match resp { + Ok((code, _)) => code, + Err(e) => e.into_response().status(), + }; + assert!( + status.is_server_error(), + "a timed-out durable upload on issue-create must fail (5xx) so the \ + client retries (got {status}), which the client trusts as a landed \ + issue that a later acquire_write would silently revert" + ); + + // The success tail (trust bump) must NOT have run: the issue was never + // durably accepted. register_agent seeded 0.0; a bump would make it 0.05. + let score = state.db.get_trust_score(owner).await.unwrap(); + assert_eq!( + score, 0.0, + "the trust bump must NOT run when the durable upload failed; the \ + issue was not durably accepted (score bumped to {score})" + ); + } +} diff --git a/crates/gitlawb-node/src/api/pulls.rs b/crates/gitlawb-node/src/api/pulls.rs index 26be6109..cfe93d20 100644 --- a/crates/gitlawb-node/src/api/pulls.rs +++ b/crates/gitlawb-node/src/api/pulls.rs @@ -224,11 +224,26 @@ pub async fn merge_pr( &pr.title, ); - // Always release the advisory lock — even on error; upload to Tigris only on success. - guard.release(merge_result.is_ok()).await; + // Always release the advisory lock, even on error; upload to Tigris only on + // success. A false return means the durable upload failed/timed out: the merge + // commit applied to local disk but never reached object storage, so the next + // acquire_write reverts it from the stale archive. Report 5xx so the client + // retries rather than marking the PR merged in the DB while the merged tree + // was silently lost. Fail BEFORE merge_pr / touch_repo / webhooks. (P1 + // data-loss guard, same as receive-pack.) + let upload_ok = guard.release(merge_result.is_ok()).await; let merge_sha = merge_result.map_err(|e| AppError::Git(e.to_string()))?; + if !upload_ok { + tracing::error!(repo = %record.name, + "durable upload failed after merge; reporting failure so the client retries"); + return Err(AppError::Git(format!( + "durable storage upload failed for {}", + record.name + ))); + } + state.db.merge_pr(&pr.id, &merger_did).await?; let _ = state.db.touch_repo(&record.id).await; diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 54b40c39..0f9516ab 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -222,9 +222,14 @@ pub async fn create_repo( .repo_store .lock_repo_blocking(&owner_did, &req.name) .await - .map_err(|e| AppError::Git(e.to_string()))? + // Both arms are a transient resource condition, not a create failure: the + // lock pool is pinned (Err from acquiring the lock connection) or the key + // stayed held through the bounded retry (None: a live writer or a purge). + // Return a retryable 503 so the client backs off and retries, as the + // call-site comment promises, rather than a misleading 500 git_error. + .map_err(|e| AppError::Unavailable(e.to_string()))? .ok_or_else(|| { - AppError::Git(format!( + AppError::Unavailable(format!( "could not acquire repo lock for {owner_did}/{} — held by a live writer or purge", req.name )) diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index b6e1ab5a..5c93c2f5 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -44,6 +44,9 @@ pub enum AppError { #[error("git service timed out: {0}")] Timeout(String), + #[error("service unavailable: {0}")] + Unavailable(String), + #[error("database error: {0}")] Db(#[from] sqlx::Error), @@ -136,6 +139,14 @@ impl IntoResponse for AppError { // 504, distinct from the 500 git_error and from the read-gate's 404 / // the auth 401, so the client can tell a deadline from a failure. AppError::Timeout(msg) => (StatusCode::GATEWAY_TIMEOUT, "git_timeout", msg.clone()), + // 503, retryable: a transient resource contention (e.g. the repo lock + // pool is pinned by a write burst or a purge holds the key), distinct + // from the 500 git_error so the client knows to back off and retry. + AppError::Unavailable(msg) => ( + StatusCode::SERVICE_UNAVAILABLE, + "service_unavailable", + msg.clone(), + ), AppError::Db(e) if db_unavailable(e) => ( StatusCode::SERVICE_UNAVAILABLE, DB_UNAVAILABLE_CODE, @@ -176,4 +187,20 @@ mod tests { StatusCode::INTERNAL_SERVER_ERROR ); } + + #[test] + fn unavailable_maps_to_503_distinct_from_git_500() { + // The create-repo lock-acquire failure returns Unavailable so a client sees + // a retryable 503, not a terminal 500 git_error. + assert_eq!( + AppError::Unavailable("lock pool pinned".into()) + .into_response() + .status(), + StatusCode::SERVICE_UNAVAILABLE + ); + assert_eq!( + AppError::Git("x".into()).into_response().status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + } } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 9e2d6faa..022bb107 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -626,6 +626,7 @@ impl RepoWriteGuard { /// half-applied or otherwise inconsistent repo would propagate corruption to /// Tigris (and to every node that later downloads it). The lock is always /// released regardless, to avoid stale locks blocking future writes. + #[must_use = "a false return means the durable upload failed; the write must be reported as failed, not 2xx"] pub async fn release(mut self, success: bool) -> bool { // Whether the durable copy is safe. Stays `true` when there is nothing to // upload (no object store, or `success == false` where no upload is @@ -951,8 +952,9 @@ mod tests { ); // After the writer releases (on its pinned connection), the lock is free - // and the single connection is back in the pool. - guard.release(true).await; + // and the single connection is back in the pool. No object store here, so + // release always returns true, so the value is irrelevant to this test. + let _ = guard.release(true).await; let after = store.try_lock_repo(owner, name).await.unwrap(); assert!( after.is_some(), @@ -986,7 +988,8 @@ mod tests { connection (Ok(None)); Ok(Some) would mean the lock is not held" ); - guard.release(true).await; + // No object store, so release always returns true, so value irrelevant here. + let _ = guard.release(true).await; let after = store.try_lock_repo(owner, name).await.unwrap(); assert!(after.is_some(), "lock is free after release"); after.unwrap().release().await; @@ -1229,7 +1232,8 @@ mod tests { for _ in 0..3 { let guard = store.acquire_write(owner, name).await.unwrap(); - guard.release(true).await; + // No object store, so release always returns true, so value irrelevant. + let _ = guard.release(true).await; } } @@ -1499,7 +1503,8 @@ mod tests { ); for g in guards { - g.release(true).await; + // No object store, so release always returns true, so value irrelevant. + let _ = g.release(true).await; } } @@ -1532,7 +1537,8 @@ mod tests { ); for g in guards { - g.release(true).await; + // No object store, so release always returns true, so value irrelevant. + let _ = g.release(true).await; } } @@ -1561,7 +1567,8 @@ mod tests { } assert_eq!(guards.len(), N as usize); for g in guards { - g.release(true).await; + // No object store, so release always returns true, so value irrelevant. + let _ = g.release(true).await; } } @@ -1598,9 +1605,11 @@ mod tests { "the lock must be held while the guard is alive" ); - // release(success) -> the upload stalls, but the bound caps the hold. + // release(success) -> the upload stalls, but the bound caps the hold. The + // return here is false (the upload timed out), but this test asserts only + // the timing bound, so the value is intentionally ignored. let start = std::time::Instant::now(); - guard.release(true).await; + let _ = guard.release(true).await; assert!( start.elapsed() < std::time::Duration::from_secs(5), "release() must return within the upload bound, not hang on the stalled PUT" From 31b175495090904776940556304cf27087ff4736 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 22:45:35 -0500 Subject: [PATCH 41/61] refactor(node): route the per-DID 429 through the shared too_many_requests helper --- crates/gitlawb-node/src/rate_limit.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index 316e3880..88967b26 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -209,12 +209,7 @@ pub async fn rate_limit_by_did(request: Request, next: Next) -> Response { if let (Some(limiter), Some(did)) = (limiter, did) { if let Some(retry_after) = limiter.check_retry(&did).await { - return ( - StatusCode::TOO_MANY_REQUESTS, - [("retry-after", retry_after.as_secs().to_string())], - "rate limit exceeded — try again later", - ) - .into_response(); + return too_many_requests(retry_after); } } @@ -317,9 +312,9 @@ impl axum::extract::FromRequestParts for PeerAddr { } } -/// The shared 429 response for the per-IP flood brakes. Route-agnostic: this -/// middleware now serves the push path AND the peer-sync routes, so the message -/// stays generic (the offending path is recorded in the warn log below). +/// The shared 429 response for the rate-limit middlewares (per-DID and per-IP +/// alike). Route-agnostic: the message stays generic (the offending path, when +/// logged, is recorded at the call site). pub fn too_many_requests(retry_after: Duration) -> Response { ( StatusCode::TOO_MANY_REQUESTS, From 4e785c0cc67439d83bec0de60eb7006c4f1b1de0 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 22:50:18 -0500 Subject: [PATCH 42/61] fix(node): init's background upload waits out the creator's advisory lock (#196) create_repo holds the per-repo lock across init, so the wait=false spawn try-locked against its creator and died permanently, leaving a new repo with no object-store archive until its first later access. RED observed: init_upload_waits_out_creators_lock_then_uploads failed (PUT count 0) against the old code; the bounded-skip negative is retained as init_upload_gives_up_after_bounded_lock_wait. --- crates/gitlawb-node/src/git/repo_store.rs | 103 +++++++++++++++++++--- 1 file changed, 90 insertions(+), 13 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 022bb107..561d2dd3 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -332,14 +332,16 @@ impl RepoStore { store::init_bare(&local_path).context("initializing bare repo")?; // Upload to the object store in the background, UNDER the per-repo lock - // so the PUT can't race a purge. Skips on contention; a skipped init - // upload self-heals — the first write's release re-uploads the state. + // so the PUT can't race a purge. Waits (bounded) for the lock: + // create_repo calls init while holding this repo's lock, so the upload + // waits out the creator's own insert-and-release rather than dying on it + // (a skipped init upload has no retry until the first write). if self.object_store.is_some() { let store = self.clone(); let did = owner_did.to_string(); let name = repo_name.to_string(); tokio::spawn(async move { - store.upload_locked(&did, &name, false).await; + store.upload_locked(&did, &name, true).await; }); } @@ -372,10 +374,12 @@ impl RepoStore { /// the dir under its lock, so a post-purge uploader finds it gone and skips. /// Returns true iff a PUT was performed. A no-op when no store is configured. /// - /// `wait`: fork's foreground upload waits (bounded) for the lock; the - /// background migration/init uploads pass `false` and skip on contention — - /// they self-heal (lazy migration retries on the next `acquire`, init's - /// state is re-uploaded by the first write's release). + /// `wait`: fork's foreground upload and init's background upload wait + /// (bounded) for the lock; a skipped upload on those paths has no later + /// retry (init's next re-upload is the first write's release, which may + /// never come). The background lazy-migration upload passes `false` and + /// skips on contention; that skip genuinely self-heals, since the next + /// `acquire` retries the migration. async fn upload_locked(&self, owner_did: &str, repo_name: &str, wait: bool) -> bool { let Some(ref store) = self.object_store else { return false; @@ -1291,10 +1295,77 @@ mod tests { .join(format!("{name}.git")) } - // R7/R8/AE6: init's background upload must SKIP while a live writer holds the - // repo lock. Pre-fix (unlocked upload) it PUTs regardless -> RED. + // R1/R7: init's background upload must WAIT (bounded) for the creator's own + // lock rather than dying on it. create_repo holds the per-repo lock across + // existence-check -> init -> row insert, so the spawned upload always finds + // it held at first. Create-shaped flow: hold the lock, init under it, + // release shortly after; no PUT while held, exactly one PUT after release. + // Pre-fix (wait=false) the task try-locks once, skips, and the PUT never + // arrives -> RED on the post-release assert. The "still empty at 500ms" + // assert alone would pass vacuously during the new wait window; the + // post-release upload assert is what makes this test load-bearing. #[sqlx::test] - async fn init_upload_skips_while_repo_locked( + async fn init_upload_waits_out_creators_lock_then_uploads( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + let pool = pool_opts + .max_connections(5) + .connect_with(connect_opts) + .await + .unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + let ts = GatedStore::new(false); + let uploads = ts.uploads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkInitWaitAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "initwait"; + let slug = owner.replace([':', '/'], "_"); + + let held = store.try_lock_repo(owner, name).await.unwrap().unwrap(); + store.init(owner, name).await.unwrap(); + // While the creator still holds the lock the upload must not PUT. + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + uploads.lock().unwrap().is_empty(), + "init's upload must not PUT while the creator still holds the lock" + ); + held.release().await; + + // The waiting task retries every 200ms; the PUT must land soon after. + let mut landed = false; + for _ in 0..50 { + if uploads.lock().unwrap().len() == 1 { + landed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + assert!( + landed, + "init's upload must PUT after the creator releases the lock" + ); + let ups = uploads.lock().unwrap(); + assert_eq!( + ups.len(), + 1, + "init's upload must PUT exactly once after the lock frees" + ); + assert_eq!(ups[0], (slug, name.to_string())); + } + + // R1/R7 negative: the bounded wait must not become an unbounded one. A lock + // held past the full 30 x 200ms window yields NO PUT: the task logs a skip + // and exits. The post-release grace assert is what proves boundedness; an + // unbounded waiter would still be parked at release time and would PUT once + // the lock frees. (Reworked from init_upload_skips_while_repo_locked, whose + // skip-immediately premise inverts under wait=true.) + #[sqlx::test] + async fn init_upload_gives_up_after_bounded_lock_wait( pool_opts: sqlx::postgres::PgPoolOptions, connect_opts: sqlx::postgres::PgConnectOptions, ) { @@ -1316,13 +1387,19 @@ mod tests { let held = store.try_lock_repo(owner, name).await.unwrap().unwrap(); store.init(owner, name).await.unwrap(); - // Let the spawned upload attempt-and-skip. - tokio::time::sleep(std::time::Duration::from_millis(500)).await; + // Hold past the entire bounded window (29 sleeps x 200ms, ~5.8s). + tokio::time::sleep(std::time::Duration::from_millis(8_000)).await; assert!( uploads.lock().unwrap().is_empty(), - "init's background upload must skip while the repo is locked by a live writer" + "init's upload must give up, not PUT, when the lock outlives the bounded wait" ); held.release().await; + // A still-parked (unbounded) waiter would PUT within ~200ms of release. + tokio::time::sleep(std::time::Duration::from_millis(1_500)).await; + assert!( + uploads.lock().unwrap().is_empty(), + "no late PUT after release; the upload task must have exited at the bound" + ); } // R8/AE6: fork's foreground upload (release_after_write) WAITS for the lock From 2d52589dae1d1b32f7b41f75f9e69b60907411cf Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 22:50:18 -0500 Subject: [PATCH 43/61] fix(node): roll back the local issue ref before unlock when the durable upload fails (#196) The failed-upload 500 previously left refs/gitlawb/issues/{id} on disk, so a client retry minted a second issue once the store recovered. The delete now runs via RepoWriteGuard::release_with_failure_cleanup while the advisory lock is still held, closing the unlock-to-delete window a concurrent write could have archived the orphan ref through. RED observed on both new tests (orphan ref survived; retry yielded 2 issues); the retry test pins the store double's exists()=false contract so it cannot pass via a stale-archive revert. --- crates/gitlawb-node/src/api/issues.rs | 197 +++++++++++++++++++++- crates/gitlawb-node/src/git/issues.rs | 19 +++ crates/gitlawb-node/src/git/repo_store.rs | 26 ++- 3 files changed, 240 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 27b164c0..a0850744 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -76,7 +76,19 @@ pub async fn create_issue( // next acquire_write would revert it from the stale archive. Report 5xx so // the client retries rather than trusting an issue that never landed durably, // and skip the trust bump below. (Same P1 data-loss guard as receive-pack.) - let upload_ok = guard.release(create_result.is_ok()).await; + // On that failure, roll back the local ref BEFORE the lock releases — an + // orphan ref would make the retry a duplicate, and in an unlock-to-delete + // window a concurrent same-repo write could upload an archive still + // carrying it. Best-effort: a failed delete logs and the 5xx still returns. + let upload_ok = guard + .release_with_failure_cleanup(create_result.is_ok(), |path| { + if let Err(e) = git_issues::delete_issue_ref(path, &issue_id) { + tracing::warn!(issue = %issue_id, err = %e, + "failed to roll back local issue ref after failed durable upload; \ + a retry may duplicate this issue"); + } + }) + .await; create_result.map_err(|e| AppError::Git(e.to_string()))?; @@ -339,6 +351,189 @@ mod tests { } } + // Object store whose upload succeeds immediately. exists() is false so + // acquire_write never performs a reverting refresh over the local dir — + // the retry test below pins that contract explicitly. + struct OkStore; + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for OkStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(false) + } + async fn upload(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + // Shared setup for the failed-upload rollback tests: a state whose repo + // store stalls uploads (200ms release timeout), a public repo row for + // `owner/name`, and an initialized bare repo at the returned path. + async fn stall_state_with_repo( + pool: sqlx::PgPool, + repos_dir: &tempfile::TempDir, + owner: &str, + name: &str, + ) -> (crate::state::AppState, std::path::PathBuf) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(StallStore)), + pool.clone(), + ) + .with_release_upload_timeout(Duration::from_millis(200)); + + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + + std::fs::create_dir_all(&bare).unwrap(); + let out = std::process::Command::new("git") + .args(["init", "--bare", "-q", &bare.to_string_lossy()]) + .output() + .unwrap(); + assert!(out.status.success(), "git init --bare failed"); + + (state, bare) + } + + async fn create_issue_status( + state: &crate::state::AppState, + owner: &str, + name: &str, + ) -> axum::http::StatusCode { + use axum::response::IntoResponse; + let resp = super::create_issue( + State(state.clone()), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + AxPath((owner.to_string(), name.to_string())), + axum::Json(CreateIssueRequest { + title: "t".into(), + body: None, + signed_payload: None, + }), + ) + .await; + match resp { + Ok((code, _)) => code, + Err(e) => e.into_response().status(), + } + } + + fn issue_refs(bare: &StdPath) -> Vec { + let out = std::process::Command::new("git") + .args([ + "for-each-ref", + "--format=%(refname)", + "refs/gitlawb/issues/", + ]) + .current_dir(bare) + .output() + .unwrap(); + assert!(out.status.success(), "git for-each-ref failed"); + String::from_utf8_lossy(&out.stdout) + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| l.trim().to_string()) + .collect() + } + + // Rollback half of the #196 round-four issue-create fix. When the durable + // upload fails/times out, the handler 500s (covered below) — but the issue + // ref it wrote to LOCAL disk must also be rolled back, while the advisory + // lock is still held. An orphan ref makes the 500 dishonest: a retry + // re-creates the issue under a new uuid while the orphan is still listed, + // and a concurrent same-repo write can upload an archive carrying it. + // RED pre-fix: the orphan ref survives the failed create. + #[sqlx::test] + async fn issue_create_failed_upload_rolls_back_local_ref(pool: sqlx::PgPool) { + let repos_dir = tempfile::TempDir::new().unwrap(); + let owner = "z6issuerollbackowner"; + let name = "issuerollback"; + let (state, bare) = stall_state_with_repo(pool, &repos_dir, owner, name).await; + + let status = create_issue_status(&state, owner, name).await; + assert!( + status.is_server_error(), + "stalled upload must 500, got {status}" + ); + + let refs = issue_refs(&bare); + assert!( + refs.is_empty(), + "a failed durable upload must roll back the local issue ref while \ + the advisory lock is held; orphan ref(s) survived: {refs:?}" + ); + } + + // Retry half: after the failed-upload 500, a retry with a WORKING store + // must yield exactly one issue. Pre-fix the orphan ref from the failed + // attempt makes the retry a duplicate (two issues in list_issues). + #[sqlx::test] + async fn issue_create_failed_upload_retry_does_not_duplicate(pool: sqlx::PgPool) { + use crate::git::tigris::ObjectStore as _; + + let repos_dir = tempfile::TempDir::new().unwrap(); + let owner = "z6issueretryowner"; + let name = "issueretry"; + let (state, bare) = stall_state_with_repo(pool.clone(), &repos_dir, owner, name).await; + + let status = create_issue_status(&state, owner, name).await; + assert!( + status.is_server_error(), + "stalled upload must 500, got {status}" + ); + + // PIN the double's contract: exists() stays false after the failed + // upload, so the retry's acquire_write performs no reverting refresh + // from a stale archive. Without this pin, a download could revert the + // orphan ref and the assertion below would pass without the rollback + // fix — it must rest on the rollback, not a stale-archive revert. + let owner_slug = owner.replace([':', '/'], "_"); + assert!( + !OkStore.exists(&owner_slug, name).await.unwrap(), + "test double contract broken: exists() must stay false" + ); + + // Retry with a store whose upload succeeds. + let mut retry_state = state.clone(); + retry_state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(OkStore)), + pool.clone(), + ) + .with_release_upload_timeout(Duration::from_millis(200)); + + let status = create_issue_status(&retry_state, owner, name).await; + assert_eq!( + status, + StatusCode::CREATED, + "retry with working store must 201" + ); + + let issues = crate::git::issues::list_issues(&bare).unwrap(); + assert_eq!( + issues.len(), + 1, + "retry after a failed-upload 500 must yield exactly ONE issue; the \ + failed attempt's orphan ref duplicated it (got {} issues)", + issues.len() + ); + } + // P1-class data-loss regression, issue-create path. create_issue writes the // issue ref to local disk, then release(true) uploads the repo to durable // storage. When that upload times out the local write is NOT persisted, so diff --git a/crates/gitlawb-node/src/git/issues.rs b/crates/gitlawb-node/src/git/issues.rs index 73098302..1f507af6 100644 --- a/crates/gitlawb-node/src/git/issues.rs +++ b/crates/gitlawb-node/src/git/issues.rs @@ -61,6 +61,25 @@ pub fn create_issue(repo_path: &Path, issue_id: &str, json: &str) -> Result<()> Ok(()) } +/// Delete an issue ref. Used to roll back a locally written issue whose +/// durable upload failed, so a retry does not duplicate it. The dangling +/// blob object is harmless. +pub fn delete_issue_ref(repo_path: &Path, issue_id: &str) -> Result<()> { + let ref_name = format!("refs/gitlawb/issues/{issue_id}"); + let output = Command::new("git") + .args(["update-ref", "-d", &ref_name]) + .current_dir(repo_path) + .output() + .context("failed to run git update-ref -d")?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + anyhow::bail!("git update-ref -d failed: {stderr}"); + } + + Ok(()) +} + /// List all issue refs and return their JSON content. pub fn list_issues(repo_path: &Path) -> Result> { // List all refs under refs/gitlawb/issues/ diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 561d2dd3..2373aa15 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -631,7 +631,23 @@ impl RepoWriteGuard { /// Tigris (and to every node that later downloads it). The lock is always /// released regardless, to avoid stale locks blocking future writes. #[must_use = "a false return means the durable upload failed; the write must be reported as failed, not 2xx"] - pub async fn release(mut self, success: bool) -> bool { + pub async fn release(self, success: bool) -> bool { + self.release_with_failure_cleanup(success, |_| {}).await + } + + /// Like [`release`](Self::release), but when an ATTEMPTED upload fails or + /// times out, runs `cleanup` on the local repo path BEFORE the advisory + /// lock is freed. Lets a caller roll back a local write that never reached + /// durable storage without an unlock-to-cleanup window in which a + /// concurrent same-repo write could upload an archive still carrying it. + /// `cleanup` does NOT run when `success == false` (the write itself + /// failed, no upload attempted) or when the upload succeeded. + #[must_use = "a false return means the durable upload failed; the write must be reported as failed, not 2xx"] + pub async fn release_with_failure_cleanup( + mut self, + success: bool, + cleanup: impl FnOnce(&Path), + ) -> bool { // Whether the durable copy is safe. Stays `true` when there is nothing to // upload (no object store, or `success == false` where no upload is // attempted) — those are not upload failures. Only an upload that was @@ -670,6 +686,14 @@ impl RepoWriteGuard { warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo"); } + // Failed-upload cleanup runs HERE, while the advisory lock is still + // held: after the unlock below, a concurrent same-repo write could + // upload an archive still carrying the state the caller is rolling + // back, which a later download would resurrect. + if !upload_ok { + cleanup(&self.local_path); + } + // Unlock on the SAME connection that took the lock, then TAKE the // connection so it returns to the pool on drop and the guard's `Drop` // sees `None` (no close-on-drop). If this future is cancelled before the From 2a0d6f79e340481f73a2d8344120cdde248703cc Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 23:13:06 -0500 Subject: [PATCH 44/61] fix(node): fork publishes only after a durable upload, serialized on the target-namespace lock (#196) fork_repo now takes the per-repo advisory lock before its conflict check and holds it through clone, upload, and row insert, 503ing on contention exactly as create_repo does. The upload runs under the held guard via upload_under_guard, PUT bounded by release_upload_timeout, returning false only for an attempted upload that failed or timed out (no-store nodes fork unchanged); on failure the mirror is removed and the fork 500s before any row is built. release_after_write is retired; its tests migrated onto the new method. RED observed on the serialization, durability, and contention tests; defect injection (row insert reordered above the upload) turned the durability test RED and was reverted. --- crates/gitlawb-node/src/api/repos.rs | 361 +++++++++++++++++++++- crates/gitlawb-node/src/git/repo_store.rs | 154 ++++++--- 2 files changed, 476 insertions(+), 39 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 0f9516ab..6def2a84 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1658,6 +1658,24 @@ pub async fn fork_repo( )); } + // Serialize against a concurrent same-key purge or creation on the FORK + // TARGET namespace, mirroring create_repo's lock above (same rationale, + // same 503 mapping). Held across the conflict check, clone, durable upload, + // and row insert, so none of it can interleave with a purge's delete-row + + // remove-dir window, and so the row is only published after the archive + // durably landed. Released explicitly on both exits below; the guard's + // Drop frees the lock on every intermediate error path. + let repo_lock = state + .repo_store + .lock_repo_blocking(&forker_did, &fork_name) + .await + .map_err(|e| AppError::Unavailable(e.to_string()))? + .ok_or_else(|| { + AppError::Unavailable(format!( + "could not acquire repo lock for {forker_did}/{fork_name}: held by a live writer or purge" + )) + })?; + // Check no name conflict under the forker's ownership let forker_short = crate::db::normalize_owner_key(&forker_did); if state.db.get_repo(forker_short, &fork_name).await?.is_some() { @@ -1696,11 +1714,26 @@ pub async fn fork_repo( ))); } - // Upload fork to Tigris - state + // Upload the fork to durable storage under the held target lock, bounded by + // the release-upload timeout. Publish-after-durability: an attempted upload + // that failed or timed out fails the fork (the sibling write paths' 5xx) + // BEFORE any RepoRecord exists, and removes the cloned mirror so no later + // acquire serves a fork whose archive never landed. Store-less nodes take + // the success arm (nothing to upload is not a failure). + if !state .repo_store - .release_after_write(&forker_did, &fork_name) - .await; + .upload_under_guard(&forker_did, &fork_name, &repo_lock) + .await + { + if let Err(e) = std::fs::remove_dir_all(&disk_path) { + tracing::warn!(fork = %fork_name, err = %e, + "failed to remove fork mirror after failed durable upload"); + } + repo_lock.release().await; + return Err(AppError::Git(format!( + "durable storage upload failed for {fork_name}" + ))); + } let now = Utc::now(); let record = crate::db::RepoRecord { @@ -1719,6 +1752,10 @@ pub async fn fork_repo( state.db.create_repo(&record).await?; + // Row, archive, and on-disk dir now all exist consistently; the race window + // is closed, so the target lock can be released before the best-effort tail. + repo_lock.release().await; + // Persist the proof so the fork carries it when it propagates to peers. if let Some(p) = verified_proof { if let Err(e) = p.record_for_repo(&state.db, &record.id).await { @@ -3908,6 +3945,322 @@ mod tests { ); } + // ── U3/R2/R4: fork tail, guarded span, publish only after durability ───── + + const FORK_SRC_OWNER: &str = "z6forksrcowner"; + const FORK_SRC_NAME: &str = "forksrc"; + + /// Build a state whose repo_store AND config.repos_dir point at `repos_dir` + /// (fork_repo computes its clone target from config.repos_dir, so the two + /// must agree), seeded with a bare public source repo any caller may fork. + async fn fork_test_state( + repos_dir: &std::path::Path, + repo_store: crate::git::repo_store::RepoStore, + pool: sqlx::PgPool, + ) -> AppState { + let mut state = crate::test_support::test_state(pool).await; + state.repo_store = repo_store; + let mut cfg = (*state.config).clone(); + cfg.repos_dir = repos_dir.to_path_buf(); + state.config = std::sync::Arc::new(cfg); + + let bare = repos_dir + .join(FORK_SRC_OWNER) // slug == owner: no ':' or '/' to replace + .join(format!("{FORK_SRC_NAME}.git")); + std::fs::create_dir_all(bare.parent().unwrap()).unwrap(); + let out = std::process::Command::new("git") + .args(["init", "--bare", "-q", &bare.to_string_lossy()]) + .output() + .unwrap(); + assert!(out.status.success(), "git init --bare failed"); + state + .db + .upsert_mirror_repo( + FORK_SRC_OWNER, + FORK_SRC_NAME, + &bare.to_string_lossy(), + None, + false, + ) + .await + .unwrap(); + state + } + + async fn call_fork( + state: &AppState, + forker: &str, + fork_name: &str, + ) -> Result { + use axum::extract::{Path as AxPath, State}; + use axum::response::IntoResponse; + use axum::Extension; + super::fork_repo( + State(state.clone()), + Extension(crate::auth::AuthenticatedDid(forker.to_string())), + AxPath((FORK_SRC_OWNER.to_string(), FORK_SRC_NAME.to_string())), + axum::http::HeaderMap::new(), + Json(ForkRepoRequest { + name: Some(fork_name.to_string()), + }), + ) + .await + .map(|r| r.into_response()) + } + + // R2: fork must take the SAME per-owner/name advisory lock on its TARGET + // namespace that purge and create hold, so it cannot interleave with a + // purge's delete-row + remove-dir (or another creator) on that key. + // Deterministic form: hold the target lock, prove the fork BLOCKS (no clone + // dir, no repos row) rather than proceeding, then completes cleanly once the + // lock frees. RED on base: fork takes no lock and completes in the window. + #[sqlx::test] + async fn fork_serializes_against_target_namespace_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use std::time::Duration; + + // Multi-connection pool: the held guard pins one connection, so the + // fork's own lock attempt must get a DIFFERENT one and observe the key + // held (Ok(None) -> retry), not merely block on pool exhaustion. + let pool = pool_opts + .max_connections(5) + .connect_with(connect_opts) + .await + .unwrap(); + let repos_dir = tempfile::TempDir::new().unwrap(); + let repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + let state = fork_test_state(repos_dir.path(), repo_store, pool).await; + + let forker = "did:key:z6MkForkSerializeAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let fork_name = "serialfork"; + + // A purge (or another creator) holds the fork target's advisory lock. + let held = state + .repo_store + .try_lock_repo(forker, fork_name) + .await + .unwrap() + .expect("target lock free before the fork"); + + let state2 = state.clone(); + let forker2 = forker.to_string(); + let fork_name2 = fork_name.to_string(); + let handle = tokio::spawn(async move { call_fork(&state2, &forker2, &fork_name2).await }); + + // While the target lock is held the fork must make no progress past it. + tokio::time::sleep(Duration::from_millis(500)).await; + let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name); + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_none(), + "fork must not insert a repos row while the target-namespace lock is \ + held (RED on base: fork takes no lock and the row lands in the window)" + ); + assert!( + !clone_dir.exists(), + "fork must not clone the mirror while the target-namespace lock is held" + ); + assert!( + !handle.is_finished(), + "fork must be blocked on the target lock, not have completed" + ); + + held.release().await; + let resp = tokio::time::timeout(Duration::from_secs(8), handle) + .await + .expect("fork should finish once the lock frees") + .expect("fork task join") + .expect("fork_repo returns Ok once it wins the lock"); + assert_eq!(resp.status(), StatusCode::CREATED); + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_some(), + "repos row present after the fork wins the lock" + ); + assert!(clone_dir.exists(), "cloned mirror present after the fork"); + } + + // R4: publish-after-durability. When the fork's durable upload stalls, the + // fork must fail (5xx) within the release-upload bound, insert NO repos row, + // and remove the half-created mirror from disk. RED on base: the foreground + // upload's unbounded PUT hangs the handler on a stall (and an erroring + // upload would return 201 + insert the row despite the failed upload). + #[sqlx::test] + async fn fork_durable_upload_failure_fails_before_row_insert(pool: sqlx::PgPool) { + use axum::response::IntoResponse; + use std::path::Path as StdPath; + use std::time::Duration; + + // Object store whose upload() parks forever: the fork-tail upload bound + // is the only thing that unblocks it. `exists()` is false so acquire + // never downloads over the local source repo. + struct StallStore; + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for StallStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(false) + } + async fn upload(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + std::future::pending::<()>().await; + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + let repos_dir = tempfile::TempDir::new().unwrap(); + let repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(StallStore)), + pool.clone(), + ) + .with_release_upload_timeout(Duration::from_millis(200)); + let state = fork_test_state(repos_dir.path(), repo_store, pool).await; + + let forker = "did:key:z6MkForkUploadFailAAAAAAAAAAAAAAAAAAAAAAAAA"; + let fork_name = "failfork"; + + let resp = + tokio::time::timeout(Duration::from_secs(5), call_fork(&state, forker, fork_name)) + .await + .expect( + "fork must fail within the upload bound when the durable upload \ + stalls (RED on base: the unbounded foreground PUT hangs the handler)", + ); + let status = match resp { + Ok(r) => r.status(), + Err(e) => e.into_response().status(), + }; + assert!( + status.is_server_error(), + "a failed durable upload must fail the fork (5xx); got {status}, \ + which the client trusts as a fork whose archive never landed" + ); + + // Publish-after-durability: no repos row for a fork with no archive. + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_none(), + "no repos row may exist for a fork whose durable upload failed" + ); + let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name); + assert!( + !clone_dir.exists(), + "the cloned mirror must be removed when the durable upload fails" + ); + } + + // KTD-2 trap case (must-not-break negative): a store-less node has nothing + // to upload, so fork succeeds exactly as today: "no object store" is + // nothing-to-do, never a failed upload. GREEN both before and after the fix. + #[sqlx::test] + async fn fork_without_object_store_succeeds(pool: sqlx::PgPool) { + let repos_dir = tempfile::TempDir::new().unwrap(); + let repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + let state = fork_test_state(repos_dir.path(), repo_store, pool).await; + + let forker = "did:key:z6MkForkNoStoreAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let fork_name = "nostorefork"; + + let resp = call_fork(&state, forker, fork_name) + .await + .expect("a store-less fork must succeed"); + assert_eq!(resp.status(), StatusCode::CREATED); + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_some(), + "repos row present after a store-less fork" + ); + let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name); + assert!(clone_dir.exists(), "cloned mirror present after the fork"); + } + + // R2: a target lock held past the fork's bounded wait must surface as the + // same retryable 503 create_repo returns, with no repos row and no clone + // residue. RED on base: fork ignores the lock and completes with a 201. + #[sqlx::test] + async fn fork_lock_contention_returns_503( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use axum::response::IntoResponse; + + let pool = pool_opts + .max_connections(5) + .connect_with(connect_opts) + .await + .unwrap(); + let repos_dir = tempfile::TempDir::new().unwrap(); + let repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + let state = fork_test_state(repos_dir.path(), repo_store, pool).await; + + let forker = "did:key:z6MkForkContendedAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let fork_name = "contendedfork"; + + // Held for the whole test: the fork's bounded wait must give up. + let held = state + .repo_store + .try_lock_repo(forker, fork_name) + .await + .unwrap() + .expect("target lock free before the fork"); + + let resp = call_fork(&state, forker, fork_name).await; + let status = match resp { + Ok(r) => r.status(), + Err(e) => e.into_response().status(), + }; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a target lock held past the bounded wait must 503 like create_repo \ + (RED on base: fork ignores the lock and returns 201)" + ); + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_none(), + "no repos row after a 503'd fork" + ); + let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name); + assert!(!clone_dir.exists(), "no clone residue after a 503'd fork"); + held.release().await; + } + // create_repo must NOT take its serialization lock from the APP pool. Doing so // pins an app-pool connection in the lock guard while the create's own work // (get_repo / proof.consume / db.create_repo, all on state.db = the app pool) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 2373aa15..53b65f58 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -3,7 +3,8 @@ //! Every handler that needs access to a git repo on disk goes through `RepoStore`: //! //! - `acquire()` — ensures the repo is on local disk (downloads from Tigris on cache miss). -//! - `release_after_write()` — uploads the updated repo to Tigris after a write operation. +//! - `upload_under_guard()`: uploads the updated repo to Tigris after a write, under a +//! per-repo advisory lock the caller already holds (the fork tail's durability step). //! - `init()` — creates a new bare repo locally and uploads to Tigris. //! //! When Tigris is disabled (bucket empty), this is a simple passthrough to local disk. @@ -348,15 +349,66 @@ impl RepoStore { Ok(local_path) } - /// Upload a repo to Tigris after a write operation (fork, etc.). Call this - /// after any operation that modifies the git repo on disk. Uploads UNDER the - /// per-repo advisory lock so the PUT cannot race a concurrent purge (which - /// deletes the repo under the same lock) and resurrect a deleted archive. - /// Waits (bounded) for the lock — a skipped fork upload would have no later - /// retry; in practice the fork target's lock is uncontended (its DB row is - /// created after this upload). - pub async fn release_after_write(&self, owner_did: &str, repo_name: &str) { - self.upload_locked(owner_did, repo_name, true).await; + /// Upload a repo to the object store under a per-repo advisory lock the + /// CALLER already holds. The fork tail uses this: it takes the target's + /// lock before its conflict check and holds it through clone, upload, and + /// row insert, so the PUT cannot race a concurrent purge (which deletes the + /// repo under the same lock) and the repos row is only published after the + /// archive durably landed. The PUT is bounded by `release_upload_timeout`, + /// exactly as `RepoWriteGuard::release` bounds its upload: an unbounded PUT + /// under a held namespace lock is the INV-22 hazard. A timeout counts as an + /// attempted-and-failed upload. + /// + /// Return semantics follow [`RepoWriteGuard::release`], NOT `upload_locked`: + /// `false` only when an ATTEMPTED upload failed or timed out. No configured + /// object store means nothing to upload, so store-less nodes see success and + /// their writes proceed unchanged. The local dir gone under the lock (a + /// purge won the key before the caller took it) is likewise a skip, not a + /// failure: nothing exists to upload and a deleted archive must stay gone. + /// A path-validation failure returns `false` (fail closed: never report + /// durable success for a path we refused to touch). + #[must_use = "a false return means the durable upload failed; the write must be reported as failed, not 2xx"] + pub async fn upload_under_guard( + &self, + owner_did: &str, + repo_name: &str, + _guard: &RepoLockGuard, + ) -> bool { + let Some(ref store) = self.object_store else { + // No durable backend configured: nothing to upload, nothing failed. + return true; + }; + let (owner_slug, local_path) = match self.local_path(owner_did, repo_name) { + Ok(p) => p, + Err(e) => { + warn!(repo = %repo_name, err = %e, "rejected unsafe path before object-store upload"); + return false; + } + }; + // Re-check under the lock: a purge removes the on-disk dir under this + // same lock, so a caller that took the key post-purge finds the dir gone + // and must NOT recreate the archive. + if !local_path.exists() { + warn!(repo = %repo_name, "object-store upload skipped: local repo dir gone under lock (purged?)"); + return true; + } + match tokio::time::timeout( + self.release_upload_timeout, + store.upload(&owner_slug, repo_name, &local_path), + ) + .await + { + Ok(Ok(())) => true, + Ok(Err(e)) => { + warn!(repo = %repo_name, err = %e, "failed to upload repo to object store"); + false + } + Err(_) => { + warn!(repo = %repo_name, timeout_secs = self.release_upload_timeout.as_secs(), + "object-store upload timed out under the held repo lock"); + false + } + } } /// Whether an object store is configured. Used by purge-spam to decide @@ -374,12 +426,14 @@ impl RepoStore { /// the dir under its lock, so a post-purge uploader finds it gone and skips. /// Returns true iff a PUT was performed. A no-op when no store is configured. /// - /// `wait`: fork's foreground upload and init's background upload wait - /// (bounded) for the lock; a skipped upload on those paths has no later - /// retry (init's next re-upload is the first write's release, which may - /// never come). The background lazy-migration upload passes `false` and - /// skips on contention; that skip genuinely self-heals, since the next - /// `acquire` retries the migration. + /// `wait`: init's background upload waits (bounded) for the lock, since the + /// creator holds this key across create_repo's insert-and-release and a + /// skipped init upload has no later retry (the next re-upload is the first + /// write's release, which may never come). The background lazy-migration + /// upload passes `false` and skips on contention; that skip genuinely + /// self-heals, since the next `acquire` retries the migration. The fork + /// tail does NOT come through here: it already holds the target's lock and + /// uploads via `upload_under_guard` (a second acquire would self-contend). async fn upload_locked(&self, owner_did: &str, repo_name: &str, wait: bool) -> bool { let Some(ref store) = self.object_store else { return false; @@ -434,13 +488,13 @@ impl RepoStore { uploaded } - /// Bounded-wait lock-only acquire — the fork foreground upload's counterpart - /// to `acquire_write`'s spin, without any object-store I/O. Retries - /// `try_lock_repo` with short backoff. In practice the fork target's lock is - /// uncontended (its DB row is created after the upload), so this returns on - /// the first attempt. Also used by `create_repo` to serialize its - /// existence-check -> init -> insert against a concurrent same-key purge - /// (which holds this same lock across delete-row + remove-dir). + /// Bounded-wait lock-only acquire, without any object-store I/O. Retries + /// `try_lock_repo` with short backoff. Used by `create_repo` and `fork_repo` + /// to serialize their existence-check -> write -> row-insert spans against a + /// concurrent same-key purge (which holds this same lock across delete-row + + /// remove-dir), and by init's background upload to wait out the creator's + /// own insert-and-release. A key held past the bounded wait surfaces as + /// `Ok(None)`, which the create/fork handlers map to a retryable 503. pub async fn lock_repo_blocking( &self, owner_did: &str, @@ -1426,11 +1480,13 @@ mod tests { ); } - // R8/AE6: fork's foreground upload (release_after_write) WAITS for the lock - // rather than skipping, then PUTs once after it frees. Pre-fix it PUTs - // immediately while the lock is held -> RED on the "still empty" assert. + // R8/AE6: the fork tail's lock acquire WAITS for a held target lock rather + // than skipping, then the under-guard upload PUTs once after it frees. This + // runs the exact lock_repo_blocking -> upload_under_guard sequence + // fork_repo runs (migrated from release_after_write when it was retired; + // assertions unchanged). #[sqlx::test] - async fn release_after_write_waits_then_uploads_after_lock_frees( + async fn under_guard_upload_waits_then_uploads_after_lock_frees( pool_opts: sqlx::postgres::PgPoolOptions, connect_opts: sqlx::postgres::PgConnectOptions, ) { @@ -1456,7 +1512,16 @@ mod tests { let store2 = store.clone(); let owner2 = owner.to_string(); let name2 = name.to_string(); - let h = tokio::spawn(async move { store2.release_after_write(&owner2, &name2).await }); + let h = tokio::spawn(async move { + let g = store2 + .lock_repo_blocking(&owner2, &name2) + .await + .unwrap() + .expect("target lock frees within the bounded wait"); + let ok = store2.upload_under_guard(&owner2, &name2, &g).await; + g.release().await; + ok + }); tokio::time::sleep(std::time::Duration::from_millis(500)).await; assert!( uploads.lock().unwrap().is_empty(), @@ -1464,7 +1529,10 @@ mod tests { ); held.release().await; - h.await.unwrap(); + assert!( + h.await.unwrap(), + "the under-guard upload reports success once the lock frees" + ); let ups = uploads.lock().unwrap(); assert_eq!( ups.len(), @@ -1496,7 +1564,9 @@ mod tests { let dir = repo_dir_of(tmp.path(), owner, name); std::fs::create_dir_all(&dir).unwrap(); - store.release_after_write(owner, name).await; + let g = store.try_lock_repo(owner, name).await.unwrap().unwrap(); + assert!(store.upload_under_guard(owner, name, &g).await); + g.release().await; assert!(exists.load(SeqCst), "archive exists after the first upload"); assert_eq!(uploads.lock().unwrap().len(), 1); @@ -1506,8 +1576,11 @@ mod tests { assert_eq!(deletes.lock().unwrap().len(), 1, "archive delete recorded"); assert!(!exists.load(SeqCst), "archive is deleted by the purge"); - // A late upload attempt must find the dir gone and skip, not resurrect. - store.release_after_write(owner, name).await; + // A late upload attempt must find the dir gone under the lock and skip, + // not resurrect. The skip is nothing-to-do (true), never a PUT. + let g = store.try_lock_repo(owner, name).await.unwrap().unwrap(); + assert!(store.upload_under_guard(owner, name, &g).await); + g.release().await; assert!( !exists.load(SeqCst), "a purged archive must stay deleted — the upload found no dir and skipped" @@ -1519,9 +1592,11 @@ mod tests { ); } - // Negative: an uncontended fork upload PUTs exactly once (the lock is free). + // Negative: an uncontended fork upload PUTs exactly once (the lock is free, + // so the fork-tail sequence acquires it first try and uploads under it). + // Migrated from release_after_write when it was retired; assertion unchanged. #[sqlx::test] - async fn uncontended_release_after_write_uploads_once(pool: sqlx::PgPool) { + async fn uncontended_under_guard_upload_uploads_once(pool: sqlx::PgPool) { let tmp = tempfile::TempDir::new().unwrap(); let ts = GatedStore::new(false); let uploads = ts.uploads.clone(); @@ -1534,7 +1609,16 @@ mod tests { let name = "uncontended"; std::fs::create_dir_all(repo_dir_of(tmp.path(), owner, name)).unwrap(); - store.release_after_write(owner, name).await; + let g = store + .lock_repo_blocking(owner, name) + .await + .unwrap() + .expect("uncontended lock acquires first try"); + assert!( + store.upload_under_guard(owner, name, &g).await, + "an uncontended under-guard upload reports success" + ); + g.release().await; assert_eq!( uploads.lock().unwrap().len(), 1, From bf640db654e55dad77b4810814db26bce58e06b3 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 23:27:33 -0500 Subject: [PATCH 45/61] fix(node): read-path downloads publish under the purge lock, without pinning it across the network (#196) acquire's cache-miss and every acquire_fresh refresh now coalesce on a per-repo download mutex, download to a temp sibling with no advisory lock held, then try-lock, re-check the archive exists under the lock, and swap - so an in-flight read can no longer resurrect a repository purge just deleted (RED observed on both the cache-miss and refresh-over-existing repros), a post-purge reader degrades instead of publishing, and concurrent cold reads download once. The hit path stays lock-free (deterministic held-lock fence) and no lock-pool connection is held while a download is parked. --- crates/gitlawb-node/src/git/repo_store.rs | 559 +++++++++++++++++++++- 1 file changed, 537 insertions(+), 22 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 53b65f58..f6c3323e 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -9,7 +9,7 @@ //! //! When Tigris is disabled (bucket empty), this is a simple passthrough to local disk. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -44,6 +44,25 @@ pub struct RepoStore { /// Tracks repos already confirmed to exist in the object store — avoids /// redundant HEAD checks and background uploads for repos we've migrated. migrated: Arc>>, + /// Per-repo async download coordination (KTD-3): concurrent readers of the + /// same repo serialize here so only one runs the network download while + /// the rest await it and serve what the winner published. Entries are + /// created only after a caller has confirmed the archive exists (reached + /// the download branch) and are removed when the holder finishes, so + /// permissionless requests for arbitrary names cannot grow the map. + download_locks: Arc>>>>, +} + +/// Outcome of a coordinated read-path download (KTD-3). +enum DownloadOutcome { + /// The local dir is present: this reader downloaded and published it under + /// the advisory lock, or a concurrent reader did and this one served it. + Published, + /// The download was discarded without publishing: the advisory lock was + /// contended (a live writer or purge holds it), the lock pool errored, or + /// the archive vanished under the lock (purged mid-download). The caller + /// degrades to its missing-path or serve-local-copy outcome. + Skipped, } impl RepoStore { @@ -55,6 +74,7 @@ impl RepoStore { lock_pool, release_upload_timeout: DEFAULT_RELEASE_UPLOAD_TIMEOUT, migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), + download_locks: Arc::new(Mutex::new(HashMap::new())), } } @@ -69,6 +89,7 @@ impl RepoStore { lock_pool, release_upload_timeout: DEFAULT_RELEASE_UPLOAD_TIMEOUT, migrated: Arc::new(Mutex::new(HashSet::new())), + download_locks: Arc::new(Mutex::new(HashMap::new())), } } @@ -133,16 +154,26 @@ impl RepoStore { if let Some(ref tigris) = self.object_store { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "cache miss — downloading from tigris"); - tigris - .download(&owner_slug, repo_name, &local_path) + match self + .download_published(owner_did, repo_name, &owner_slug, &local_path, tigris) .await - .context("downloading repo from tigris")?; - // Mark as migrated since we just downloaded it - self.migrated - .lock() - .await - .insert(format!("{owner_slug}/{repo_name}")); - return Ok(local_path); + .context("downloading repo from tigris")? + { + DownloadOutcome::Published => { + // Mark as migrated since we just downloaded it + self.migrated + .lock() + .await + .insert(format!("{owner_slug}/{repo_name}")); + return Ok(local_path); + } + DownloadOutcome::Skipped => { + // Degraded: the archive vanished under the lock (purged + // mid-download) or a writer/purge holds the advisory + // lock. Publish nothing and fall through to the + // missing-path outcome below. + } + } } } @@ -161,20 +192,33 @@ impl RepoStore { if let Some(ref tigris) = self.object_store { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "acquire_fresh: downloading latest from tigris"); - if let Err(e) = tigris.download(&owner_slug, repo_name, &local_path).await { - // The Tigris archive is present (HEAD ok) but unreadable — a - // corrupt/partial upload, or a transient GET failure. If we have a - // valid local copy, proceed with it rather than blocking the write; - // the post-write upload re-syncs (self-heals) Tigris. Only hard-fail - // when there is no local copy to fall back to. - if local_path.exists() { - warn!(repo = %repo_name, err = %e, - "acquire_fresh: tigris download failed — falling back to local copy"); + match self + .download_published(owner_did, repo_name, &owner_slug, &local_path, tigris) + .await + { + // Published covers both the refreshed copy and a concurrent + // reader's just-published one. Skipped is the degraded + // outcome: the archive vanished under the lock (purged) or + // a writer/purge holds the advisory lock — serve the local + // copy when one still exists (the same fallback as the + // download-error arm), else the missing path. + Ok(DownloadOutcome::Published) | Ok(DownloadOutcome::Skipped) => { return Ok(local_path); } - return Err(e).context("downloading repo from tigris (fresh)"); + Err(e) => { + // The Tigris archive is present (HEAD ok) but unreadable — a + // corrupt/partial upload, or a transient GET failure. If we have a + // valid local copy, proceed with it rather than blocking the write; + // the post-write upload re-syncs (self-heals) Tigris. Only hard-fail + // when there is no local copy to fall back to. + if local_path.exists() { + warn!(repo = %repo_name, err = %e, + "acquire_fresh: tigris download failed — falling back to local copy"); + return Ok(local_path); + } + return Err(e).context("downloading repo from tigris (fresh)"); + } } - return Ok(local_path); } } @@ -182,6 +226,141 @@ impl RepoStore { Ok(local_path) } + /// Coordinated read-path download (KTD-3). Serializes concurrent readers + /// of the same repo on a per-repo async mutex: the first one in becomes + /// the holder and runs [`download_and_publish`](Self::download_and_publish); + /// a contended reader awaits the holder, re-checks the local dir on wake, + /// and serves what the holder published instead of re-downloading. Only + /// callers that already confirmed the archive exists reach this, and the + /// map entry is removed on completion, so the map cannot grow per + /// arbitrary requested name. + async fn download_published( + &self, + owner_did: &str, + repo_name: &str, + owner_slug: &str, + local_path: &Path, + store: &Arc, + ) -> Result { + let map_key = format!("{owner_slug}/{repo_name}"); + let entry = { + let mut map = self.download_locks.lock().await; + Arc::clone( + map.entry(map_key.clone()) + .or_insert_with(|| Arc::new(Mutex::new(()))), + ) + }; + let held = match entry.try_lock() { + Ok(g) => g, + Err(_) => { + let g = entry.lock().await; + if local_path.exists() { + // A concurrent reader published while we waited — serve it. + drop(g); + self.remove_download_entry(&map_key, &entry).await; + return Ok(DownloadOutcome::Published); + } + // The holder degraded without publishing; this reader takes over. + g + } + }; + let outcome = self + .download_and_publish(owner_did, repo_name, owner_slug, local_path, store) + .await; + self.remove_download_entry(&map_key, &entry).await; + drop(held); + outcome + } + + /// Remove a completed download-coordination entry, but only when the map + /// still holds THIS entry — a newer entry inserted after an earlier + /// removal must not be clobbered. + async fn remove_download_entry(&self, key: &str, entry: &Arc>) { + let mut map = self.download_locks.lock().await; + if map.get(key).is_some_and(|cur| Arc::ptr_eq(cur, entry)) { + map.remove(key); + } + } + + /// The holder's half of the coordinated download: fetch and extract the + /// archive into a temp sibling with NO advisory lock held (the network + /// phase must never pin a lock-pool connection — a cold-read burst across + /// distinct repos must not drain the writer-sized pool), then take the + /// per-repo advisory lock only around the publish: re-check the archive + /// still exists under the lock (a purge deletes it under this same lock, + /// so a post-purge downloader discards rather than resurrecting), swap + /// the temp copy into place, release. Advisory contention or a lock-pool + /// error discards the temp copy and degrades — no blocking lock or pool + /// wait exists anywhere on the read path. + async fn download_and_publish( + &self, + owner_did: &str, + repo_name: &str, + owner_slug: &str, + local_path: &Path, + store: &Arc, + ) -> Result { + let parent = local_path.parent().context("repo path has no parent")?; + std::fs::create_dir_all(parent).context("creating repo parent dir")?; + let file_name = local_path + .file_name() + .context("repo path has no file name")? + .to_string_lossy(); + // Unique per-download temp target (same parent as local_path so the + // publish rename stays on one filesystem); mirrors the extract temp + // naming in tigris.rs. + let tmp_path = parent.join(format!( + ".{file_name}.tmp-download.{}", + uuid::Uuid::new_v4() + )); + + if let Err(e) = store.download(owner_slug, repo_name, &tmp_path).await { + let _ = std::fs::remove_dir_all(&tmp_path); + return Err(e); + } + + let guard = match self.try_lock_repo(owner_did, repo_name).await { + Ok(Some(g)) => g, + Ok(None) => { + debug!(repo = %repo_name, + "read download discarded — repo lock held by a live writer or purge"); + let _ = std::fs::remove_dir_all(&tmp_path); + return Ok(DownloadOutcome::Skipped); + } + Err(e) => { + warn!(repo = %repo_name, err = %e, + "read download discarded — could not acquire repo lock"); + let _ = std::fs::remove_dir_all(&tmp_path); + return Ok(DownloadOutcome::Skipped); + } + }; + // Re-check under the lock: the archive gone here means a purge won the + // key mid-download — discard, never publish (the read-side twin of the + // upload paths' under-lock dir re-check). An exists() error degrades + // the same way: fail closed rather than publish unconfirmed state. + if !store.exists(owner_slug, repo_name).await.unwrap_or(false) { + warn!(repo = %repo_name, + "read download discarded — archive gone under lock (purged?)"); + let _ = std::fs::remove_dir_all(&tmp_path); + guard.release().await; + return Ok(DownloadOutcome::Skipped); + } + let swapped = (|| -> std::io::Result<()> { + if local_path.exists() { + std::fs::remove_dir_all(local_path)?; + } + std::fs::rename(&tmp_path, local_path) + })(); + guard.release().await; + match swapped { + Ok(()) => Ok(DownloadOutcome::Published), + Err(e) => { + let _ = std::fs::remove_dir_all(&tmp_path); + Err(e).context("publishing downloaded repo into place") + } + } + } + /// Take a write lock (Postgres advisory lock), ensure repo is local, return guard. /// The lock prevents concurrent writes to the same repo across machines. pub async fn acquire_write(&self, owner_did: &str, repo_name: &str) -> Result { @@ -1093,6 +1272,10 @@ mod tests { upload_gate: Option>, uploads: std::sync::Arc>>, deletes: std::sync::Arc>>, + // Records every download() call (pushed BEFORE the gate park, so a test + // can poll "a download is in flight" while the gate holds it). U4's + // concurrent-cold-read test counts these. + downloads: std::sync::Arc>>, } impl GatedStore { @@ -1103,6 +1286,7 @@ mod tests { upload_gate: None, uploads: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), deletes: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), + downloads: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), } } } @@ -1124,10 +1308,21 @@ mod tests { self.exists.store(true, std::sync::atomic::Ordering::SeqCst); Ok(()) } - async fn download(&self, _o: &str, _r: &str, _p: &std::path::Path) -> anyhow::Result<()> { + async fn download(&self, o: &str, r: &str, p: &std::path::Path) -> anyhow::Result<()> { + self.downloads + .lock() + .unwrap() + .push((o.to_string(), r.to_string())); if let Some(gate) = &self.download_gate { gate.notified().await; } + // Materialize a valid bare repo at the target path so a published + // download is observable on disk (mirrors the real store's + // extract-then-swap, which replaces any existing copy). + if p.exists() { + std::fs::remove_dir_all(p)?; + } + crate::git::store::init_bare(p)?; Ok(()) } async fn delete(&self, o: &str, r: &str) -> anyhow::Result<()> { @@ -1804,4 +1999,324 @@ mod tests { "the advisory lock must be freed after the bounded (timed-out) upload" ); } + + // ── U4: read-path downloads participate in the purge lock (R5, R7) ────── + + async fn poll_until_true(mut cond: impl FnMut() -> bool) -> bool { + for _ in 0..50 { + if cond() { + return true; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + false + } + + // R5: an in-flight cache-miss download must not resurrect a purged repo. + // The reader parks mid-download (holding NO advisory lock, so the purge + // proceeds), the purge runs to completion (archive deleted), and the + // resumed reader's under-lock exists() re-check must discard rather than + // publish. Pre-fix the resumed download publishes straight onto the local + // path and the repo dir returns -> RED. + #[sqlx::test] + async fn download_does_not_resurrect_a_purged_repo(pool: sqlx::PgPool) { + use std::sync::atomic::Ordering::SeqCst; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let exists = ts.exists.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkDlResurrectAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "dlresurrect"; + let dir = repo_dir_of(tmp.path(), owner, name); + + // Cache miss: the reader enters the download path and parks on the gate. + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the reader's download must be in flight" + ); + + // Purge runs to completion while the download is parked (no local dir + // exists on this cache-miss path; the archive delete is the purge). + store.delete_archive(owner, name).await.unwrap(); + assert!(!exists.load(SeqCst), "archive deleted by the purge"); + + // Resume the download; the reader must NOT publish the purged repo. + gate.notify_one(); + let res = h.await.unwrap().unwrap(); + assert_eq!(res, dir); + assert!( + !dir.exists(), + "a purged repo must not be resurrected by an in-flight read download" + ); + } + + // R5: same interleaving through acquire_fresh's refresh-over-an-existing- + // copy arm — a local dir is present when the refresh starts, and the purge + // removes BOTH the dir and the archive mid-download. Pre-fix the resumed + // refresh publishes onto the purged path and the dir returns -> RED. + #[sqlx::test] + async fn fresh_refresh_does_not_resurrect_a_purged_repo(pool: sqlx::PgPool) { + use std::sync::atomic::Ordering::SeqCst; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let exists = ts.exists.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkFreshResurrectAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "freshresurrect"; + let dir = repo_dir_of(tmp.path(), owner, name); + std::fs::create_dir_all(&dir).unwrap(); + + // The refresh always downloads, even over an existing local copy. + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire_fresh(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the refresh download must be in flight" + ); + + // Purge runs to completion mid-download: local dir AND archive removed. + std::fs::remove_dir_all(&dir).unwrap(); + store.delete_archive(owner, name).await.unwrap(); + assert!(!exists.load(SeqCst), "archive deleted by the purge"); + + gate.notify_one(); + let res = h.await.unwrap().unwrap(); + assert_eq!(res, dir); + assert!( + !dir.exists(), + "a purged repo must not be resurrected by an in-flight refresh download" + ); + } + + // R5: the degraded outcome of the under-lock skip, pinned in full. The + // purge completes before the reader reaches the advisory lock; the + // reader's under-lock exists() re-check sees false and publishes nothing: + // Ok(missing path), no dir, no republished archive, exactly one download. + // Its RED form is the same interleaving as the two resurrection tests + // above; this test pins the post-fix outcome shape. + #[sqlx::test] + async fn post_purge_download_skips_under_lock(pool: sqlx::PgPool) { + use std::sync::atomic::Ordering::SeqCst; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let uploads = ts.uploads.clone(); + let exists = ts.exists.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkPostPurgeSkipAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "postpurgeskip"; + let dir = repo_dir_of(tmp.path(), owner, name); + + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the reader's download must be in flight" + ); + store.delete_archive(owner, name).await.unwrap(); + gate.notify_one(); + + let res = h.await.unwrap().unwrap(); + assert_eq!( + res, dir, + "degraded read returns the missing path, not an error" + ); + assert!(!dir.exists(), "the skipped download must publish nothing"); + assert!( + !exists.load(SeqCst), + "the purged archive must stay deleted after the skip" + ); + assert!( + uploads.lock().unwrap().is_empty(), + "the read path must never upload" + ); + assert_eq!( + downloads.lock().unwrap().len(), + 1, + "exactly one download attempt was made" + ); + } + + // R7: two simultaneous cache-miss reads of the same repo coalesce on the + // per-repo download mutex: exactly one download occurs, both callers end + // with the published dir, neither errors. Pre-fix each caller runs its own + // download -> the count hits 2 -> RED. + #[sqlx::test] + async fn concurrent_cold_reads_download_once(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkColdCoalesceAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "coldcoalesce"; + let dir = repo_dir_of(tmp.path(), owner, name); + + let store1 = store.clone(); + let (o1, n1) = (owner.to_string(), name.to_string()); + let h1 = tokio::spawn(async move { store1.acquire(&o1, &n1).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the first reader's download must be in flight" + ); + + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h2 = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + // Give the second reader time to reach the coordination point. + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + assert_eq!( + downloads.lock().unwrap().len(), + 1, + "the second cold read must await the in-flight download, not start its own" + ); + + // Release the parked download. The second notify covers the pre-fix + // topology where both readers park on the gate; post-fix it leaves an + // unconsumed permit, which is harmless. + gate.notify_one(); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + gate.notify_one(); + + let p1 = h1.await.unwrap().unwrap(); + let p2 = h2.await.unwrap().unwrap(); + assert_eq!(p1, dir); + assert_eq!(p2, dir); + assert!(dir.exists(), "both callers end with the published dir"); + assert_eq!( + downloads.lock().unwrap().len(), + 1, + "exactly one download for two concurrent cold reads" + ); + } + + // R7/INV-15: the long network phase must not pin a lock-pool connection — + // while a download is parked mid-flight, that repo's advisory lock is + // observably FREE (out-of-pool observer). This is the distinct-repo-burst + // pool guard: N cold reads across N repos must not drain the writer-sized + // lock pool. GREEN by construction pre-fix too (the old download path held + // no lock either); it fences the new design against regression. + #[sqlx::test] + async fn no_advisory_lock_held_while_download_parked( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkNoLockParkedAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "nolockparked"; + let dir = repo_dir_of(tmp.path(), owner, name); + let key = lock_key_for(owner, name); + let mut observer = connect_opts.connect().await.unwrap(); + + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the reader's download must be in flight" + ); + + assert!( + advisory_lock_is_free(&mut observer, key).await, + "the repo's advisory lock must be free while the download is parked mid-flight" + ); + + gate.notify_one(); + let res = h.await.unwrap().unwrap(); + assert_eq!(res, dir); + assert!(dir.exists(), "the resumed download publishes normally"); + } + + // R7: acquire's local-dir hit path stays lock-free — deterministic form. + // An out-of-pool observer HOLDS the repo's advisory lock for the whole + // call window; a dir-present acquire must succeed promptly regardless (a + // lock-touching cache hit would park until the hold ends). GREEN by + // construction pre-fix (the hit path never locked); it fences the new + // design's promise that the hit path stays byte-identical and lock-free. + #[sqlx::test] + async fn cache_hit_never_touches_the_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + // exists=true so the lazy-migration spawn sees "already in tigris" and + // neither uploads nor waits on anything. + let ts = GatedStore::new(true); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkCacheHitNoLockAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "cachehitnolock"; + let dir = repo_dir_of(tmp.path(), owner, name); + std::fs::create_dir_all(&dir).unwrap(); + let key = lock_key_for(owner, name); + + // The observer takes and HOLDS the repo's advisory lock. + let mut observer = connect_opts.connect().await.unwrap(); + let (got,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut observer) + .await + .unwrap(); + assert!(got, "observer must win the free lock"); + + let res = tokio::time::timeout( + std::time::Duration::from_secs(2), + store.acquire(owner, name), + ) + .await + .expect("a dir-present acquire must not wait on the held advisory lock") + .unwrap(); + assert_eq!(res, dir); + + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut observer) + .await; + } } From ed88bb673b4f0a5ba28e34271746347f52b38365 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 23:55:24 -0500 Subject: [PATCH 46/61] fix(node): bound every under-lock object-store call and dedup the upload path (#196) Round-two review found the durability fixes reintroduced the unbounded-work-under-lock class they were meant to close: upload_locked's PUT (now reachable under the lock via init's wait=true), the read-path download, and the under-lock exists() re-check all ran with no timeout while pinning the advisory lock and its lock-pool connection. All three now wrap the call in timeout(release_upload_timeout) and fail closed on elapse, matching RepoWriteGuard::release's existing INV-22 pattern. upload_under_guard and upload_locked's duplicated resolve/recheck/upload body is extracted into upload_dir_locked, threading the load-bearing dir-gone signal (fork needs success, lazy migration needs skip-and-retry) through a parameter rather than aligning it. RED observed: upload_locked_put_stall_is_bounded failed (lock never freed) against the un-timeout'd PUT. --- crates/gitlawb-node/src/git/repo_store.rs | 161 ++++++++++++++++++---- 1 file changed, 133 insertions(+), 28 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index f6c3323e..e5407bd5 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -314,9 +314,28 @@ impl RepoStore { uuid::Uuid::new_v4() )); - if let Err(e) = store.download(owner_slug, repo_name, &tmp_path).await { - let _ = std::fs::remove_dir_all(&tmp_path); - return Err(e); + // Bound the network download: a stalled GET would park this reader (and, + // via the per-repo download mutex, every coalesced reader) indefinitely. + // A timeout takes the SAME cleanup arm as a download error — drop the + // temp dir and return Err — so `download_published` frees the map entry + // and wakes waiters rather than leaving them parked forever. + match tokio::time::timeout( + self.release_upload_timeout, + store.download(owner_slug, repo_name, &tmp_path), + ) + .await + { + Ok(Ok(())) => {} + Ok(Err(e)) => { + let _ = std::fs::remove_dir_all(&tmp_path); + return Err(e); + } + Err(_) => { + let _ = std::fs::remove_dir_all(&tmp_path); + warn!(repo = %repo_name, timeout_secs = self.release_upload_timeout.as_secs(), + "read download timed out — discarding"); + return Err(anyhow::anyhow!("read download timed out")); + } } let guard = match self.try_lock_repo(owner_did, repo_name).await { @@ -336,9 +355,19 @@ impl RepoStore { }; // Re-check under the lock: the archive gone here means a purge won the // key mid-download — discard, never publish (the read-side twin of the - // upload paths' under-lock dir re-check). An exists() error degrades - // the same way: fail closed rather than publish unconfirmed state. - if !store.exists(owner_slug, repo_name).await.unwrap_or(false) { + // upload paths' under-lock dir re-check). Bounded so a stalled HEAD + // cannot pin the advisory lock + its lock-pool connection. Both a + // timeout and an exists() error collapse to "not present" and take the + // discard arm: fail closed rather than publish unconfirmed state. + let archive_present = matches!( + tokio::time::timeout( + self.release_upload_timeout, + store.exists(owner_slug, repo_name), + ) + .await, + Ok(Ok(true)) + ); + if !archive_present { warn!(repo = %repo_name, "read download discarded — archive gone under lock (purged?)"); let _ = std::fs::remove_dir_all(&tmp_path); @@ -557,6 +586,39 @@ impl RepoStore { // No durable backend configured: nothing to upload, nothing failed. return true; }; + // Dir-gone-under-lock is SUCCESS here (`true`): the fork tail + // (api/repos.rs) treats a `false` return as a hard failure and deletes + // the freshly-cloned mirror, so a purge that removed the dir before this + // caller took the lock is nothing-to-upload, not a failed durable write. + self.upload_dir_locked(store, owner_did, repo_name, true) + .await + } + + /// Shared under-lock upload body for [`upload_under_guard`](Self::upload_under_guard) + /// and [`upload_locked`](Self::upload_locked). Both resolve+validate the + /// local path, re-check the on-disk dir still exists UNDER the caller's + /// advisory lock (a purge removes the dir under the same lock, so a caller + /// that took the key post-purge must NOT recreate the archive), then upload + /// bounded by `release_upload_timeout`. An unbounded PUT under a held + /// namespace lock is the INV-22 hazard, so the bound guarantees the guard's + /// pinned lock connection returns to the pool within a fixed window even if + /// the PUT stalls. A path-validation failure, an upload error, and a timeout + /// all return `false` (fail closed: never report durable success for a write + /// that did not land). + /// + /// `dir_gone_is_success` threads the ONE place the two callers diverge, and + /// both values are load-bearing. `upload_under_guard` passes `true`: its + /// fork-tail caller reads `false` as a hard failure and deletes the mirror, + /// so a dir purged out from under it must read as nothing-to-upload success. + /// `upload_locked` passes `false`: its lazy-migration caller must then NOT + /// mark the repo migrated, so the next `acquire` retries the migration. + async fn upload_dir_locked( + &self, + store: &Arc, + owner_did: &str, + repo_name: &str, + dir_gone_is_success: bool, + ) -> bool { let (owner_slug, local_path) = match self.local_path(owner_did, repo_name) { Ok(p) => p, Err(e) => { @@ -569,7 +631,7 @@ impl RepoStore { // and must NOT recreate the archive. if !local_path.exists() { warn!(repo = %repo_name, "object-store upload skipped: local repo dir gone under lock (purged?)"); - return true; + return dir_gone_is_success; } match tokio::time::timeout( self.release_upload_timeout, @@ -617,13 +679,6 @@ impl RepoStore { let Some(ref store) = self.object_store else { return false; }; - let (owner_slug, local_path) = match self.local_path(owner_did, repo_name) { - Ok(p) => p, - Err(e) => { - warn!(repo = %repo_name, err = %e, "rejected unsafe path before object-store upload"); - return false; - } - }; let guard = if wait { match self.lock_repo_blocking(owner_did, repo_name).await { Ok(Some(g)) => g, @@ -649,20 +704,13 @@ impl RepoStore { } } }; - // Re-check under the lock: a purge removes the on-disk dir under its lock, - // so a post-purge uploader must find it gone and NOT recreate the archive. - if !local_path.exists() { - warn!(repo = %repo_name, "object-store upload skipped — local repo dir gone under lock (purged?)"); - guard.release().await; - return false; - } - let uploaded = match store.upload(&owner_slug, repo_name, &local_path).await { - Ok(()) => true, - Err(e) => { - warn!(repo = %repo_name, err = %e, "failed to upload repo to object store"); - false - } - }; + // Shared under-lock body: re-check the dir exists and PUT bounded by + // `release_upload_timeout` (INV-22 — an untimed PUT here would pin this + // guard's lock connection forever). Dir-gone returns `false` so the + // lazy-migration caller does NOT mark the repo migrated and retries. + let uploaded = self + .upload_dir_locked(store, owner_did, repo_name, false) + .await; guard.release().await; uploaded } @@ -1363,6 +1411,16 @@ mod tests { false } + async fn poll_until_held(conn: &mut sqlx::PgConnection, key: i64) -> bool { + for _ in 0..50 { + if !advisory_lock_is_free(conn, key).await { + return true; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + false + } + fn lock_key_for(owner: &str, name: &str) -> i64 { advisory_lock_key(&owner.replace([':', '/'], "_"), name) } @@ -2000,6 +2058,53 @@ mod tests { ); } + // R2/INV-22: the OTHER under-lock upload path — upload_locked's PUT (init's + // background upload, wait=true) — must be bounded exactly like release()'s. + // With a stalled PUT, upload_locked acquires the advisory lock, parks on the + // PUT, and the timeout is the only thing that frees the lock. Pre-fix + // (untimed store.upload) the lock is pinned forever -> poll_until_free RED. + #[sqlx::test] + async fn upload_locked_put_stall_is_bounded( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(false); + // The PUT parks forever; the upload bound is what must unblock it. + ts.upload_gate = Some(std::sync::Arc::new(tokio::sync::Notify::new())); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ) + .with_release_upload_timeout(std::time::Duration::from_millis(300)); + let owner = "did:key:z6MkUploadLockedStallAAAAAAAAAAAAAAAAAAAAAA"; + let name = "uploadlockedstall"; + let key = lock_key_for(owner, name); + let mut observer = connect_opts.connect().await.unwrap(); + + // init spawns upload_locked(wait=true); it inits the bare dir, acquires + // the lock, then parks on the stalled PUT. + store.init(owner, name).await.unwrap(); + + // The spawned upload must first take the advisory lock (then stall on the + // PUT) — proving the lock IS held, so the free-within-bound assert below + // is load-bearing and not vacuously green on a never-acquired lock. + assert!( + poll_until_held(&mut observer, key).await, + "the spawned init upload must acquire the advisory lock before its PUT" + ); + + // The PUT is parked forever: only the bound can free the lock. Pre-fix + // (untimed PUT) it never frees within the 5s poll window -> RED. + assert!( + poll_until_free(&mut observer, key).await, + "upload_locked's PUT must be bounded — the advisory lock must free within the timeout" + ); + } + // ── U4: read-path downloads participate in the purge lock (R5, R7) ────── async fn poll_until_true(mut cond: impl FnMut() -> bool) -> bool { From 3341cac03c5cc047f620bc181e44fbea08294679 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 00:13:35 -0500 Subject: [PATCH 47/61] fix(node): harden the read-path download against cancellation, stale swaps, and lock contention (#196) Four read-path gaps the review surfaced. A TempDownloadDir RAII guard now cleans the temp download dir if the handler future is cancelled on client disconnect (the read handlers have no spawn shield, and a post-download disconnect leaked the full populated dir), plus an entry-time sweep of stale siblings. A cold or refresh reader that downloaded while a concurrent writer published fresher on-disk state now discards its stale temp and serves the writer's copy instead of swapping over it (detected by presence for the cold path, mtime for the refresh path). The takeover path re-registers its download-mutex entry so late readers coalesce instead of starting a duplicate download. And the try_lock Ok(None) degrade arm - the KTD-3 invariant that a reader degrades rather than resurrecting under a live writer/purge - is now tested by holding the advisory lock via an out-of-pool observer. RED observed on both stale-swap tests. --- crates/gitlawb-node/src/git/repo_store.rs | 446 ++++++++++++++++++++-- 1 file changed, 423 insertions(+), 23 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index e5407bd5..92aa61bc 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -65,6 +65,40 @@ enum DownloadOutcome { Skipped, } +/// RAII cleanup for a read-path download temp dir (finding 1). Its `Drop` +/// removes the dir (best effort), so a handler future cancelled mid-download +/// (axum drops it on client disconnect, the same hazard `RepoWriteGuard` / +/// `RepoLockGuard` Drop impls exist for) cannot leak a fully-populated repo +/// dir. `disarm` forgets the dir once the publish rename has consumed it. +struct TempDownloadDir { + path: PathBuf, + armed: bool, +} + +impl TempDownloadDir { + fn new(path: PathBuf) -> Self { + Self { path, armed: true } + } + + fn path(&self) -> &Path { + &self.path + } + + /// Disarm: the temp has been consumed (renamed into place), so `Drop` must + /// not try to remove it. + fn disarm(mut self) { + self.armed = false; + } +} + +impl Drop for TempDownloadDir { + fn drop(&mut self) { + if self.armed { + let _ = std::fs::remove_dir_all(&self.path); + } + } +} + impl RepoStore { #[cfg(test)] pub fn for_testing(repos_dir: PathBuf, lock_pool: PgPool) -> Self { @@ -155,7 +189,14 @@ impl RepoStore { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "cache miss — downloading from tigris"); match self - .download_published(owner_did, repo_name, &owner_slug, &local_path, tigris) + .download_published( + owner_did, + repo_name, + &owner_slug, + &local_path, + tigris, + false, + ) .await .context("downloading repo from tigris")? { @@ -193,7 +234,14 @@ impl RepoStore { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "acquire_fresh: downloading latest from tigris"); match self - .download_published(owner_did, repo_name, &owner_slug, &local_path, tigris) + .download_published( + owner_did, + repo_name, + &owner_slug, + &local_path, + tigris, + true, + ) .await { // Published covers both the refreshed copy and a concurrent @@ -241,6 +289,7 @@ impl RepoStore { owner_slug: &str, local_path: &Path, store: &Arc, + local_existed_at_entry: bool, ) -> Result { let map_key = format!("{owner_slug}/{repo_name}"); let entry = { @@ -261,11 +310,28 @@ impl RepoStore { return Ok(DownloadOutcome::Published); } // The holder degraded without publishing; this reader takes over. + // The prior holder removed the map entry on its way out, so + // re-register THIS entry before downloading, so readers arriving + // now coalesce onto it instead of starting a duplicate download + // (finding 3). Narrows the duplicate-download window; publishes + // stay serialized by the advisory lock regardless. + { + let mut map = self.download_locks.lock().await; + map.entry(map_key.clone()) + .or_insert_with(|| Arc::clone(&entry)); + } g } }; let outcome = self - .download_and_publish(owner_did, repo_name, owner_slug, local_path, store) + .download_and_publish( + owner_did, + repo_name, + owner_slug, + local_path, + store, + local_existed_at_entry, + ) .await; self.remove_download_entry(&map_key, &entry).await; drop(held); @@ -292,6 +358,12 @@ impl RepoStore { /// the temp copy into place, release. Advisory contention or a lock-pool /// error discards the temp copy and degrades — no blocking lock or pool /// wait exists anywhere on the read path. + /// + /// `local_existed_at_entry` selects the stale-swap guard (finding 2): a cold + /// `acquire` passes `false`, an `acquire_fresh` refresh passes `true`. Under + /// the lock, before the swap, we detect a writer that published a fresher + /// copy during our unlocked download window and serve theirs instead of + /// clobbering it with our now-stale temp. async fn download_and_publish( &self, owner_did: &str, @@ -299,6 +371,7 @@ impl RepoStore { owner_slug: &str, local_path: &Path, store: &Arc, + local_existed_at_entry: bool, ) -> Result { let parent = local_path.parent().context("repo path has no parent")?; std::fs::create_dir_all(parent).context("creating repo parent dir")?; @@ -306,32 +379,56 @@ impl RepoStore { .file_name() .context("repo path has no file name")? .to_string_lossy(); + + // Best-effort sweep of leftover temp siblings from a prior download whose + // RAII guard dropped, or whose extract spawn_blocking completed + // uninterruptibly AFTER that guard's Drop ran (finding 1). Serialized per + // repo by the download mutex, so no concurrent same-repo download owns a + // live temp here; scoped to THIS repo's prefix, so other repos are + // untouched. + let tmp_prefix = format!(".{file_name}.tmp-download."); + if let Ok(entries) = std::fs::read_dir(parent) { + for entry in entries.flatten() { + if entry.file_name().to_string_lossy().starts_with(&tmp_prefix) { + let _ = std::fs::remove_dir_all(entry.path()); + } + } + } + + // Stale-swap signal (finding 2): capture the local dir's mtime BEFORE the + // unlocked download. Under the lock, a changed mtime (acquire_fresh, + // refreshing over an existing copy) or the dir's mere appearance (cold + // acquire) means a writer published a fresher copy during our download + // window, so we serve theirs rather than swap our stale temp over it. + let entry_mtime = std::fs::metadata(local_path) + .and_then(|m| m.modified()) + .ok(); + // Unique per-download temp target (same parent as local_path so the // publish rename stays on one filesystem); mirrors the extract temp - // naming in tigris.rs. - let tmp_path = parent.join(format!( + // naming in tigris.rs. Wrapped in an RAII guard so a handler future + // cancelled mid-download (axum drops it on client disconnect) cannot + // leak the populated temp dir (the same hazard the guard Drop impls + // below exist for, finding 1). + let temp = TempDownloadDir::new(parent.join(format!( ".{file_name}.tmp-download.{}", uuid::Uuid::new_v4() - )); + ))); // Bound the network download: a stalled GET would park this reader (and, // via the per-repo download mutex, every coalesced reader) indefinitely. - // A timeout takes the SAME cleanup arm as a download error — drop the - // temp dir and return Err — so `download_published` frees the map entry - // and wakes waiters rather than leaving them parked forever. + // A timeout takes the SAME cleanup arm as a download error (return Err, + // the temp guard's Drop removes the dir), so `download_published` frees + // the map entry and wakes waiters rather than leaving them parked forever. match tokio::time::timeout( self.release_upload_timeout, - store.download(owner_slug, repo_name, &tmp_path), + store.download(owner_slug, repo_name, temp.path()), ) .await { Ok(Ok(())) => {} - Ok(Err(e)) => { - let _ = std::fs::remove_dir_all(&tmp_path); - return Err(e); - } + Ok(Err(e)) => return Err(e), Err(_) => { - let _ = std::fs::remove_dir_all(&tmp_path); warn!(repo = %repo_name, timeout_secs = self.release_upload_timeout.as_secs(), "read download timed out — discarding"); return Err(anyhow::anyhow!("read download timed out")); @@ -343,13 +440,11 @@ impl RepoStore { Ok(None) => { debug!(repo = %repo_name, "read download discarded — repo lock held by a live writer or purge"); - let _ = std::fs::remove_dir_all(&tmp_path); return Ok(DownloadOutcome::Skipped); } Err(e) => { warn!(repo = %repo_name, err = %e, "read download discarded — could not acquire repo lock"); - let _ = std::fs::remove_dir_all(&tmp_path); return Ok(DownloadOutcome::Skipped); } }; @@ -370,23 +465,50 @@ impl RepoStore { if !archive_present { warn!(repo = %repo_name, "read download discarded — archive gone under lock (purged?)"); - let _ = std::fs::remove_dir_all(&tmp_path); guard.release().await; return Ok(DownloadOutcome::Skipped); } + + // Stale-swap guard (finding 2): a writer that locked, published a fresher + // copy, and released during our unlocked download window must not have + // its work clobbered by our now-stale temp. The only concurrent mutator + // of local_path is a writer or a purge (readers of the same repo + // serialize on the download mutex), so the signals below unambiguously + // mean "a writer published here." + let writer_published = match (local_existed_at_entry, entry_mtime) { + // acquire_fresh over an existing copy: presence is always true, so it + // cannot signal a writer; a CHANGED mtime can (the writer replaced + // the dir via rename, minting a fresh mtime). + (true, Some(entry_m)) => std::fs::metadata(local_path) + .and_then(|m| m.modified()) + .map(|now| now != entry_m) + .unwrap_or(false), + // Cold acquire, or acquire_fresh whose dir was absent at entry: the + // dir being present now means a writer published it during the window. + _ => local_path.exists(), + }; + if writer_published { + warn!(repo = %repo_name, + "read download discarded: a writer published a fresher copy during the download window"); + guard.release().await; + // temp guard's Drop removes the stale temp; serve the fresher local copy. + return Ok(DownloadOutcome::Published); + } + let swapped = (|| -> std::io::Result<()> { if local_path.exists() { std::fs::remove_dir_all(local_path)?; } - std::fs::rename(&tmp_path, local_path) + std::fs::rename(temp.path(), local_path) })(); guard.release().await; match swapped { - Ok(()) => Ok(DownloadOutcome::Published), - Err(e) => { - let _ = std::fs::remove_dir_all(&tmp_path); - Err(e).context("publishing downloaded repo into place") + Ok(()) => { + temp.disarm(); // the rename consumed the temp; nothing to clean up + Ok(DownloadOutcome::Published) } + // temp guard's Drop removes the temp if the rename left it behind. + Err(e) => Err(e).context("publishing downloaded repo into place"), } } @@ -2424,4 +2546,282 @@ mod tests { .execute(&mut observer) .await; } + + // ── Finding 2: stale-swap over a fresher writer copy ──────────────────── + + // A cold `acquire` reader downloads to a temp holding NO advisory lock; + // during that window a writer can lock, publish a fresher on-disk copy, and + // release. When the reader finally takes the lock, a presence-only check + // would swap its now-stale temp over the writer's fresher dir. The guard + // detects the writer's dir (absent at entry, present under the lock) and + // serves it instead. Pre-fix the reader swaps and the writer's copy is lost + // -> RED (the sentinel disappears). + #[sqlx::test] + async fn cold_read_defers_to_writer_published_during_download(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let uploads = ts.uploads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkColdDeferAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "colddefer"; + let dir = repo_dir_of(tmp.path(), owner, name); + + // Cache miss: the reader enters the download path and parks (no local dir). + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the reader's download must be in flight" + ); + + // A writer publishes a fresher copy onto the local path during the + // window (simulated directly: the writer would hold the lock only while + // it works, which is over before the reader takes the lock). + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("writer_sentinel"), b"fresher").unwrap(); + + gate.notify_one(); + let res = h.await.unwrap().unwrap(); + assert_eq!(res, dir); + assert!( + dir.join("writer_sentinel").exists(), + "the reader must serve the writer's fresher copy, not swap its stale temp over it" + ); + assert!( + uploads.lock().unwrap().is_empty(), + "the read path must never upload" + ); + assert_eq!(downloads.lock().unwrap().len(), 1, "exactly one download"); + } + + // acquire_fresh refreshes over an EXISTING local copy, so a presence check + // cannot detect a concurrent writer (the dir is always present). The guard + // compares the local dir's mtime captured at entry against its value under + // the lock; a writer that replaced the dir during the window changed the + // mtime, so the reader serves the writer's copy rather than swapping its + // stale refresh over it. Pre-fix the reader swaps -> RED (version B lost). + #[sqlx::test] + async fn fresh_refresh_defers_to_writer_published_during_download(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkFreshDeferAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "freshdefer"; + let dir = repo_dir_of(tmp.path(), owner, name); + + // Version A already on disk when the refresh starts. + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("version_a"), b"A").unwrap(); + let mtime_before = std::fs::metadata(&dir).unwrap().modified().unwrap(); + + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire_fresh(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the refresh download must be in flight" + ); + + // Cross a filesystem mtime tick, then a writer replaces the dir with + // version B. The long sleep guarantees B's mtime differs from A's even + // on a coarse-granularity filesystem, so the comparison is deterministic. + tokio::time::sleep(std::time::Duration::from_millis(1_100)).await; + std::fs::remove_dir_all(&dir).unwrap(); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("version_b"), b"B").unwrap(); + let mtime_after = std::fs::metadata(&dir).unwrap().modified().unwrap(); + assert!( + mtime_after != mtime_before, + "precondition: the writer's replacement must change the dir mtime for the guard to fire" + ); + + gate.notify_one(); + let res = h.await.unwrap().unwrap(); + assert_eq!(res, dir); + assert!( + dir.join("version_b").exists(), + "the refresh must serve the writer's fresher copy (B), not swap version A's stale temp over it" + ); + assert!( + !dir.join("version_a").exists(), + "version A must be gone; the writer replaced it, and the reader must not resurrect it" + ); + assert_eq!(downloads.lock().unwrap().len(), 1, "exactly one download"); + } + + // ── Finding 4: the try_lock_repo Ok(None) contention degrade ──────────── + + fn has_leftover_temp(parent: &std::path::Path) -> bool { + std::fs::read_dir(parent) + .map(|rd| { + rd.flatten() + .any(|e| e.file_name().to_string_lossy().contains(".tmp-download.")) + }) + .unwrap_or(false) + } + + // KTD-3 degrade: while a live writer/purge holds the repo's advisory lock, a + // read-path downloader that reaches try_lock AFTER its fetch must discard its + // temp and degrade. This exercises the Ok(None) arm the six U4 tests never + // reach (they simulate purge via delete_archive and never hold the lock + // while a downloader reaches try_lock). An out-of-pool observer holds the + // lock; GatedStore.download_gate parks the reader past the fetch so it + // reaches try_lock while the observer still holds it. + // + // acquire variant: degrades to the missing path, publishes nothing. + #[sqlx::test] + async fn acquire_download_degrades_when_writer_holds_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let uploads = ts.uploads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkAcquireDegradeAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "acqdegrade"; + let dir = repo_dir_of(tmp.path(), owner, name); + let key = lock_key_for(owner, name); + + // An out-of-pool observer holds the repo's advisory lock for the window. + let mut observer = connect_opts.connect().await.unwrap(); + let (got,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut observer) + .await + .unwrap(); + assert!(got, "observer must win the free lock"); + + // Cache miss: reader downloads to temp (no lock held), parks on the gate. + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the reader's download must be in flight" + ); + + // Release the gate: the reader reaches try_lock while the observer holds + // the lock -> Ok(None) -> discard temp, degrade to the missing path. + gate.notify_one(); + let res = h.await.unwrap().unwrap(); + assert_eq!( + res, dir, + "degraded acquire returns the missing path, not an error" + ); + assert!(!dir.exists(), "the contended download must publish nothing"); + assert!( + uploads.lock().unwrap().is_empty(), + "the read path must never upload" + ); + assert_eq!( + downloads.lock().unwrap().len(), + 1, + "exactly one download attempt" + ); + assert!( + !has_leftover_temp(dir.parent().unwrap()), + "the discarded download temp must be cleaned up" + ); + + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut observer) + .await; + } + + // acquire_fresh variant: degrades to serving the existing local copy. + #[sqlx::test] + async fn fresh_download_degrades_to_local_copy_when_writer_holds_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let uploads = ts.uploads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkFreshDegradeAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "freshdegrade"; + let dir = repo_dir_of(tmp.path(), owner, name); + let key = lock_key_for(owner, name); + + // An existing local copy is present when the refresh starts. + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("local_copy"), b"keep").unwrap(); + + let mut observer = connect_opts.connect().await.unwrap(); + let (got,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut observer) + .await + .unwrap(); + assert!(got, "observer must win the free lock"); + + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire_fresh(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the refresh download must be in flight" + ); + + gate.notify_one(); + let res = h.await.unwrap().unwrap(); + assert_eq!(res, dir, "degraded acquire_fresh returns the local path"); + assert!( + dir.join("local_copy").exists(), + "the contended refresh must serve the existing local copy untouched" + ); + assert!( + uploads.lock().unwrap().is_empty(), + "the read path must never upload" + ); + assert_eq!( + downloads.lock().unwrap().len(), + 1, + "exactly one download attempt" + ); + assert!( + !has_leftover_temp(dir.parent().unwrap()), + "the discarded download temp must be cleaned up" + ); + + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut observer) + .await; + } } From 5e3e98b0e1ebbf0a1c550011fe2381c10147e278 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 00:26:47 -0500 Subject: [PATCH 48/61] fix(node): keep the fork's unbounded work out of the target-lock span (#196) The fork tail held the target-namespace advisory lock across two unbounded operations. The source acquire, which on a cold source triggers a full object-store download plus a nested lock-pool acquire on the same writer-sized pool, now runs before the target lock is taken (it is read-only on a different namespace and nothing under the lock depends on it) so a cold-source fork can no longer demand two connections from a one-per-writer pool at once. The git clone --mirror now runs via tokio::process with kill_on_drop under a timeout(git_service_timeout_secs) ceiling, cleaning the partial mirror and releasing the lock on elapse. RED observed by executing the pool-size-1 cold-source fork with the source acquire moved back under the lock: PoolTimedOut, source never publishes, clone fails. --- crates/gitlawb-node/src/api/repos.rs | 196 ++++++++++++++++++++++++--- 1 file changed, 179 insertions(+), 17 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 6def2a84..b0318c1c 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1658,6 +1658,20 @@ pub async fn fork_repo( )); } + // Materialize the SOURCE on local disk BEFORE taking the target lock + // (downloads from Tigris on cache miss). On a cold source (archive-only, no + // local dir) this download takes a nested advisory lock on the write + // lock_pool for the source's OWN namespace; doing it under the held target + // lock would need two lock-pool connections at once from a pool sized + // one-per-writer. The source is a read-only, different-namespace concern and + // needs no target-namespace protection, and nothing under the lock below + // depends on it beyond the clone reading source_path. + let source_path = state + .repo_store + .acquire(&source.owner_did, &source.name) + .await + .map_err(|e| AppError::Git(e.to_string()))?; + // Serialize against a concurrent same-key purge or creation on the FORK // TARGET namespace, mirroring create_repo's lock above (same rationale, // same 503 mapping). Held across the conflict check, clone, durable upload, @@ -1687,25 +1701,55 @@ pub async fn fork_repo( // Request is admissible — spend the proof now, immediately before the write. let verified_proof = proof.consume(&state.db).await?; - // Ensure source repo is on local disk (downloads from Tigris on cache miss) - let source_path = state - .repo_store - .acquire(&source.owner_did, &source.name) - .await - .map_err(|e| AppError::Git(e.to_string()))?; - let disk_path = store::repo_disk_path(&state.config.repos_dir, &forker_did, &fork_name); - // Clone the source repo as a mirror - let output = std::process::Command::new("git") - .args([ - "clone", - "--mirror", - source_path.to_str().unwrap_or(""), - disk_path.to_str().unwrap_or(""), - ]) - .output() - .map_err(|e| AppError::Git(format!("git clone --mirror failed: {e}")))?; + // Clone the source repo as a mirror, bounded so a pathological or huge source + // cannot pin the held target lock (and its lock-pool connection) + // indefinitely. Reuses the served-git ceiling (git_service_timeout_secs, + // generous for large clones); after the source reorder above source_path is a + // LOCAL path so a normal clone is fast and never approaches the bound, which + // is a safety ceiling only. tokio::process with kill_on_drop tears the child + // down when the timeout drops the future. + let clone_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let output = match tokio::time::timeout( + clone_timeout, + tokio::process::Command::new("git") + .args([ + "clone", + "--mirror", + source_path.to_str().unwrap_or(""), + disk_path.to_str().unwrap_or(""), + ]) + .kill_on_drop(true) + .output(), + ) + .await + { + Ok(Ok(output)) => output, + // Spawn/IO failure, or the clone exceeded the bound (child killed on + // drop). Clear any partial mirror and fail the fork with the same + // retryable shape the durable-upload arm below returns, before any + // RepoRecord exists. + Ok(Err(e)) => { + if let Err(rm) = std::fs::remove_dir_all(&disk_path) { + tracing::warn!(fork = %fork_name, err = %rm, + "failed to remove fork mirror after a failed clone spawn"); + } + repo_lock.release().await; + return Err(AppError::Git(format!("git clone --mirror failed: {e}"))); + } + Err(_elapsed) => { + if let Err(rm) = std::fs::remove_dir_all(&disk_path) { + tracing::warn!(fork = %fork_name, err = %rm, + "failed to remove fork mirror after a clone timeout"); + } + repo_lock.release().await; + return Err(AppError::Git(format!( + "git clone --mirror timed out after {}s for {fork_name}", + clone_timeout.as_secs() + ))); + } + }; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); @@ -4261,6 +4305,124 @@ mod tests { held.release().await; } + // Finding 1: forking a COLD source (archive-only, no local dir) must + // materialize the source out from under the target lock, so it works even + // when the write lock_pool is sized 1. On a cold source, acquire()'s + // download takes a nested advisory lock on that SAME lock_pool for the + // SOURCE's namespace; if the target lock (which pins the pool's one + // connection) were held first, that nested acquire would find the pool + // exhausted, PoolTimedOut -> the source publish degrades -> the source never + // lands on disk -> the clone of a missing source fails the fork. RED on the + // pre-reorder code (source acquire AFTER lock_repo_blocking): fork fails + // with a 5xx at a size-1 lock pool. GREEN after: acquire runs before the + // lock, so the two nested acquires never overlap. + #[sqlx::test] + async fn fork_cold_source_materializes_before_target_lock_at_pool_size_one( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use std::path::Path as StdPath; + + // An object store standing in for a cold source: exists() is always + // true, and download() materializes a valid bare repo at the target path + // (mirroring the real store's extract-then-swap), so acquire() publishes + // the source on disk. upload() succeeds so the fork's own durable upload + // is a no-op success. + struct ColdMaterializeStore; + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for ColdMaterializeStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(true) + } + async fn upload(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, p: &StdPath) -> anyhow::Result<()> { + if p.exists() { + std::fs::remove_dir_all(p)?; + } + crate::git::store::init_bare(p)?; + Ok(()) + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + // App pool for state.db (get_repo / proof.consume / create_repo), sized + // generously so unrelated app work never contends. The hazard under test + // is on the SEPARATE lock pool below. + let app_pool = pool_opts + .max_connections(5) + .connect_with(connect_opts.clone()) + .await + .unwrap(); + // The write lock_pool sized to ONE connection: a target lock plus an + // overlapping source-namespace lock cannot both be satisfied from it, so + // this size is exactly what surfaces the double-hold hazard. + let lock_pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(std::time::Duration::from_secs(2)) + .min_connections(0) + .connect_with(connect_opts) + .await + .unwrap(); + + let repos_dir = tempfile::TempDir::new().unwrap(); + let repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(ColdMaterializeStore)), + lock_pool, + ); + + // Build state WITHOUT creating the source bare on disk: the source lives + // only in the object store, so acquire() must download it. (fork_test_state + // would materialize it locally, which is the warm path, not this one.) + let mut state = crate::test_support::test_state(app_pool).await; + state.repo_store = repo_store; + let mut cfg = (*state.config).clone(); + cfg.repos_dir = repos_dir.path().to_path_buf(); + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo( + FORK_SRC_OWNER, + FORK_SRC_NAME, + "unused-cold-path", + None, + false, + ) + .await + .unwrap(); + // The source must NOT be on local disk, or acquire takes the warm path + // and never exercises the cold download + nested lock. + let src_local = + store::repo_disk_path(&state.config.repos_dir, FORK_SRC_OWNER, FORK_SRC_NAME); + assert!(!src_local.exists(), "source must be cold (absent locally)"); + + let forker = "did:key:z6MkForkColdSrcAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let fork_name = "coldfork"; + + let resp = call_fork(&state, forker, fork_name) + .await + .expect("cold-source fork must succeed at a size-1 lock pool"); + assert_eq!(resp.status(), StatusCode::CREATED); + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_some(), + "repos row present after the cold-source fork" + ); + let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name); + assert!( + clone_dir.exists(), + "cloned mirror present after the cold-source fork" + ); + } + // create_repo must NOT take its serialization lock from the APP pool. Doing so // pins an app-pool connection in the lock guard while the create's own work // (get_repo / proof.consume / db.create_repo, all on state.db = the app pool) From 41f6dce5e4eec92b8446c773f5e47d74b6acf944 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 00:35:28 -0500 Subject: [PATCH 49/61] refactor(node): move read-path download coordination into a repo_store submodule (#196) repo_store.rs had grown past 2800 lines. The download-coordination code (DownloadOutcome, TempDownloadDir, download_published, download_and_publish, remove_download_entry) moves to a private repo_store/download.rs child module, which reaches the parent's private fields and methods without widening visibility beyond pub(super) on the two items the parent names. Pure move: no behavior change, the same 44 git::repo_store tests pass unchanged. --- crates/gitlawb-node/src/git/repo_store.rs | 287 +---------------- .../src/git/repo_store/download.rs | 297 ++++++++++++++++++ 2 files changed, 300 insertions(+), 284 deletions(-) create mode 100644 crates/gitlawb-node/src/git/repo_store/download.rs diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 92aa61bc..dd8bdaa5 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -21,6 +21,9 @@ use tracing::{debug, info, warn}; use super::store; use super::tigris::ObjectStore; +mod download; +use download::DownloadOutcome; + /// Default bound on `release()`'s post-write archive upload (INV-22). Generous /// for a large-repo PUT while still guaranteeing the guard's pinned advisory- /// lock connection returns to the pool within a fixed window even if the upload @@ -53,52 +56,6 @@ pub struct RepoStore { download_locks: Arc>>>>, } -/// Outcome of a coordinated read-path download (KTD-3). -enum DownloadOutcome { - /// The local dir is present: this reader downloaded and published it under - /// the advisory lock, or a concurrent reader did and this one served it. - Published, - /// The download was discarded without publishing: the advisory lock was - /// contended (a live writer or purge holds it), the lock pool errored, or - /// the archive vanished under the lock (purged mid-download). The caller - /// degrades to its missing-path or serve-local-copy outcome. - Skipped, -} - -/// RAII cleanup for a read-path download temp dir (finding 1). Its `Drop` -/// removes the dir (best effort), so a handler future cancelled mid-download -/// (axum drops it on client disconnect, the same hazard `RepoWriteGuard` / -/// `RepoLockGuard` Drop impls exist for) cannot leak a fully-populated repo -/// dir. `disarm` forgets the dir once the publish rename has consumed it. -struct TempDownloadDir { - path: PathBuf, - armed: bool, -} - -impl TempDownloadDir { - fn new(path: PathBuf) -> Self { - Self { path, armed: true } - } - - fn path(&self) -> &Path { - &self.path - } - - /// Disarm: the temp has been consumed (renamed into place), so `Drop` must - /// not try to remove it. - fn disarm(mut self) { - self.armed = false; - } -} - -impl Drop for TempDownloadDir { - fn drop(&mut self) { - if self.armed { - let _ = std::fs::remove_dir_all(&self.path); - } - } -} - impl RepoStore { #[cfg(test)] pub fn for_testing(repos_dir: PathBuf, lock_pool: PgPool) -> Self { @@ -274,244 +231,6 @@ impl RepoStore { Ok(local_path) } - /// Coordinated read-path download (KTD-3). Serializes concurrent readers - /// of the same repo on a per-repo async mutex: the first one in becomes - /// the holder and runs [`download_and_publish`](Self::download_and_publish); - /// a contended reader awaits the holder, re-checks the local dir on wake, - /// and serves what the holder published instead of re-downloading. Only - /// callers that already confirmed the archive exists reach this, and the - /// map entry is removed on completion, so the map cannot grow per - /// arbitrary requested name. - async fn download_published( - &self, - owner_did: &str, - repo_name: &str, - owner_slug: &str, - local_path: &Path, - store: &Arc, - local_existed_at_entry: bool, - ) -> Result { - let map_key = format!("{owner_slug}/{repo_name}"); - let entry = { - let mut map = self.download_locks.lock().await; - Arc::clone( - map.entry(map_key.clone()) - .or_insert_with(|| Arc::new(Mutex::new(()))), - ) - }; - let held = match entry.try_lock() { - Ok(g) => g, - Err(_) => { - let g = entry.lock().await; - if local_path.exists() { - // A concurrent reader published while we waited — serve it. - drop(g); - self.remove_download_entry(&map_key, &entry).await; - return Ok(DownloadOutcome::Published); - } - // The holder degraded without publishing; this reader takes over. - // The prior holder removed the map entry on its way out, so - // re-register THIS entry before downloading, so readers arriving - // now coalesce onto it instead of starting a duplicate download - // (finding 3). Narrows the duplicate-download window; publishes - // stay serialized by the advisory lock regardless. - { - let mut map = self.download_locks.lock().await; - map.entry(map_key.clone()) - .or_insert_with(|| Arc::clone(&entry)); - } - g - } - }; - let outcome = self - .download_and_publish( - owner_did, - repo_name, - owner_slug, - local_path, - store, - local_existed_at_entry, - ) - .await; - self.remove_download_entry(&map_key, &entry).await; - drop(held); - outcome - } - - /// Remove a completed download-coordination entry, but only when the map - /// still holds THIS entry — a newer entry inserted after an earlier - /// removal must not be clobbered. - async fn remove_download_entry(&self, key: &str, entry: &Arc>) { - let mut map = self.download_locks.lock().await; - if map.get(key).is_some_and(|cur| Arc::ptr_eq(cur, entry)) { - map.remove(key); - } - } - - /// The holder's half of the coordinated download: fetch and extract the - /// archive into a temp sibling with NO advisory lock held (the network - /// phase must never pin a lock-pool connection — a cold-read burst across - /// distinct repos must not drain the writer-sized pool), then take the - /// per-repo advisory lock only around the publish: re-check the archive - /// still exists under the lock (a purge deletes it under this same lock, - /// so a post-purge downloader discards rather than resurrecting), swap - /// the temp copy into place, release. Advisory contention or a lock-pool - /// error discards the temp copy and degrades — no blocking lock or pool - /// wait exists anywhere on the read path. - /// - /// `local_existed_at_entry` selects the stale-swap guard (finding 2): a cold - /// `acquire` passes `false`, an `acquire_fresh` refresh passes `true`. Under - /// the lock, before the swap, we detect a writer that published a fresher - /// copy during our unlocked download window and serve theirs instead of - /// clobbering it with our now-stale temp. - async fn download_and_publish( - &self, - owner_did: &str, - repo_name: &str, - owner_slug: &str, - local_path: &Path, - store: &Arc, - local_existed_at_entry: bool, - ) -> Result { - let parent = local_path.parent().context("repo path has no parent")?; - std::fs::create_dir_all(parent).context("creating repo parent dir")?; - let file_name = local_path - .file_name() - .context("repo path has no file name")? - .to_string_lossy(); - - // Best-effort sweep of leftover temp siblings from a prior download whose - // RAII guard dropped, or whose extract spawn_blocking completed - // uninterruptibly AFTER that guard's Drop ran (finding 1). Serialized per - // repo by the download mutex, so no concurrent same-repo download owns a - // live temp here; scoped to THIS repo's prefix, so other repos are - // untouched. - let tmp_prefix = format!(".{file_name}.tmp-download."); - if let Ok(entries) = std::fs::read_dir(parent) { - for entry in entries.flatten() { - if entry.file_name().to_string_lossy().starts_with(&tmp_prefix) { - let _ = std::fs::remove_dir_all(entry.path()); - } - } - } - - // Stale-swap signal (finding 2): capture the local dir's mtime BEFORE the - // unlocked download. Under the lock, a changed mtime (acquire_fresh, - // refreshing over an existing copy) or the dir's mere appearance (cold - // acquire) means a writer published a fresher copy during our download - // window, so we serve theirs rather than swap our stale temp over it. - let entry_mtime = std::fs::metadata(local_path) - .and_then(|m| m.modified()) - .ok(); - - // Unique per-download temp target (same parent as local_path so the - // publish rename stays on one filesystem); mirrors the extract temp - // naming in tigris.rs. Wrapped in an RAII guard so a handler future - // cancelled mid-download (axum drops it on client disconnect) cannot - // leak the populated temp dir (the same hazard the guard Drop impls - // below exist for, finding 1). - let temp = TempDownloadDir::new(parent.join(format!( - ".{file_name}.tmp-download.{}", - uuid::Uuid::new_v4() - ))); - - // Bound the network download: a stalled GET would park this reader (and, - // via the per-repo download mutex, every coalesced reader) indefinitely. - // A timeout takes the SAME cleanup arm as a download error (return Err, - // the temp guard's Drop removes the dir), so `download_published` frees - // the map entry and wakes waiters rather than leaving them parked forever. - match tokio::time::timeout( - self.release_upload_timeout, - store.download(owner_slug, repo_name, temp.path()), - ) - .await - { - Ok(Ok(())) => {} - Ok(Err(e)) => return Err(e), - Err(_) => { - warn!(repo = %repo_name, timeout_secs = self.release_upload_timeout.as_secs(), - "read download timed out — discarding"); - return Err(anyhow::anyhow!("read download timed out")); - } - } - - let guard = match self.try_lock_repo(owner_did, repo_name).await { - Ok(Some(g)) => g, - Ok(None) => { - debug!(repo = %repo_name, - "read download discarded — repo lock held by a live writer or purge"); - return Ok(DownloadOutcome::Skipped); - } - Err(e) => { - warn!(repo = %repo_name, err = %e, - "read download discarded — could not acquire repo lock"); - return Ok(DownloadOutcome::Skipped); - } - }; - // Re-check under the lock: the archive gone here means a purge won the - // key mid-download — discard, never publish (the read-side twin of the - // upload paths' under-lock dir re-check). Bounded so a stalled HEAD - // cannot pin the advisory lock + its lock-pool connection. Both a - // timeout and an exists() error collapse to "not present" and take the - // discard arm: fail closed rather than publish unconfirmed state. - let archive_present = matches!( - tokio::time::timeout( - self.release_upload_timeout, - store.exists(owner_slug, repo_name), - ) - .await, - Ok(Ok(true)) - ); - if !archive_present { - warn!(repo = %repo_name, - "read download discarded — archive gone under lock (purged?)"); - guard.release().await; - return Ok(DownloadOutcome::Skipped); - } - - // Stale-swap guard (finding 2): a writer that locked, published a fresher - // copy, and released during our unlocked download window must not have - // its work clobbered by our now-stale temp. The only concurrent mutator - // of local_path is a writer or a purge (readers of the same repo - // serialize on the download mutex), so the signals below unambiguously - // mean "a writer published here." - let writer_published = match (local_existed_at_entry, entry_mtime) { - // acquire_fresh over an existing copy: presence is always true, so it - // cannot signal a writer; a CHANGED mtime can (the writer replaced - // the dir via rename, minting a fresh mtime). - (true, Some(entry_m)) => std::fs::metadata(local_path) - .and_then(|m| m.modified()) - .map(|now| now != entry_m) - .unwrap_or(false), - // Cold acquire, or acquire_fresh whose dir was absent at entry: the - // dir being present now means a writer published it during the window. - _ => local_path.exists(), - }; - if writer_published { - warn!(repo = %repo_name, - "read download discarded: a writer published a fresher copy during the download window"); - guard.release().await; - // temp guard's Drop removes the stale temp; serve the fresher local copy. - return Ok(DownloadOutcome::Published); - } - - let swapped = (|| -> std::io::Result<()> { - if local_path.exists() { - std::fs::remove_dir_all(local_path)?; - } - std::fs::rename(temp.path(), local_path) - })(); - guard.release().await; - match swapped { - Ok(()) => { - temp.disarm(); // the rename consumed the temp; nothing to clean up - Ok(DownloadOutcome::Published) - } - // temp guard's Drop removes the temp if the rename left it behind. - Err(e) => Err(e).context("publishing downloaded repo into place"), - } - } - /// Take a write lock (Postgres advisory lock), ensure repo is local, return guard. /// The lock prevents concurrent writes to the same repo across machines. pub async fn acquire_write(&self, owner_did: &str, repo_name: &str) -> Result { diff --git a/crates/gitlawb-node/src/git/repo_store/download.rs b/crates/gitlawb-node/src/git/repo_store/download.rs new file mode 100644 index 00000000..55028d54 --- /dev/null +++ b/crates/gitlawb-node/src/git/repo_store/download.rs @@ -0,0 +1,297 @@ +//! Read-path download coordination (KTD-3) for [`RepoStore`]. +//! +//! The read path's cache-miss handling lives here: the per-repo async mutex +//! that serializes concurrent readers of the same repo, the holder's fetch + +//! extract + under-lock publish, and the RAII temp-dir cleanup. `acquire` and +//! `acquire_fresh` stay in the parent module and call into `download_published`. +//! These methods reach the parent's private fields (`download_locks`) and +//! private methods (`try_lock_repo`) directly, because a child module can see +//! its parent's private items. + +use super::*; + +/// Outcome of a coordinated read-path download (KTD-3). +pub(super) enum DownloadOutcome { + /// The local dir is present: this reader downloaded and published it under + /// the advisory lock, or a concurrent reader did and this one served it. + Published, + /// The download was discarded without publishing: the advisory lock was + /// contended (a live writer or purge holds it), the lock pool errored, or + /// the archive vanished under the lock (purged mid-download). The caller + /// degrades to its missing-path or serve-local-copy outcome. + Skipped, +} + +/// RAII cleanup for a read-path download temp dir (finding 1). Its `Drop` +/// removes the dir (best effort), so a handler future cancelled mid-download +/// (axum drops it on client disconnect, the same hazard `RepoWriteGuard` / +/// `RepoLockGuard` Drop impls exist for) cannot leak a fully-populated repo +/// dir. `disarm` forgets the dir once the publish rename has consumed it. +struct TempDownloadDir { + path: PathBuf, + armed: bool, +} + +impl TempDownloadDir { + fn new(path: PathBuf) -> Self { + Self { path, armed: true } + } + + fn path(&self) -> &Path { + &self.path + } + + /// Disarm: the temp has been consumed (renamed into place), so `Drop` must + /// not try to remove it. + fn disarm(mut self) { + self.armed = false; + } +} + +impl Drop for TempDownloadDir { + fn drop(&mut self) { + if self.armed { + let _ = std::fs::remove_dir_all(&self.path); + } + } +} + +impl RepoStore { + /// Coordinated read-path download (KTD-3). Serializes concurrent readers + /// of the same repo on a per-repo async mutex: the first one in becomes + /// the holder and runs [`download_and_publish`](Self::download_and_publish); + /// a contended reader awaits the holder, re-checks the local dir on wake, + /// and serves what the holder published instead of re-downloading. Only + /// callers that already confirmed the archive exists reach this, and the + /// map entry is removed on completion, so the map cannot grow per + /// arbitrary requested name. + pub(super) async fn download_published( + &self, + owner_did: &str, + repo_name: &str, + owner_slug: &str, + local_path: &Path, + store: &Arc, + local_existed_at_entry: bool, + ) -> Result { + let map_key = format!("{owner_slug}/{repo_name}"); + let entry = { + let mut map = self.download_locks.lock().await; + Arc::clone( + map.entry(map_key.clone()) + .or_insert_with(|| Arc::new(Mutex::new(()))), + ) + }; + let held = match entry.try_lock() { + Ok(g) => g, + Err(_) => { + let g = entry.lock().await; + if local_path.exists() { + // A concurrent reader published while we waited — serve it. + drop(g); + self.remove_download_entry(&map_key, &entry).await; + return Ok(DownloadOutcome::Published); + } + // The holder degraded without publishing; this reader takes over. + // The prior holder removed the map entry on its way out, so + // re-register THIS entry before downloading, so readers arriving + // now coalesce onto it instead of starting a duplicate download + // (finding 3). Narrows the duplicate-download window; publishes + // stay serialized by the advisory lock regardless. + { + let mut map = self.download_locks.lock().await; + map.entry(map_key.clone()) + .or_insert_with(|| Arc::clone(&entry)); + } + g + } + }; + let outcome = self + .download_and_publish( + owner_did, + repo_name, + owner_slug, + local_path, + store, + local_existed_at_entry, + ) + .await; + self.remove_download_entry(&map_key, &entry).await; + drop(held); + outcome + } + + /// Remove a completed download-coordination entry, but only when the map + /// still holds THIS entry — a newer entry inserted after an earlier + /// removal must not be clobbered. + async fn remove_download_entry(&self, key: &str, entry: &Arc>) { + let mut map = self.download_locks.lock().await; + if map.get(key).is_some_and(|cur| Arc::ptr_eq(cur, entry)) { + map.remove(key); + } + } + + /// The holder's half of the coordinated download: fetch and extract the + /// archive into a temp sibling with NO advisory lock held (the network + /// phase must never pin a lock-pool connection — a cold-read burst across + /// distinct repos must not drain the writer-sized pool), then take the + /// per-repo advisory lock only around the publish: re-check the archive + /// still exists under the lock (a purge deletes it under this same lock, + /// so a post-purge downloader discards rather than resurrecting), swap + /// the temp copy into place, release. Advisory contention or a lock-pool + /// error discards the temp copy and degrades — no blocking lock or pool + /// wait exists anywhere on the read path. + /// + /// `local_existed_at_entry` selects the stale-swap guard (finding 2): a cold + /// `acquire` passes `false`, an `acquire_fresh` refresh passes `true`. Under + /// the lock, before the swap, we detect a writer that published a fresher + /// copy during our unlocked download window and serve theirs instead of + /// clobbering it with our now-stale temp. + async fn download_and_publish( + &self, + owner_did: &str, + repo_name: &str, + owner_slug: &str, + local_path: &Path, + store: &Arc, + local_existed_at_entry: bool, + ) -> Result { + let parent = local_path.parent().context("repo path has no parent")?; + std::fs::create_dir_all(parent).context("creating repo parent dir")?; + let file_name = local_path + .file_name() + .context("repo path has no file name")? + .to_string_lossy(); + + // Best-effort sweep of leftover temp siblings from a prior download whose + // RAII guard dropped, or whose extract spawn_blocking completed + // uninterruptibly AFTER that guard's Drop ran (finding 1). Serialized per + // repo by the download mutex, so no concurrent same-repo download owns a + // live temp here; scoped to THIS repo's prefix, so other repos are + // untouched. + let tmp_prefix = format!(".{file_name}.tmp-download."); + if let Ok(entries) = std::fs::read_dir(parent) { + for entry in entries.flatten() { + if entry.file_name().to_string_lossy().starts_with(&tmp_prefix) { + let _ = std::fs::remove_dir_all(entry.path()); + } + } + } + + // Stale-swap signal (finding 2): capture the local dir's mtime BEFORE the + // unlocked download. Under the lock, a changed mtime (acquire_fresh, + // refreshing over an existing copy) or the dir's mere appearance (cold + // acquire) means a writer published a fresher copy during our download + // window, so we serve theirs rather than swap our stale temp over it. + let entry_mtime = std::fs::metadata(local_path) + .and_then(|m| m.modified()) + .ok(); + + // Unique per-download temp target (same parent as local_path so the + // publish rename stays on one filesystem); mirrors the extract temp + // naming in tigris.rs. Wrapped in an RAII guard so a handler future + // cancelled mid-download (axum drops it on client disconnect) cannot + // leak the populated temp dir (the same hazard the guard Drop impls + // below exist for, finding 1). + let temp = TempDownloadDir::new(parent.join(format!( + ".{file_name}.tmp-download.{}", + uuid::Uuid::new_v4() + ))); + + // Bound the network download: a stalled GET would park this reader (and, + // via the per-repo download mutex, every coalesced reader) indefinitely. + // A timeout takes the SAME cleanup arm as a download error (return Err, + // the temp guard's Drop removes the dir), so `download_published` frees + // the map entry and wakes waiters rather than leaving them parked forever. + match tokio::time::timeout( + self.release_upload_timeout, + store.download(owner_slug, repo_name, temp.path()), + ) + .await + { + Ok(Ok(())) => {} + Ok(Err(e)) => return Err(e), + Err(_) => { + warn!(repo = %repo_name, timeout_secs = self.release_upload_timeout.as_secs(), + "read download timed out — discarding"); + return Err(anyhow::anyhow!("read download timed out")); + } + } + + let guard = match self.try_lock_repo(owner_did, repo_name).await { + Ok(Some(g)) => g, + Ok(None) => { + debug!(repo = %repo_name, + "read download discarded — repo lock held by a live writer or purge"); + return Ok(DownloadOutcome::Skipped); + } + Err(e) => { + warn!(repo = %repo_name, err = %e, + "read download discarded — could not acquire repo lock"); + return Ok(DownloadOutcome::Skipped); + } + }; + // Re-check under the lock: the archive gone here means a purge won the + // key mid-download — discard, never publish (the read-side twin of the + // upload paths' under-lock dir re-check). Bounded so a stalled HEAD + // cannot pin the advisory lock + its lock-pool connection. Both a + // timeout and an exists() error collapse to "not present" and take the + // discard arm: fail closed rather than publish unconfirmed state. + let archive_present = matches!( + tokio::time::timeout( + self.release_upload_timeout, + store.exists(owner_slug, repo_name), + ) + .await, + Ok(Ok(true)) + ); + if !archive_present { + warn!(repo = %repo_name, + "read download discarded — archive gone under lock (purged?)"); + guard.release().await; + return Ok(DownloadOutcome::Skipped); + } + + // Stale-swap guard (finding 2): a writer that locked, published a fresher + // copy, and released during our unlocked download window must not have + // its work clobbered by our now-stale temp. The only concurrent mutator + // of local_path is a writer or a purge (readers of the same repo + // serialize on the download mutex), so the signals below unambiguously + // mean "a writer published here." + let writer_published = match (local_existed_at_entry, entry_mtime) { + // acquire_fresh over an existing copy: presence is always true, so it + // cannot signal a writer; a CHANGED mtime can (the writer replaced + // the dir via rename, minting a fresh mtime). + (true, Some(entry_m)) => std::fs::metadata(local_path) + .and_then(|m| m.modified()) + .map(|now| now != entry_m) + .unwrap_or(false), + // Cold acquire, or acquire_fresh whose dir was absent at entry: the + // dir being present now means a writer published it during the window. + _ => local_path.exists(), + }; + if writer_published { + warn!(repo = %repo_name, + "read download discarded: a writer published a fresher copy during the download window"); + guard.release().await; + // temp guard's Drop removes the stale temp; serve the fresher local copy. + return Ok(DownloadOutcome::Published); + } + + let swapped = (|| -> std::io::Result<()> { + if local_path.exists() { + std::fs::remove_dir_all(local_path)?; + } + std::fs::rename(temp.path(), local_path) + })(); + guard.release().await; + match swapped { + Ok(()) => { + temp.disarm(); // the rename consumed the temp; nothing to clean up + Ok(DownloadOutcome::Published) + } + // temp guard's Drop removes the temp if the rename left it behind. + Err(e) => Err(e).context("publishing downloaded repo into place"), + } + } +} From 22ce83b95b65881c3d00ea45e1647b9d88925f65 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 07:47:18 -0500 Subject: [PATCH 50/61] fix(node): bound acquire_write's download and free the download-map entry on cancel (#196) Two gaps a second-model adversarial pass caught that the in-family review missed. acquire_write's under-lock exists()/download() ran with no timeout, so a stalled Tigris GET pinned the write lock and its lock-pool connection forever (the local-copy fallback only fires on an error, never a hang) - both are now wrapped in timeout(release_upload_timeout), a timeout mapping to the existing download-error fallback, which also makes the prior 'bound every under-lock call' claim true. The download_locks map entry was removed only on normal return, not on a cancelled cold read, so a disconnecting caller could grow the map across public repos; a DownloadEntryGuard now prunes it in Drop (outer map moved to std::sync::Mutex, held only for map ops, never across an await). RED observed on both new tests, including a real mid-download cancellation. --- crates/gitlawb-node/src/git/repo_store.rs | 167 ++++++++++++++++-- .../src/git/repo_store/download.rs | 60 +++++-- 2 files changed, 197 insertions(+), 30 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index dd8bdaa5..c18e1ab9 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -51,9 +51,14 @@ pub struct RepoStore { /// same repo serialize here so only one runs the network download while /// the rest await it and serve what the winner published. Entries are /// created only after a caller has confirmed the archive exists (reached - /// the download branch) and are removed when the holder finishes, so - /// permissionless requests for arbitrary names cannot grow the map. - download_locks: Arc>>>>, + /// the download branch) and are removed when the holder finishes (on every + /// exit, including handler cancellation, via `DownloadEntryGuard`), so + /// permissionless requests for arbitrary names cannot grow the map. The + /// OUTER map is a std::sync::Mutex (only ever held briefly for a get/insert/ + /// remove, never across an await), so the RAII entry guard's synchronous + /// `Drop` can prune it. The INNER per-repo `Arc>` stays a + /// tokio::sync::Mutex — it is the async serialization primitive readers await. + download_locks: Arc>>>>, } impl RepoStore { @@ -65,7 +70,7 @@ impl RepoStore { lock_pool, release_upload_timeout: DEFAULT_RELEASE_UPLOAD_TIMEOUT, migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), - download_locks: Arc::new(Mutex::new(HashMap::new())), + download_locks: Arc::new(std::sync::Mutex::new(HashMap::new())), } } @@ -80,7 +85,7 @@ impl RepoStore { lock_pool, release_upload_timeout: DEFAULT_RELEASE_UPLOAD_TIMEOUT, migrated: Arc::new(Mutex::new(HashSet::new())), - download_locks: Arc::new(Mutex::new(HashMap::new())), + download_locks: Arc::new(std::sync::Mutex::new(HashMap::new())), } } @@ -298,20 +303,45 @@ impl RepoStore { // Always download the latest from the object store before writing. // Local disk may be stale if another machine pushed since our last access. + // + // Both the exists() re-check and the download() are bounded by + // release_upload_timeout (INV-22): they run while the write advisory lock + // is held on a pinned lock-pool connection, so a stalled (never-erroring) + // GET would pin the lock and its connection indefinitely, and the local- + // copy fallback below (which only fires on Err) would never trigger on a + // hang. On timeout, exists() collapses to "not present" (skip the + // download, matching download_and_publish's bounded exists re-check), and + // download() takes the SAME arm as a download error so a hang bails + // instead of hanging. The guard was wrapped before this await (KTD-2), so + // every bail here still frees the lock via the guard's Drop. if let Some(ref store) = self.object_store { - if store - .exists(&guard.owner_slug, &guard.repo_name) - .await - .unwrap_or(false) - { + let archive_present = matches!( + tokio::time::timeout( + self.release_upload_timeout, + store.exists(&guard.owner_slug, &guard.repo_name), + ) + .await, + Ok(Ok(true)) + ); + if archive_present { debug!(repo = %guard.repo_name, "write acquire: downloading latest from object store"); - if let Err(e) = store - .download(&guard.owner_slug, &guard.repo_name, &guard.local_path) - .await + let download_result = match tokio::time::timeout( + self.release_upload_timeout, + store.download(&guard.owner_slug, &guard.repo_name, &guard.local_path), + ) + .await { + Ok(res) => res, + Err(_) => Err(anyhow::anyhow!( + "object-store download timed out after {}s under the write lock", + self.release_upload_timeout.as_secs() + )), + }; + if let Err(e) = download_result { // Same self-healing fallback as acquire_fresh: a corrupt/unreadable - // archive must not block a write when a valid local copy - // exists — release(success) will re-upload a good archive. + // archive OR a stalled-then-timed-out GET must not block a write + // when a valid local copy exists — release(success) will re-upload + // a good archive. if guard.local_path.exists() { warn!(repo = %guard.repo_name, err = %e, "write acquire: object-store download failed — falling back to local copy"); @@ -1899,6 +1929,61 @@ mod tests { ); } + // FINDING 3 / R2 / INV-22: acquire_write's under-lock freshness DOWNLOAD must + // be bounded, not just release()'s upload. acquire_write holds the write + // advisory lock on a pinned lock-pool connection, then does store.exists() + + // store.download() before returning the guard. A stalled (never-erroring) GET + // would pin that lock and its connection forever, and the local-copy fallback + // (which only fires on Err) never triggers on a hang. With a short bound and a + // download that never completes, acquire_write returns within the bound and + // the lock is freed. Pre-fix (untimed download) acquire_write hangs on the + // stalled GET and never returns -> the outer 5s timeout fires -> RED. + #[sqlx::test] + async fn acquire_write_download_stall_is_bounded( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + // The download parks forever; the acquire bound is what must unblock it. + ts.download_gate = Some(std::sync::Arc::new(tokio::sync::Notify::new())); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ) + .with_release_upload_timeout(std::time::Duration::from_millis(300)); + let owner = "did:key:z6MkAcquireWriteStallAAAAAAAAAAAAAAAAAAAAAA"; + let name = "acqwritestall"; + let key = lock_key_for(owner, name); + let mut observer = connect_opts.connect().await.unwrap(); + + // No local copy exists, so the timed-out download has nothing to fall back + // to and bails; the guard drops and its Drop frees the advisory lock. The + // call must RETURN within the bound rather than hang on the stalled GET. + let start = std::time::Instant::now(); + let res = tokio::time::timeout( + std::time::Duration::from_secs(5), + store.acquire_write(owner, name), + ) + .await + .expect("acquire_write must return within the download bound, not hang on the stalled GET"); + assert!( + res.is_err(), + "with no local copy to fall back to, a timed-out download must bail" + ); + assert!( + start.elapsed() < std::time::Duration::from_secs(5), + "acquire_write must return within the download bound" + ); + assert!( + poll_until_free(&mut observer, key).await, + "the advisory lock must be freed after the bounded (timed-out) download" + ); + } + // R2/INV-22: the OTHER under-lock upload path — upload_locked's PUT (init's // background upload, wait=true) — must be bounded exactly like release()'s. // With a stalled PUT, upload_locked acquires the advisory lock, parks on the @@ -2543,4 +2628,56 @@ mod tests { .execute(&mut observer) .await; } + + // ── Finding 4: the download-coordination map entry is freed on cancellation ─ + + // A cold-read holder parks mid-download, then its handler future is dropped + // (axum cancels it on client disconnect). The per-repo map entry must be + // removed anyway — the pre-fix code removed it only at explicit return points, + // which a cancelled future skips, so a permissionless caller could fan cold + // GETs across many public repos and disconnect to grow the map. The RAII + // DownloadEntryGuard's Drop closes that gap. This reads the private + // download_locks map directly (the test module is a child of RepoStore's + // module). Pre-fix: after abort the entry LEAKS -> still present -> RED. + #[sqlx::test] + async fn cancelled_cold_read_frees_the_map_entry(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkCancelMapEntryAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "cancelmapentry"; + let map_key = format!("{}/{name}", owner.replace([':', '/'], "_")); + + // Cache miss: the holder enters download_published, registers the map + // entry, and parks mid-download on the gate. + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the reader's download must be in flight" + ); + // Precondition: the map entry is registered while the holder runs. + assert!( + store.download_locks.lock().unwrap().contains_key(&map_key), + "the coordination entry must be registered while the holder downloads" + ); + + // Cancel the handler future mid-download (the client-disconnect hazard). + h.abort(); + + // The RAII guard's Drop must remove the entry despite the cancellation. + let freed = poll_until_true(|| store.download_locks.lock().unwrap().is_empty()).await; + assert!( + freed, + "a cold-read future cancelled mid-download must still free its map entry" + ); + } } diff --git a/crates/gitlawb-node/src/git/repo_store/download.rs b/crates/gitlawb-node/src/git/repo_store/download.rs index 55028d54..c0e2eabf 100644 --- a/crates/gitlawb-node/src/git/repo_store/download.rs +++ b/crates/gitlawb-node/src/git/repo_store/download.rs @@ -56,6 +56,35 @@ impl Drop for TempDownloadDir { } } +/// RAII removal of a download-coordination map entry (finding 4). Its `Drop` +/// removes the entry from `download_locks` whenever the map still holds THIS +/// Arc (ptr_eq-guarded so a newer entry inserted after an earlier removal is +/// never clobbered). Removing on every exit — normal return, `?`, or a handler +/// future cancelled mid cold-read (axum drops it on client disconnect) — is +/// what keeps the map from growing per arbitrary requested name: the pre-fix +/// explicit removals ran only at the labelled return points, which a cancelled +/// future skipped, leaking the entry. The outer map is a std::sync::Mutex so +/// this Drop can prune it synchronously (Drop cannot be async); it is only ever +/// held briefly for a get/insert/remove, never across an await. +struct DownloadEntryGuard { + locks: Arc>>>>, + key: String, + entry: Arc>, +} + +impl Drop for DownloadEntryGuard { + fn drop(&mut self) { + // A poisoned map is still safe to prune — recover the guard and remove. + let mut map = self.locks.lock().unwrap_or_else(|p| p.into_inner()); + if map + .get(&self.key) + .is_some_and(|cur| Arc::ptr_eq(cur, &self.entry)) + { + map.remove(&self.key); + } + } +} + impl RepoStore { /// Coordinated read-path download (KTD-3). Serializes concurrent readers /// of the same repo on a per-repo async mutex: the first one in becomes @@ -76,20 +105,30 @@ impl RepoStore { ) -> Result { let map_key = format!("{owner_slug}/{repo_name}"); let entry = { - let mut map = self.download_locks.lock().await; + let mut map = self.download_locks.lock().unwrap(); Arc::clone( map.entry(map_key.clone()) .or_insert_with(|| Arc::new(Mutex::new(()))), ) }; + // RAII: remove the map entry on EVERY exit of this function — normal + // return, `?`, or handler cancellation (finding 4). Its Drop is ptr_eq- + // guarded, so it never clobbers a newer entry, and it subsumes the + // pre-fix explicit removals (which a cancelled future skipped, leaking + // the entry). Declared here so it outlives `held` below and prunes the + // map only after the inner per-repo lock has been released. + let _entry_guard = DownloadEntryGuard { + locks: Arc::clone(&self.download_locks), + key: map_key.clone(), + entry: Arc::clone(&entry), + }; let held = match entry.try_lock() { Ok(g) => g, Err(_) => { let g = entry.lock().await; if local_path.exists() { // A concurrent reader published while we waited — serve it. - drop(g); - self.remove_download_entry(&map_key, &entry).await; + // `_entry_guard`'s Drop removes the map entry on return. return Ok(DownloadOutcome::Published); } // The holder degraded without publishing; this reader takes over. @@ -99,7 +138,7 @@ impl RepoStore { // (finding 3). Narrows the duplicate-download window; publishes // stay serialized by the advisory lock regardless. { - let mut map = self.download_locks.lock().await; + let mut map = self.download_locks.lock().unwrap(); map.entry(map_key.clone()) .or_insert_with(|| Arc::clone(&entry)); } @@ -116,19 +155,10 @@ impl RepoStore { local_existed_at_entry, ) .await; - self.remove_download_entry(&map_key, &entry).await; drop(held); outcome - } - - /// Remove a completed download-coordination entry, but only when the map - /// still holds THIS entry — a newer entry inserted after an earlier - /// removal must not be clobbered. - async fn remove_download_entry(&self, key: &str, entry: &Arc>) { - let mut map = self.download_locks.lock().await; - if map.get(key).is_some_and(|cur| Arc::ptr_eq(cur, entry)) { - map.remove(key); - } + // `_entry_guard`'s Drop removes the map entry here (after `held` and on + // every early return above), including when this future is cancelled. } /// The holder's half of the coordinated download: fetch and extract the From 8d2fec18b5701ba7cb635a3d91ef1436c10c78d7 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 07:47:18 -0500 Subject: [PATCH 51/61] fix(node): roll back the fork mirror and archive on late clone/insert failures (#196) The bounded-clone rework cleaned up the mirror on the timeout and spawn arms but not on a clone that creates the destination then exits nonzero, so a retry hit 'destination already exists' and could never complete that fork name. And because upload_under_guard is now required and runs before the row insert, a create_repo failure reliably orphaned the durable archive too, not just the local dir. Both failure arms now remove the mirror, delete the fork's archive (no-op on a store-less node), release the lock, and return. RED observed on both new tests (a non-empty-dest clone-128 repro and a decoy-row insert-failure repro). --- crates/gitlawb-node/src/api/repos.rs | 241 ++++++++++++++++++++++++++- 1 file changed, 240 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b0318c1c..6769dec9 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1753,6 +1753,16 @@ pub async fn fork_repo( if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); + // git clone --mirror can create the destination dir then exit non-zero + // (authz, corrupt/partial source), leaving a half-created mirror. Clear + // it best-effort and free the lock, matching the timeout/spawn arms, so + // a retry is not blocked by an existing dest and a same-name create sees + // an empty path. + if let Err(rm) = std::fs::remove_dir_all(&disk_path) { + tracing::warn!(fork = %fork_name, err = %rm, + "failed to remove fork mirror after a non-zero clone exit"); + } + repo_lock.release().await; return Err(AppError::Git(format!( "git clone --mirror failed: {stderr}" ))); @@ -1794,7 +1804,28 @@ pub async fn fork_repo( machine_id: state.machine_id.clone(), }; - state.db.create_repo(&record).await?; + // The mirror is cloned and (with a store configured) the durable archive is + // already uploaded, so a create_repo failure here would orphan BOTH with no + // repos row: a retry blocks on the existing dest, and a later same-key + // create could download the stale archive. Roll both back best-effort, free + // the lock, then fail the fork. delete_archive is a no-op on a store-less + // node, so this is safe regardless of configuration. + if let Err(e) = state.db.create_repo(&record).await { + if let Err(rm) = std::fs::remove_dir_all(&disk_path) { + tracing::warn!(fork = %fork_name, err = %rm, + "failed to remove fork mirror after a failed row insert"); + } + if let Err(del) = state + .repo_store + .delete_archive(&forker_did, &fork_name) + .await + { + tracing::warn!(fork = %fork_name, err = %del, + "failed to delete fork archive after a failed row insert"); + } + repo_lock.release().await; + return Err(e.into()); + } // Row, archive, and on-disk dir now all exist consistently; the race window // is closed, so the target lock can be released before the best-effort tail. @@ -4423,6 +4454,214 @@ mod tests { ); } + // Finding 1: a `git clone --mirror` that exits non-zero after the dest dir + // exists must clear that dest and free the lock, so a retry of the same fork + // name is not permanently blocked by a leftover half-mirror. Driven by + // pre-seeding the dest so the clone fails "destination already exists" + // (exit 128, the `!output.status.success()` arm, not the timeout/spawn + // arms). RED on base: the arm returns without removing the dest, so the dest + // survives and the second fork's clone fails the same way (5xx forever). + #[sqlx::test] + async fn fork_nonzero_clone_exit_cleans_dest_and_allows_retry(pool: sqlx::PgPool) { + let repos_dir = tempfile::TempDir::new().unwrap(); + let repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + let state = fork_test_state(repos_dir.path(), repo_store, pool).await; + + let forker = "did:key:z6MkForkCloneNonzeroAAAAAAAAAAAAAAAAAAAAAA"; + let fork_name = "nonzerofork"; + + // Seed the dest as a non-empty dir so `git clone --mirror` refuses it and + // exits 128, standing in for a half-created mirror a prior failed clone + // (authz, corrupt source) would leave behind. + let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name); + std::fs::create_dir_all(&clone_dir).unwrap(); + std::fs::write(clone_dir.join("stale.txt"), b"half-created mirror").unwrap(); + + let resp = call_fork(&state, forker, fork_name).await; + let status = match resp { + Ok(r) => r.status(), + Err(e) => { + use axum::response::IntoResponse; + e.into_response().status() + } + }; + assert!( + status.is_server_error(), + "a non-zero clone exit must fail the fork (5xx); got {status}" + ); + assert!( + !clone_dir.exists(), + "the leftover dest must be removed after a non-zero clone exit \ + (RED on base: the !success arm leaves it, blocking every retry)" + ); + + // The dest is clean and the lock freed, so re-forking the same name now + // clones fresh and succeeds. + let resp2 = call_fork(&state, forker, fork_name) + .await + .expect("a retry after the dest is cleaned must succeed"); + assert_eq!(resp2.status(), StatusCode::CREATED); + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_some(), + "repos row present after the successful retry" + ); + assert!(clone_dir.exists(), "cloned mirror present after the retry"); + } + + // Finding 2: when the row insert fails AFTER the durable upload landed, the + // fork must roll back BOTH the on-disk mirror and the just-uploaded archive, + // so no orphaned dest blocks a retry and no stale archive can be downloaded + // into a later same-key repo. The insert failure is injected deterministically + // via the `disk_path UNIQUE` constraint: a decoy row occupying the fork's + // exact disk_path passes the owner+name conflict guard (different owner/name) + // yet collides on insert, which happens only after upload_under_guard. RED on + // base: the `?` short-circuits, leaving the mirror dir and the archive behind. + #[sqlx::test] + async fn fork_row_insert_failure_rolls_back_mirror_and_archive(pool: sqlx::PgPool) { + use axum::response::IntoResponse; + use std::path::Path as StdPath; + use std::sync::{Arc, Mutex}; + + // Store double that uploads OK and records/removes the archive key so the + // test can assert the rollback deleted it. exists() is false so acquire + // never downloads over the local source. + type Archives = Arc>>; + struct TrackingStore { + archives: Archives, + } + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for TrackingStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(false) + } + async fn upload(&self, o: &str, r: &str, _p: &StdPath) -> anyhow::Result<()> { + self.archives + .lock() + .unwrap() + .insert((o.to_string(), r.to_string())); + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn delete(&self, o: &str, r: &str) -> anyhow::Result<()> { + self.archives + .lock() + .unwrap() + .remove(&(o.to_string(), r.to_string())); + Ok(()) + } + } + + let archives: Archives = Arc::new(Mutex::new(std::collections::HashSet::new())); + let repos_dir = tempfile::TempDir::new().unwrap(); + let repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(Arc::new(TrackingStore { + archives: archives.clone(), + })), + pool.clone(), + ); + let state = fork_test_state(repos_dir.path(), repo_store, pool).await; + + let forker = "did:key:z6MkForkInsertFailAAAAAAAAAAAAAAAAAAAAAAAAA"; + let fork_name = "insertfailfork"; + // The store keys on the slugified owner DID (`:`/`/` -> `_`), the same + // transform local_path applies. Track the FORK's archive specifically: + // the source repo is independently (re)uploaded via the read acquire, so + // the set is not empty even when the fork's archive is correctly rolled + // back. + let fork_archive = (forker.replace([':', '/'], "_"), fork_name.to_string()); + + // Pre-insert a decoy row that occupies the fork's exact disk_path but has + // a different owner/name, so the fork's own insert violates disk_path's + // UNIQUE constraint. get_repo(forker, fork_name) still returns None, so the + // handler proceeds past its conflict guard, clones, and uploads first. + let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name); + let now = Utc::now(); + let decoy = crate::db::RepoRecord { + id: Uuid::new_v4().to_string(), + name: "decoyname".to_string(), + owner_did: "did:key:z6MkDecoyOwnerBBBBBBBBBBBBBBBBBBBBBBBBBBBB".to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: clone_dir.to_string_lossy().to_string(), + forked_from: None, + machine_id: None, + }; + state.db.create_repo(&decoy).await.unwrap(); + + let resp = call_fork(&state, forker, fork_name).await; + let status = match resp { + Ok(r) => r.status(), + Err(e) => e.into_response().status(), + }; + assert!( + status.is_server_error(), + "a row-insert failure must fail the fork (5xx); got {status}" + ); + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_none(), + "no fork row may exist after the insert failure" + ); + assert!( + !clone_dir.exists(), + "the cloned mirror must be removed after the failed insert \ + (RED on base: the `?` short-circuits and leaves it)" + ); + assert!( + !archives.lock().unwrap().contains(&fork_archive), + "the uploaded fork archive must be deleted after the failed insert \ + (RED on base: the archive is orphaned)" + ); + + // The target lock must be free again for a retry. + let probe = state + .repo_store + .try_lock_repo(forker, fork_name) + .await + .unwrap() + .expect("target lock freed after the rolled-back fork"); + probe.release().await; + + // Remove the decoy so the retry's insert can land, then re-fork the same + // name: dest is clean, lock is free, and the archive is re-uploaded. + state.db.delete_repo_by_id(&decoy.id).await.unwrap(); + let resp2 = call_fork(&state, forker, fork_name) + .await + .expect("a retry after the rollback must succeed"); + assert_eq!(resp2.status(), StatusCode::CREATED); + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_some(), + "repos row present after the successful retry" + ); + assert!( + archives.lock().unwrap().contains(&fork_archive), + "the retry re-uploads the fork archive" + ); + } + // create_repo must NOT take its serialization lock from the APP pool. Doing so // pins an app-pool connection in the lock guard while the create's own work // (get_repo / proof.consume / db.create_repo, all on state.db = the app pool) From 3c07adcab6e01741caf7d399a63bedc62a14465b Mon Sep 17 00:00:00 2001 From: t Date: Wed, 22 Jul 2026 12:18:12 -0500 Subject: [PATCH 52/61] fix(node): serialize repo purge cascade with mirror ingest via advisory lock delete_repo_by_id ran a point-in-time sibling check then cascaded slug-scoped deletes; upsert_mirror_repo (peer sync) did not serialize against that window, so a colliding mirror could land mid-purge and get half-wiped. Both now take a transaction-scoped pg_advisory_xact_lock keyed on the normalized slug. (F1, #196) --- crates/gitlawb-node/src/db/mod.rs | 187 +++++++++++++++++++++++++++++- 1 file changed, 182 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index dcde7c29..77d42347 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -928,6 +928,23 @@ fn owner_key_case_sql(col: &str) -> String { ) } +/// Take a transaction-scoped Postgres advisory lock keyed on the normalized repo +/// slug (`normalize_owner_key(owner)/name`). `delete_repo_by_id`'s purge cascade and +/// `upsert_mirror_repo`'s ingest both take this lock on the identical slug string, so +/// a peer-sync mirror insert cannot interleave between the purge's point-in-time +/// sibling check and its slug-scoped deletes (which would half-wipe a live mirror's +/// child rows / bounty). `hashtextextended($1, 0)` folds the slug to the `bigint` the +/// lock API needs; both call sites hash the identical string, so their keys agree. +/// XACT-scoped, so it auto-releases on commit/rollback with no unlock bookkeeping +/// (mirrors the session-scoped `pg_advisory_lock` migration guard, but self-releasing). +async fn lock_repo_slug(conn: &mut sqlx::PgConnection, slug: &str) -> Result<()> { + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") + .bind(slug) + .execute(conn) + .await?; + Ok(()) +} + #[cfg(test)] mod normalize_owner_key_tests { use super::normalize_owner_key; @@ -1030,6 +1047,14 @@ impl Db { ) -> Result<()> { let now = Utc::now().to_rfc3339(); let id = format!("{owner_short}/{name}"); + // Slug the purge cascade keys on. Take the shared advisory lock on it inside a + // tx so a concurrent `delete_repo_by_id` purge of a colliding row cannot + // interleave its sibling-check-then-cascade around this insert and wipe the + // mirror we are ingesting. Normalize the owner the same way the cascade does so + // both call sites hash the identical slug string. + let slug = format!("{}/{}", normalize_owner_key(owner_short), name); + let mut tx = self.pool.begin().await?; + lock_repo_slug(&mut *tx, &slug).await?; // `quarantined` is set only on first insert (the admission decision). // A re-sync (ON CONFLICT) preserves the existing flag — admission runs // once, and an operator's later release must not be silently reverted. @@ -1050,8 +1075,9 @@ impl Db { .bind(disk_path) .bind(machine_id) .bind(quarantined) - .execute(&self.pool) + .execute(&mut *tx) .await?; + tx.commit().await?; Ok(()) } @@ -1342,6 +1368,13 @@ impl Db { }; let slug = format!("{}/{}", normalize_owner_key(&owner_did), name); + // Serialize this purge against a concurrent `upsert_mirror_repo` ingest that + // resolves to the same slug: both take this xact-scoped advisory lock on the + // identical slug string BEFORE the sibling check below, so a peer-sync insert + // cannot land between that point-in-time check and the slug-scoped cascade + // deletes (which would half-wipe the live mirror). Auto-released on commit. + lock_repo_slug(&mut *tx, &slug).await?; + // Grandchildren first: PR reviews/comments key on pr_id, so delete them // before the parent PRs vanish. for table in ["pr_reviews", "pr_comments"] { @@ -1381,10 +1414,12 @@ impl Db { // a surviving sibling's records. Gate the slug-scoped deletes on the absence // of any OTHER repos row resolving to the same slug, using the same // OWNER_KEY_CASE_SQL the slug was built from so the check is byte-identical to - // the cascade key. (Residual: this is a point-in-time check inside one READ - // COMMITTED tx, not full serialization against a concurrent colliding create; - // see the plan's KTD4. Purge is the sole caller, so the window is - // operator-scoped.) + // the cascade key. The xact advisory lock taken at the top of this tx (and + // matched in `upsert_mirror_repo` on the identical slug) serializes this + // sibling-check-then-cascade against a concurrent peer-sync insert for the same + // slug, so this is no longer a bare point-in-time read a mirror upsert can slip + // past (closing the old KTD4 residual: the concurrent inserter is peer sync, + // not an operator, so the window was never operator-scoped). let sibling_exists: bool = sqlx::query_scalar(&format!( "SELECT EXISTS(SELECT 1 FROM repos WHERE id <> $1 AND {key} || '/' || name = $2)", key = OWNER_KEY_CASE_SQL @@ -3951,6 +3986,148 @@ mod dedup_db_tests { ); } + /// U1 (F1): the purge cascade must be serialized with mirror ingestion on the + /// shared slug. `delete_repo_by_id` runs a point-in-time sibling check and then, + /// only if no sibling was seen, wipes the slug-scoped child rows / tombstones the + /// bounty. A concurrent `upsert_mirror_repo` that inserts a colliding mirror row + /// AFTER that check but before/around the cascade would leave a half-wipe: the + /// mirror's `repos` row survives while its live child rows / bounty are gone. + /// + /// Deterministic race: hold a session advisory lock on the same `hashtextextended` + /// slug key the cascade locks on, so the spawned purge BLOCKS at the top of its tx + /// (before the sibling check). While it is parked, land the mirror's `repos` row + /// (what ingestion writes), then release the lock. With the lock in place the purge + /// resumes, its sibling check now SEES the mirror row, and it skips the cascade — + /// both the mirror row and its child rows survive (GREEN). Remove the + /// `lock_repo_slug` call from `delete_repo_by_id` and the purge no longer blocks: it + /// runs its sibling check before the mirror row exists, cascades, and half-wipes the + /// child rows / bounty while the mirror row (inserted afterwards) survives (RED). + #[sqlx::test] + async fn purge_serializes_with_mirror_ingest_via_advisory_lock(pool: PgPool) { + let db = db(pool).await; + // Purge target: an empty canonical `did:key:` row. + let owner = "did:key:z6MkPurgeRaceMirrorFixtureForLock"; + let target = rec( + "rid-purge-race-target", + owner, + "victim", + "empty canonical", + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:00Z", + ); + db.create_repo(&target).await.unwrap(); + + // The mirror's slug-scoped live state, keyed on the shared slug. The mirror's + // `repos` row is deliberately NOT present yet — ingestion lands it mid-race. + let slug = format!( + "{}/victim", + crate::db::normalize_owner_key(owner) + ); + sqlx::query( + "INSERT INTO branch_cids (repo, ref_name, sha, cid, node_did, updated_at) + VALUES ($1, 'refs/heads/main', '1', 'cid', 'n', '2026-02-01T00:00:00Z')", + ) + .bind(&slug) + .execute(db.pool()) + .await + .unwrap(); + db.create_bounty(&BountyRecord { + id: "bnt-purge-race".to_string(), + repo_owner: "z6MkPurgeRaceMirrorFixtureForLock".to_string(), + repo_name: "victim".to_string(), + issue_id: None, + title: "mirror bounty".to_string(), + amount: 5_000, + creator_did: "did:key:z6MkCreator".to_string(), + claimant_did: None, + claimant_wallet: None, + pr_id: None, + status: "open".to_string(), + created_at: "2026-02-01T00:00:00Z".to_string(), + claimed_at: None, + submitted_at: None, + completed_at: None, + deadline_secs: 604_800, + tx_hash: None, + }) + .await + .unwrap(); + + // Hold a session-scoped advisory lock on the exact slug key the cascade takes. + // A session lock and the cascade's xact lock share Postgres' advisory lock + // space, so the purge's `pg_advisory_xact_lock` blocks on this until we unlock. + let mut lock_conn = db.pool().acquire().await.unwrap(); + sqlx::query("SELECT pg_advisory_lock(hashtextextended($1, 0))") + .bind(&slug) + .execute(&mut *lock_conn) + .await + .unwrap(); + + // Spawn the purge; with the lock present it parks at the top of its tx. + let pool_for_task = db.pool().clone(); + let purge = tokio::spawn(async move { + Db::for_testing(pool_for_task) + .delete_repo_by_id("rid-purge-race-target") + .await + }); + + // Give the purge time to reach the lock wait (GREEN) or, without the lock, to + // run its whole sibling-check-then-cascade (RED) before the mirror row lands. + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + + // Mirror ingestion lands the colliding `repos` row (slug-form id, bare owner) — + // exactly what upsert_mirror_repo writes. Raw insert here so it does not contend + // on the advisory lock the test connection is holding. + sqlx::query( + "INSERT INTO repos (id, name, owner_did, description, is_public, default_branch, + created_at, updated_at, disk_path, machine_id, quarantined) + VALUES ($1, 'victim', 'z6MkPurgeRaceMirrorFixtureForLock', 'mirrored from peer', + true, 'main', '2026-02-01T00:00:00Z', '2026-02-01T00:00:00Z', + '/srv/mirror', NULL, false)", + ) + .bind(&slug) + .execute(db.pool()) + .await + .unwrap(); + + // Release the lock so the purge can proceed (and observe the mirror row). + sqlx::query("SELECT pg_advisory_unlock(hashtextextended($1, 0))") + .bind(&slug) + .execute(&mut *lock_conn) + .await + .unwrap(); + + let removed = purge.await.unwrap().unwrap(); + assert_eq!(removed, 1, "the empty canonical target row is purged"); + + async fn count(db: &Db, sql: &str, arg: &str) -> i64 { + sqlx::query_scalar::<_, i64>(sql) + .bind(arg) + .fetch_one(db.pool()) + .await + .unwrap() + } + // No half-wipe: the mirror row AND its slug-scoped state are all present. + assert_eq!( + count(&db, "SELECT COUNT(*) FROM repos WHERE id=$1", &slug).await, + 1, + "the mirror repos row survives the purge" + ); + assert!( + count(&db, "SELECT COUNT(*) FROM branch_cids WHERE repo=$1", &slug).await > 0, + "mirror branch_cids must survive: the purge serialized behind the ingest and saw the sibling" + ); + assert_eq!( + db.get_bounty("bnt-purge-race") + .await + .unwrap() + .unwrap() + .status, + "open", + "the mirror's bounty must stay open, not be tombstoned by the racing purge" + ); + } + /// U8 (F): bounties key on (repo_owner, repo_name), not an immutable repo id, /// and `delete_repo_by_id` deliberately does not delete them (financial record). /// Without tombstoning, a purged repo's OPEN bounty stays claimable and silently From 9631acb2836f7f45c4b7d8b535bff9b31c98b6db Mon Sep 17 00:00:00 2001 From: t Date: Wed, 22 Jul 2026 12:29:44 -0500 Subject: [PATCH 53/61] fix(node): roll back local ref on failed durable upload (close/merge/receive-pack) close_issue, merge_pr, and the receive-pack task applied a local ref write then released with an empty failure cleanup, so a failed Tigris upload left stale state that the local-fast-path read served despite the client getting a 5xx. Each now captures the prior ref and restores it via release_with_failure_cleanup when the upload fails, matching create_issue. (F2, #196) --- crates/gitlawb-node/src/api/issues.rs | 148 +++++++++++++++++++++++++- crates/gitlawb-node/src/api/pulls.rs | 23 +++- crates/gitlawb-node/src/api/repos.rs | 21 +++- crates/gitlawb-node/src/git/issues.rs | 43 ++++++++ crates/gitlawb-node/src/git/store.rs | 80 ++++++++++++++ 5 files changed, 310 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index a0850744..750aaa4b 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -293,13 +293,36 @@ pub async fn close_issue( )); } + // Snapshot the issue ref's pre-close OID so a failed durable upload can roll + // the close back on local disk, mirroring create_issue. Best-effort: if the + // snapshot read fails we proceed without a rollback closure (the close still + // applies and a failed upload still surfaces as a 5xx below). + let prior_ref = git_issues::issue_ref_oid(&disk_path, &issue_id) + .ok() + .flatten(); + let close_result = git_issues::close_issue(&disk_path, &issue_id); // Always release the advisory lock, even on error; upload to Tigris only on // success. A false return means the durable upload failed: the close applied // to local disk but never reached object storage, so a later acquire_write // reverts it. Report 5xx so the client retries. (P1 data-loss guard.) - let upload_ok = guard.release(close_result.is_ok()).await; + // On that failure, roll the local close BACK to the captured open ref BEFORE + // the lock releases — otherwise the local fast path in RepoStore::acquire + // serves the "closed" state until a later write refreshes the archive, even + // though the client got a 5xx. The cleanup runs only when an upload was + // attempted and failed, while the advisory lock is still held. + let upload_ok = guard + .release_with_failure_cleanup(close_result.is_ok(), |path| { + if let Some((full_id, oid)) = &prior_ref { + if let Err(e) = git_issues::restore_issue_ref(path, full_id, oid) { + tracing::warn!(issue = %issue_id, err = %e, + "failed to roll back local issue close after failed durable upload; \ + a local-fast-path read may serve a stale closed state"); + } + } + }) + .await; let updated = close_result .map_err(|e| AppError::Git(e.to_string()))? @@ -433,6 +456,129 @@ mod tests { } } + async fn close_issue_status( + state: &crate::state::AppState, + owner: &str, + name: &str, + issue_id: &str, + ) -> axum::http::StatusCode { + use axum::response::IntoResponse; + let resp = super::close_issue( + State(state.clone()), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + AxPath((owner.to_string(), name.to_string(), issue_id.to_string())), + ) + .await; + match resp { + Ok(_) => axum::http::StatusCode::OK, + Err(e) => e.into_response().status(), + } + } + + // Seed an OPEN issue directly on local disk, bypassing the (stalling) upload + // path so the close test starts from a durably-consistent open issue. + fn seed_open_issue(bare: &StdPath, owner: &str, issue_id: &str) { + let issue = IssueRecord { + id: issue_id.to_string(), + title: "t".into(), + body: None, + author: Some(owner.to_string()), + created_at: Utc::now().to_rfc3339(), + status: "open".to_string(), + signed_payload: None, + }; + let json = serde_json::to_string(&issue).unwrap(); + git_issues::create_issue(bare, issue_id, &json).unwrap(); + } + + fn issue_status(bare: &StdPath, issue_id: &str) -> String { + let raw = git_issues::get_issue(bare, issue_id).unwrap().unwrap(); + let v: serde_json::Value = serde_json::from_str(&raw).unwrap(); + v["status"].as_str().unwrap().to_string() + } + + // U2 [F2, P1] — the failed-upload rollback on the close path. close_issue + // applies the "closed" ref to LOCAL disk, then release() uploads to durable + // storage. When that upload times out the local close is NOT persisted, so a + // later acquire_write reverts it from the stale archive — but until then, a + // local-fast-path read via RepoStore::acquire serves the stale "closed" + // state even though the client got a 5xx. The fix rolls the close back to the + // captured open ref while the advisory lock is held. RED with the cleanup + // closure reverted to `|_| {}`: the read sees "closed". + #[sqlx::test] + async fn issue_close_failed_upload_rolls_back_to_open(pool: sqlx::PgPool) { + let repos_dir = tempfile::TempDir::new().unwrap(); + let owner = "z6issuecloserollbackowner"; + let name = "issuecloserollback"; + let (state, bare) = stall_state_with_repo(pool, &repos_dir, owner, name).await; + + let issue_id = "aaaa1111-0000-0000-0000-000000000000"; + seed_open_issue(&bare, owner, issue_id); + assert_eq!(issue_status(&bare, issue_id), "open", "seed must be open"); + + // Close via the handler — the durable upload stalls and times out (5xx). + let status = close_issue_status(&state, owner, name, issue_id).await; + assert!( + status.is_server_error(), + "a stalled durable upload on issue-close must 500, got {status}" + ); + + // The local fast path must still see the issue OPEN: the failed upload + // rolled the close back while the lock was held. + assert_eq!( + issue_status(&bare, issue_id), + "open", + "a failed durable upload must roll the local close back to open; a \ + local-fast-path read served the stale closed state" + ); + } + + // Success path: a working store closes the issue (2xx) and the read reflects + // the close. Confirms the rollback wiring does not fire on success. + #[sqlx::test] + async fn issue_close_success_persists_close(pool: sqlx::PgPool) { + let repos_dir = tempfile::TempDir::new().unwrap(); + let owner = "z6issueclosesuccessowner"; + let name = "issueclosesuccess"; + let (mut state, bare) = stall_state_with_repo(pool.clone(), &repos_dir, owner, name).await; + // Swap the stalling store for one whose upload succeeds immediately. + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(OkStore)), + pool.clone(), + ) + .with_release_upload_timeout(Duration::from_millis(200)); + + let issue_id = "bbbb2222-0000-0000-0000-000000000000"; + seed_open_issue(&bare, owner, issue_id); + + let status = close_issue_status(&state, owner, name, issue_id).await; + assert_eq!(status, StatusCode::OK, "working store must close (200)"); + assert_eq!( + issue_status(&bare, issue_id), + "closed", + "a successful close must persist as closed" + ); + } + + // Pre-write error path: closing a nonexistent issue returns 404 and never + // attempts an upload (release(false)), so the stalling store cannot hang it. + #[sqlx::test] + async fn issue_close_not_found_releases_without_upload(pool: sqlx::PgPool) { + let repos_dir = tempfile::TempDir::new().unwrap(); + let owner = "z6issueclosemissingowner"; + let name = "issueclosemissing"; + let (state, _bare) = stall_state_with_repo(pool, &repos_dir, owner, name).await; + + let status = + close_issue_status(&state, owner, name, "cccc3333-0000-0000-0000-000000000000").await; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "closing a missing issue must 404 without attempting an upload" + ); + } + fn issue_refs(bare: &StdPath) -> Vec { let out = std::process::Command::new("git") .args([ diff --git a/crates/gitlawb-node/src/api/pulls.rs b/crates/gitlawb-node/src/api/pulls.rs index cfe93d20..b2a39179 100644 --- a/crates/gitlawb-node/src/api/pulls.rs +++ b/crates/gitlawb-node/src/api/pulls.rs @@ -216,6 +216,14 @@ pub async fn merge_pr( .map_err(|e| AppError::Git(e.to_string()))?; let disk_path = guard.path().to_path_buf(); let merger_did = auth.0; + + // Snapshot the target branch's pre-merge tip so a failed durable upload can + // roll the merge commit back on local disk (mirrors create_issue). Without + // this the local fast path in RepoStore::acquire serves the merged tree until + // a later write refreshes the archive, even though the client got a 5xx. + let target_ref = format!("refs/heads/{}", pr.target_branch); + let prior_target = store::ref_oid(&disk_path, &target_ref).ok().flatten(); + let merge_result = store::merge_branch( &disk_path, &pr.target_branch, @@ -230,8 +238,19 @@ pub async fn merge_pr( // acquire_write reverts it from the stale archive. Report 5xx so the client // retries rather than marking the PR merged in the DB while the merged tree // was silently lost. Fail BEFORE merge_pr / touch_repo / webhooks. (P1 - // data-loss guard, same as receive-pack.) - let upload_ok = guard.release(merge_result.is_ok()).await; + // data-loss guard, same as receive-pack.) On that failure, roll the local + // merge back to the captured tip BEFORE the lock releases. + let upload_ok = guard + .release_with_failure_cleanup(merge_result.is_ok(), |path| { + if let Some(oid) = &prior_target { + if let Err(e) = store::set_ref(path, &target_ref, oid) { + tracing::warn!(repo = %record.name, err = %e, + "failed to roll back local merge commit after failed durable upload; \ + a local-fast-path read may serve the un-uploaded merged tree"); + } + } + }) + .await; let merge_sha = merge_result.map_err(|e| AppError::Git(e.to_string()))?; diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 6769dec9..bc7c5502 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -999,13 +999,30 @@ pub async fn git_receive_pack( } }; let disk_path = guard.path().to_path_buf(); + + // Snapshot the pre-push refs so a failed durable upload can roll the + // applied pack back on local disk (mirrors create_issue). Without this + // the local fast path in RepoStore::acquire serves the pushed refs until + // a later write refreshes the archive, even though the client got a 5xx. + let pre_push_refs = store::list_refs(&disk_path).unwrap_or_default(); + tracing::debug!(repo = %record.name, path = %disk_path.display(), "running git receive-pack"); let result = smart_http::receive_pack(&disk_path, body, git_timeout).await; // Always release the advisory lock — even on error, and BEFORE the tail — // to prevent stale locks and to avoid holding the write lock across the // fan-out. Only upload to Tigris when the push succeeded; uploading a - // half-applied repo would propagate corruption. - let upload_ok = guard.release(result.is_ok()).await; + // half-applied repo would propagate corruption. On a failed durable + // upload, restore the pre-push refs BEFORE the lock releases so a + // local-fast-path read does not serve refs that never landed durably. + let upload_ok = guard + .release_with_failure_cleanup(result.is_ok(), |path| { + if let Err(e) = store::restore_refs(path, &pre_push_refs) { + tracing::warn!(repo = %record.name, err = %e, + "failed to roll back receive-pack refs after failed durable upload; \ + a local-fast-path read may serve un-uploaded refs"); + } + }) + .await; // On receive-pack error, deliver the classified error to the client and run // NO tail. On success, hand report-status to the client now, then continue diff --git a/crates/gitlawb-node/src/git/issues.rs b/crates/gitlawb-node/src/git/issues.rs index 1f507af6..31070e4f 100644 --- a/crates/gitlawb-node/src/git/issues.rs +++ b/crates/gitlawb-node/src/git/issues.rs @@ -80,6 +80,49 @@ pub fn delete_issue_ref(repo_path: &Path, issue_id: &str) -> Result<()> { Ok(()) } +/// Read the object id an issue ref currently points to (resolving an 8-char +/// prefix), returning `(full_id, oid)`. Returns None if no such issue exists. +/// Used to snapshot an issue ref's pre-mutation state so a failed durable +/// upload can restore it, mirroring `create_issue`'s rollback. +pub fn issue_ref_oid(repo_path: &Path, issue_id: &str) -> Result> { + let full_id = match resolve_issue_id(repo_path, issue_id)? { + Some(id) => id, + None => return Ok(None), + }; + let ref_name = format!("refs/gitlawb/issues/{full_id}"); + let output = Command::new("git") + .args(["rev-parse", "--verify", &ref_name]) + .current_dir(repo_path) + .output() + .context("failed to run git rev-parse")?; + if !output.status.success() { + return Ok(None); + } + let oid = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if oid.is_empty() { + return Ok(None); + } + Ok(Some((full_id, oid))) +} + +/// Reset an issue ref back to a captured object id — rolls back a local +/// mutation (e.g. a close) whose durable upload failed, so a local-fast-path +/// read does not serve the un-uploaded state. `full_id` must be the resolved +/// UUID (from [`issue_ref_oid`]). The now-dangling object is harmless. +pub fn restore_issue_ref(repo_path: &Path, full_id: &str, oid: &str) -> Result<()> { + let ref_name = format!("refs/gitlawb/issues/{full_id}"); + let output = Command::new("git") + .args(["update-ref", &ref_name, oid]) + .current_dir(repo_path) + .output() + .context("failed to run git update-ref")?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + anyhow::bail!("git update-ref failed: {stderr}"); + } + Ok(()) +} + /// List all issue refs and return their JSON content. pub fn list_issues(repo_path: &Path) -> Result> { // List all refs under refs/gitlawb/issues/ diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 229ee695..1deffbe6 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -63,6 +63,86 @@ pub fn list_refs(repo_path: &Path) -> Result> { Ok(refs) } +/// Read the object id a single ref currently points to. Returns None if the +/// ref does not exist. Used to snapshot a ref's pre-write state so a failed +/// durable upload can roll the local write back. +pub fn ref_oid(repo_path: &Path, ref_name: &str) -> Result> { + let output = Command::new("git") + .args(["rev-parse", "--verify", ref_name]) + .current_dir(repo_path) + .output() + .context("failed to run git rev-parse")?; + if !output.status.success() { + return Ok(None); + } + let oid = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if oid.is_empty() { + return Ok(None); + } + Ok(Some(oid)) +} + +/// Force a single ref back to a captured object id — rolls back a local write +/// (e.g. a merge commit) whose durable upload failed, so a local-fast-path +/// read does not serve the un-uploaded state. The now-dangling objects are +/// harmless. +pub fn set_ref(repo_path: &Path, ref_name: &str, oid: &str) -> Result<()> { + let output = Command::new("git") + .args(["update-ref", ref_name, oid]) + .current_dir(repo_path) + .output() + .context("failed to run git update-ref")?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + bail!("git update-ref failed: {stderr}"); + } + Ok(()) +} + +/// Restore a repo's refs to a previously captured snapshot: reset every +/// snapshot ref to its captured object id, and delete any ref that exists now +/// but was absent from the snapshot (created by the intervening write). Used +/// to roll back a receive-pack whose durable upload failed, so the next +/// local-fast-path read does not serve refs that never reached object storage. +/// Best-effort per ref: an individual failure is collected and reported, but +/// the remaining refs are still attempted. +pub fn restore_refs(repo_path: &Path, snapshot: &[(String, String)]) -> Result<()> { + use std::collections::HashSet; + + let mut errors: Vec = Vec::new(); + + // Reset every snapshot ref to its captured oid (recreates refs the write + // deleted, and rewinds refs the write advanced). + for (ref_name, oid) in snapshot { + if let Err(e) = set_ref(repo_path, ref_name, oid) { + errors.push(format!("{ref_name}: {e}")); + } + } + + // Delete refs that exist now but were not in the snapshot (created by the + // write being rolled back). + let snapshot_names: HashSet<&str> = snapshot.iter().map(|(r, _)| r.as_str()).collect(); + let current = list_refs(repo_path)?; + for (ref_name, _) in ¤t { + if !snapshot_names.contains(ref_name.as_str()) { + let output = Command::new("git") + .args(["update-ref", "-d", ref_name]) + .current_dir(repo_path) + .output() + .context("failed to run git update-ref -d")?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + errors.push(format!("{ref_name} (delete): {stderr}")); + } + } + } + + if !errors.is_empty() { + bail!("failed to restore {} ref(s): {}", errors.len(), errors.join("; ")); + } + Ok(()) +} + /// Read the current HEAD commit hash of a repository. /// Returns None if the repo is empty (no commits yet). pub fn head_commit(repo_path: &Path) -> Result> { From eea14cdda55d3935795eac09d8c318bdd3286109 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 22 Jul 2026 12:44:17 -0500 Subject: [PATCH 54/61] fix(node): tie cancelled cold-download cleanup to blocking extraction On a download timeout the TempDownloadDir guard dropped while the detached spawn_blocking extraction kept running and could rename its output back into the just-removed temp path; that dir was only reclaimed on a later same-repo request, so cold reads across distinct repos accumulated extracted repos on disk. The timeout arm now hands the owned download future to a detached janitor that awaits extraction completion before removing the temp dir, off the critical path so waiters are still freed immediately. (F3, #196) --- crates/gitlawb-node/src/git/repo_store.rs | 103 ++++++++++++++++++ .../src/git/repo_store/download.rs | 43 +++++++- 2 files changed, 140 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index c18e1ab9..121d364f 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -2680,4 +2680,107 @@ mod tests { "a cold-read future cancelled mid-download must still free its map entry" ); } + + // ── F3: a timed-out cold read must not resurrect its abandoned temp dir ── + + // An ObjectStore double that reproduces the detached-spawn_blocking + // resurrection. Its download() runs the extraction in spawn_blocking (as the + // real TigrisClient::download does) and awaits the join handle. A tokio + // timeout that fires while the blocking task runs does NOT cancel it: + // dropping the download future detaches the task, which then creates ("renames + // into place") the target path LATE — after the caller's temp-dir cleanup has + // already run. `extracted` is bumped once the blocking task has created the + // dir, so a test can wait for the extraction to actually land. + struct DetachExtractStore { + extract_delay: std::time::Duration, + downloads: std::sync::Arc, + extracted: std::sync::Arc, + } + + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for DetachExtractStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(true) + } + async fn upload(&self, _o: &str, _r: &str, _p: &std::path::Path) -> anyhow::Result<()> { + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, p: &std::path::Path) -> anyhow::Result<()> { + self.downloads + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let target = p.to_path_buf(); + let delay = self.extract_delay; + let extracted = self.extracted.clone(); + // Mirror TigrisClient::download: the extraction runs in spawn_blocking + // and we await the join. A timeout that drops THIS future detaches the + // blocking task, which finishes its "rename into place" after cleanup. + tokio::task::spawn_blocking(move || { + std::thread::sleep(delay); + std::fs::create_dir_all(&target).unwrap(); + // A marker so the resurrected dir looks like a real extraction. + std::fs::write(target.join("HEAD"), b"ref: refs/heads/main\n").unwrap(); + extracted.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + }) + .await + .unwrap(); + Ok(()) + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + // F3: a cold read whose download times out must not let the detached + // spawn_blocking extraction resurrect the abandoned temp dir after cleanup. + // The store's extraction lands well past the download timeout; the timeout + // fires, acquire bails, and the extraction then creates ("renames into place") + // temp.path(). The fix ties cleanup to the extraction's completion — a + // detached janitor awaits the download future to settle, then removes the temp + // dir — so no `.tmp-download.*` dir survives. Pre-fix the timeout arm drops the + // download future, the detached extraction recreates temp.path() AFTER the RAII + // guard's Drop, and it survives forever — the second poll never sees a clean + // parent -> RED. + #[sqlx::test] + async fn timed_out_cold_read_does_not_resurrect_temp_dir(pool: sqlx::PgPool) { + use std::sync::atomic::Ordering::SeqCst; + let tmp = tempfile::TempDir::new().unwrap(); + let extracted = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let downloads = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let ts = DetachExtractStore { + // The extraction lands well after the 150ms download timeout below. + extract_delay: std::time::Duration::from_millis(600), + downloads: downloads.clone(), + extracted: extracted.clone(), + }; + let store = RepoStore::new(tmp.path().to_path_buf(), Some(std::sync::Arc::new(ts)), pool) + .with_release_upload_timeout(std::time::Duration::from_millis(150)); + let owner = "did:key:z6MkF3ResurrectTempDirAAAAAAAAAAAAAAAAAAAA"; + let name = "f3resurrect"; + let dir = repo_dir_of(tmp.path(), owner, name); + let parent = dir.parent().unwrap().to_path_buf(); + + // Cold cache miss: acquire enters the download path, the download times out + // at 150ms, and the read path bails (no local copy to serve). + let res = store.acquire(owner, name).await; + assert!( + res.is_err(), + "a timed-out cold read with no local copy must bail" + ); + assert_eq!(downloads.load(SeqCst), 1, "exactly one download attempt"); + + // Wait for the detached extraction to actually finish its "rename into + // place" (the resurrection moment). + assert!( + poll_until_true(|| extracted.load(SeqCst) >= 1).await, + "the detached extraction must run to completion" + ); + + // Once the extraction has landed, no temp dir may survive under the parent. + // The fix's janitor removes temp.path() after the extraction settles; + // pre-fix the resurrected dir survives and this poll never sees it cleaned. + assert!( + poll_until_true(|| !has_leftover_temp(&parent)).await, + "a timed-out cold read must not leave a resurrected .tmp-download.* dir" + ); + } } diff --git a/crates/gitlawb-node/src/git/repo_store/download.rs b/crates/gitlawb-node/src/git/repo_store/download.rs index c0e2eabf..efbb01c8 100644 --- a/crates/gitlawb-node/src/git/repo_store/download.rs +++ b/crates/gitlawb-node/src/git/repo_store/download.rs @@ -233,17 +233,48 @@ impl RepoStore { // A timeout takes the SAME cleanup arm as a download error (return Err, // the temp guard's Drop removes the dir), so `download_published` frees // the map entry and wakes waiters rather than leaving them parked forever. - match tokio::time::timeout( - self.release_upload_timeout, - store.download(owner_slug, repo_name, temp.path()), - ) - .await - { + // + // The download future is OWNED (Box::pin) rather than borrowed so the + // timeout arm can hand the still-running extraction to a detached janitor + // instead of dropping it (finding F3). `store.download` runs its tar + // extraction in `spawn_blocking`, and a tokio timeout does NOT cancel a + // spawn_blocking task: dropping the download future on timeout detaches + // that task, which then renames its extraction into `temp.path()` AFTER + // this function's `temp` guard Drop has already removed the dir — + // resurrecting an abandoned, fully-populated temp dir that the per-repo + // sweep above only reclaims on a later download of the SAME repo, so an + // abandoned extraction for a repo never re-downloaded accumulates on disk. + let store_dl = Arc::clone(store); + let (dl_owner, dl_repo, dl_target) = ( + owner_slug.to_string(), + repo_name.to_string(), + temp.path().to_path_buf(), + ); + let mut download_fut = + Box::pin(async move { store_dl.download(&dl_owner, &dl_repo, &dl_target).await }); + match tokio::time::timeout(self.release_upload_timeout, &mut download_fut).await { Ok(Ok(())) => {} Ok(Err(e)) => return Err(e), Err(_) => { warn!(repo = %repo_name, timeout_secs = self.release_upload_timeout.as_secs(), "read download timed out — discarding"); + // Do NOT drop the download future here: its spawn_blocking + // extraction may still be running, and dropping detaches it, + // letting its rename resurrect `temp.path()` after cleanup. Tie + // cleanup to the extraction's completion instead — disarm the RAII + // guard and hand the future to a detached janitor that awaits it + // to settle (the spawn_blocking join resolves, so any rename has + // landed) and only THEN removes the temp dir. Waiters are freed + // immediately by the return below; the janitor runs off the + // critical path, bounded by the extraction itself. + let temp_path = temp.path().to_path_buf(); + temp.disarm(); + tokio::spawn(async move { + let _ = download_fut.await; + let _ = + tokio::task::spawn_blocking(move || std::fs::remove_dir_all(&temp_path)) + .await; + }); return Err(anyhow::anyhow!("read download timed out")); } } From 0b30fa0864ad7f366005d3b6f2961c47dd1536fa Mon Sep 17 00:00:00 2001 From: t Date: Wed, 22 Jul 2026 12:52:14 -0500 Subject: [PATCH 55/61] chore(node): fmt + clippy cleanup for the #196 fixes Auto-deref (&mut tx) on the lock_repo_slug calls and rustfmt of the new test lines. No behavior change. --- crates/gitlawb-node/src/db/mod.rs | 9 +++------ crates/gitlawb-node/src/git/repo_store.rs | 8 ++++++-- crates/gitlawb-node/src/git/store.rs | 6 +++++- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 77d42347..79a3de17 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1054,7 +1054,7 @@ impl Db { // both call sites hash the identical slug string. let slug = format!("{}/{}", normalize_owner_key(owner_short), name); let mut tx = self.pool.begin().await?; - lock_repo_slug(&mut *tx, &slug).await?; + lock_repo_slug(&mut tx, &slug).await?; // `quarantined` is set only on first insert (the admission decision). // A re-sync (ON CONFLICT) preserves the existing flag — admission runs // once, and an operator's later release must not be silently reverted. @@ -1373,7 +1373,7 @@ impl Db { // identical slug string BEFORE the sibling check below, so a peer-sync insert // cannot land between that point-in-time check and the slug-scoped cascade // deletes (which would half-wipe the live mirror). Auto-released on commit. - lock_repo_slug(&mut *tx, &slug).await?; + lock_repo_slug(&mut tx, &slug).await?; // Grandchildren first: PR reviews/comments key on pr_id, so delete them // before the parent PRs vanish. @@ -4019,10 +4019,7 @@ mod dedup_db_tests { // The mirror's slug-scoped live state, keyed on the shared slug. The mirror's // `repos` row is deliberately NOT present yet — ingestion lands it mid-race. - let slug = format!( - "{}/victim", - crate::db::normalize_owner_key(owner) - ); + let slug = format!("{}/victim", crate::db::normalize_owner_key(owner)); sqlx::query( "INSERT INTO branch_cids (repo, ref_name, sha, cid, node_did, updated_at) VALUES ($1, 'refs/heads/main', '1', 'cid', 'n', '2026-02-01T00:00:00Z')", diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 121d364f..3fdd1004 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -2752,8 +2752,12 @@ mod tests { downloads: downloads.clone(), extracted: extracted.clone(), }; - let store = RepoStore::new(tmp.path().to_path_buf(), Some(std::sync::Arc::new(ts)), pool) - .with_release_upload_timeout(std::time::Duration::from_millis(150)); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ) + .with_release_upload_timeout(std::time::Duration::from_millis(150)); let owner = "did:key:z6MkF3ResurrectTempDirAAAAAAAAAAAAAAAAAAAA"; let name = "f3resurrect"; let dir = repo_dir_of(tmp.path(), owner, name); diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 1deffbe6..64a6b7b4 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -138,7 +138,11 @@ pub fn restore_refs(repo_path: &Path, snapshot: &[(String, String)]) -> Result<( } if !errors.is_empty() { - bail!("failed to restore {} ref(s): {}", errors.len(), errors.join("; ")); + bail!( + "failed to restore {} ref(s): {}", + errors.len(), + errors.join("; ") + ); } Ok(()) } From 1aa8d1800ac27a220b388f0849b92d578938f253 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 22 Jul 2026 16:21:27 -0500 Subject: [PATCH 56/61] fix(node): never let a failed ref snapshot become a delete-all rollback; cover merge and push rollbacks --- crates/gitlawb-node/src/api/pulls.rs | 166 +++++++++++++++ crates/gitlawb-node/src/api/repos.rs | 288 ++++++++++++++++++++++++++- 2 files changed, 448 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/api/pulls.rs b/crates/gitlawb-node/src/api/pulls.rs index b2a39179..3be8b1d3 100644 --- a/crates/gitlawb-node/src/api/pulls.rs +++ b/crates/gitlawb-node/src/api/pulls.rs @@ -458,3 +458,169 @@ pub async fn list_comments( let comments = state.db.list_pr_comments(&pr.id).await?; Ok(Json(serde_json::json!({ "comments": comments }))) } + +#[cfg(test)] +mod tests { + // super::* brings in the handler plus the axum Extension/Path/State + // extractors and the Utc/Uuid/PullRequest types the seed uses. + use super::*; + use std::path::Path as StdPath; + use std::time::Duration; + + // Object store whose upload() parks forever, so release()'s bounded timeout + // is the only thing that completes it, modeling a stalled durable PUT. + // exists() is false so acquire_write never downloads over the local bare repo. + struct StallStore; + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for StallStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(false) + } + async fn upload(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + std::future::pending::<()>().await; + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + // The failed-upload rollback on the merge path. merge_pr writes the merge + // commit to LOCAL disk, then release() uploads to durable storage. When + // that upload times out the handler must 5xx (before merge_pr/webhooks) + // AND roll the target branch back to its pre-merge tip while the advisory + // lock is held, so a local-fast-path read never serves the un-uploaded + // merged tree. RED with the cleanup closure at ~pulls.rs:244 reverted to + // `|_| {}`: main stays at the merge commit. + #[sqlx::test] + async fn merge_failed_upload_rolls_back_target_ref(pool: sqlx::PgPool) { + use axum::response::IntoResponse; + use std::process::Command; + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(StallStore)), + pool.clone(), + ) + .with_release_upload_timeout(Duration::from_millis(200)); + + let owner = "z6mergerollbackowner"; + let name = "mergerollback"; + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + + fn git(args: &[&str], dir: &StdPath) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + // Scratch repo: c1 on main, c2 on feature; the bare clone carries both + // branches, so the merge of feature into main has real work to do. + let work = tempfile::TempDir::new().unwrap(); + git(&["init", "-q", "-b", "main", "."], work.path()); + git(&["config", "user.email", "t@t"], work.path()); + git(&["config", "user.name", "t"], work.path()); + std::fs::write(work.path().join("f.txt"), "one").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c1"], work.path()); + git(&["checkout", "-q", "-b", "feature"], work.path()); + std::fs::write(work.path().join("g.txt"), "two").unwrap(); + git(&["add", "g.txt"], work.path()); + git(&["commit", "-q", "-m", "c2"], work.path()); + + std::fs::create_dir_all(bare.parent().unwrap()).unwrap(); + let out = Command::new("git") + .args([ + "clone", + "--bare", + "-q", + &work.path().to_string_lossy(), + &bare.to_string_lossy(), + ]) + .output() + .unwrap(); + assert!( + out.status.success(), + "clone --bare failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + + let pre_merge_tip = store::ref_oid(&bare, "refs/heads/main") + .unwrap() + .expect("main exists before the merge"); + + // Seed the open PR row the handler looks up. + let record = state.db.get_repo(owner, name).await.unwrap().unwrap(); + let now = Utc::now().to_rfc3339(); + state + .db + .create_pr(&PullRequest { + id: Uuid::new_v4().to_string(), + repo_id: record.id.clone(), + number: 1, + title: "t".into(), + body: None, + author_did: owner.to_string(), + source_branch: "feature".into(), + target_branch: "main".into(), + status: "open".to_string(), + merged_by_did: None, + merged_at: None, + created_at: now.clone(), + updated_at: now, + }) + .await + .unwrap(); + + // Merge as the owner: the merge commit applies locally, the durable + // upload stalls and times out, and the handler must report 5xx. + let resp = super::merge_pr( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), name.to_string(), 1)), + ) + .await; + let status = match resp { + Ok(_) => StatusCode::OK, + Err(e) => e.into_response().status(), + }; + assert!( + status.is_server_error(), + "a stalled durable upload on merge must 5xx, got {status}" + ); + + // The target branch must still point at the pre-merge tip: the failed + // upload rolled the merge commit back while the lock was held. + let tip = store::ref_oid(&bare, "refs/heads/main") + .unwrap() + .expect("main still exists after the rollback"); + assert_eq!( + tip, pre_merge_tip, + "a failed durable upload must roll the target branch back to its \ + pre-merge tip; a local-fast-path read served the un-uploaded merge" + ); + } +} diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index bc7c5502..5767cd94 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -887,6 +887,32 @@ async fn notify_peer_of_refs( } } +/// Roll a push whose durable upload failed back to the pre-push ref snapshot. +/// `None` means the pre-push listing itself failed: rolling back to an empty +/// snapshot would delete EVERY ref in the repo, so the rollback is skipped +/// instead. A genuinely empty repo snapshots as `Some(vec![])` and still rolls +/// back (deleting the refs the failed push created). +fn rollback_push_refs( + path: &std::path::Path, + repo_name: &str, + pre_push_refs: &Option>, +) { + match pre_push_refs { + Some(refs) => { + if let Err(e) = store::restore_refs(path, refs) { + tracing::warn!(repo = %repo_name, err = %e, + "failed to roll back receive-pack refs after failed durable upload; \ + a local-fast-path read may serve un-uploaded refs"); + } + } + None => { + tracing::warn!(repo = %repo_name, + "skipping ref rollback after failed durable upload: pre-push snapshot \ + unavailable; a local-fast-path read may serve un-uploaded refs"); + } + } +} + /// POST /:owner/:repo.git/git-receive-pack (AUTH REQUIRED — enforced by middleware) pub async fn git_receive_pack( State(state): State, @@ -1004,7 +1030,10 @@ pub async fn git_receive_pack( // applied pack back on local disk (mirrors create_issue). Without this // the local fast path in RepoStore::acquire serves the pushed refs until // a later write refreshes the archive, even though the client got a 5xx. - let pre_push_refs = store::list_refs(&disk_path).unwrap_or_default(); + // `.ok()`, not `.unwrap_or_default()`: a listing error must read as + // "snapshot unavailable", never as an empty snapshot, or the rollback + // below would delete every ref in the repo. + let pre_push_refs = store::list_refs(&disk_path).ok(); tracing::debug!(repo = %record.name, path = %disk_path.display(), "running git receive-pack"); let result = smart_http::receive_pack(&disk_path, body, git_timeout).await; @@ -1016,11 +1045,7 @@ pub async fn git_receive_pack( // local-fast-path read does not serve refs that never landed durably. let upload_ok = guard .release_with_failure_cleanup(result.is_ok(), |path| { - if let Err(e) = store::restore_refs(path, &pre_push_refs) { - tracing::warn!(repo = %record.name, err = %e, - "failed to roll back receive-pack refs after failed durable upload; \ - a local-fast-path read may serve un-uploaded refs"); - } + rollback_push_refs(path, &record.name, &pre_push_refs) }) .await; @@ -3476,6 +3501,257 @@ mod tests { ); } + // Rollback completeness on the receive-pack path: a failed durable upload + // must restore the EXACT pre-push snapshot (a ref the push updated is + // rewound AND a ref the push created is deleted) while the advisory lock + // is held, so the local fast path never serves refs that never landed + // durably. RED with the cleanup closure reverted to `|_| {}` (or with + // restore_refs not deleting created refs): main stays at the pushed tip + // and/or the created ref survives. + #[sqlx::test] + async fn push_failed_upload_rolls_back_created_and_updated_refs(pool: sqlx::PgPool) { + use axum::extract::{Path as AxPath, State}; + use axum::response::IntoResponse; + use axum::Extension; + use std::path::Path as StdPath; + use std::process::{Command, Stdio}; + use std::time::Duration; + + struct StallStore; + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for StallStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(false) + } + async fn upload(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + std::future::pending::<()>().await; + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(StallStore)), + pool.clone(), + ) + .with_release_upload_timeout(Duration::from_millis(200)); + + let owner = "z6refrollbackowner"; + let name = "refrollback"; + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + + fn git(args: &[&str], dir: &std::path::Path) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + // Two commits in a scratch repo: the bare repo starts at c1 and the + // push advances main to c2 AND creates a second ref at c2. + let work = tempfile::TempDir::new().unwrap(); + git(&["init", "-q", "-b", "main", "."], work.path()); + git(&["config", "user.email", "t@t"], work.path()); + git(&["config", "user.name", "t"], work.path()); + std::fs::write(work.path().join("f.txt"), "one").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c1"], work.path()); + let old_oid = git(&["rev-parse", "HEAD"], work.path()); + std::fs::write(work.path().join("f.txt"), "two").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c2"], work.path()); + let new_oid = git(&["rev-parse", "HEAD"], work.path()); + + // Bare clone (has both commits' objects), then rewind main to c1 so the + // push is a genuine update satisfiable with an empty pack. + std::fs::create_dir_all(bare.parent().unwrap()).unwrap(); + let out = Command::new("git") + .args([ + "clone", + "--bare", + "-q", + &work.path().to_string_lossy(), + &bare.to_string_lossy(), + ]) + .output() + .unwrap(); + assert!(out.status.success(), "clone --bare failed"); + git( + &[ + "-C", + &bare.to_string_lossy(), + "update-ref", + "refs/heads/main", + &old_oid, + ], + work.path(), + ); + + let snapshot = store::list_refs(&bare).unwrap(); + assert_eq!( + snapshot, + vec![("refs/heads/main".to_string(), old_oid.clone())], + "pre-push snapshot must be exactly main at c1" + ); + + // Two commands: update main c1 -> c2, create refs/heads/feature at c2. + let zero = "0".repeat(40); + let mut body = Vec::new(); + let cmd1 = format!("{old_oid} {new_oid} refs/heads/main\0report-status\n"); + body.extend_from_slice(format!("{:04x}", cmd1.len() + 4).as_bytes()); + body.extend_from_slice(cmd1.as_bytes()); + let cmd2 = format!("{zero} {new_oid} refs/heads/feature\n"); + body.extend_from_slice(format!("{:04x}", cmd2.len() + 4).as_bytes()); + body.extend_from_slice(cmd2.as_bytes()); + body.extend_from_slice(b"0000"); + let pack = Command::new("git") + .args([ + "-C", + &bare.to_string_lossy(), + "pack-objects", + "--stdout", + "-q", + ]) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(pack.status.success(), "pack-objects failed"); + body.extend_from_slice(&pack.stdout); + let body = Bytes::from(body); + + let resp = super::git_receive_pack( + State(state.clone()), + AxPath((owner.to_string(), format!("{name}.git"))), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + body, + ) + .await; + let status = match resp { + Ok(r) => r.status(), + Err(e) => e.into_response().status(), + }; + assert!( + status.is_server_error(), + "a timed-out durable upload must fail the push (5xx), got {status}" + ); + + // The refs must equal the pre-push snapshot EXACTLY: the created ref is + // gone and the updated ref is rewound. + let after = store::list_refs(&bare).unwrap(); + assert_eq!( + after, snapshot, + "a failed durable upload must restore the exact pre-push snapshot \ + (created ref deleted, updated ref rewound)" + ); + } + + // A bare repo at HEAD with one commit on main, for the rollback-decision + // unit tests below. Returns the tempdirs (kept alive by the caller), the + // bare path, and main's oid. + fn seeded_bare_repo() -> (tempfile::TempDir, tempfile::TempDir, std::path::PathBuf, String) { + use std::process::Command; + + fn git(args: &[&str], dir: &std::path::Path) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + let work = tempfile::TempDir::new().unwrap(); + git(&["init", "-q", "-b", "main", "."], work.path()); + git(&["config", "user.email", "t@t"], work.path()); + git(&["config", "user.name", "t"], work.path()); + std::fs::write(work.path().join("f.txt"), "hi").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c"], work.path()); + let oid = git(&["rev-parse", "HEAD"], work.path()); + + let dir = tempfile::TempDir::new().unwrap(); + let bare = dir.path().join("repo.git"); + let out = Command::new("git") + .args([ + "clone", + "--bare", + "-q", + &work.path().to_string_lossy(), + &bare.to_string_lossy(), + ]) + .output() + .unwrap(); + assert!(out.status.success(), "clone --bare failed"); + (work, dir, bare, oid) + } + + // The None arm of the rollback decision: a failed pre-push listing means + // "snapshot unavailable", and the rollback must be SKIPPED: the repo's + // existing refs survive. RED on the pre-fix shape (unwrap_or_default + an + // unconditional restore_refs), which reads the error as an empty snapshot + // and deletes every ref. + #[test] + fn ref_rollback_skips_when_snapshot_unavailable() { + let (_work, _dir, bare, oid) = seeded_bare_repo(); + + super::rollback_push_refs(&bare, "r", &None); + + assert_eq!( + store::list_refs(&bare).unwrap(), + vec![("refs/heads/main".to_string(), oid)], + "a None snapshot (listing failed) must skip the rollback, not \ + mass-delete the repo's refs" + ); + } + + // The must-keep negative of the fix: a genuinely EMPTY repo snapshots as + // Some(vec![]) and still rolls back, deleting the refs the failed push + // created. Guards against the `.ok()` change accidentally widening the + // skip to empty snapshots. + #[test] + fn ref_rollback_empty_snapshot_still_deletes_created_refs() { + let (_work, _dir, bare, _oid) = seeded_bare_repo(); + + super::rollback_push_refs(&bare, "r", &Some(vec![])); + + assert_eq!( + store::list_refs(&bare).unwrap(), + Vec::<(String, String)>::new(), + "an empty (but present) snapshot must still roll back: the ref the \ + push created has to be deleted" + ); + } + /// Repo creation must be throttled by the per-IP creation limiter BEFORE /// signature verification — otherwise a DID farm (one throwaway did:key per /// repo, each carrying a valid but machine-solved iCaptcha proof) walks past From 42c51e8e362a53e2570070d99f656fa9b9b97a40 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 22 Jul 2026 16:21:27 -0500 Subject: [PATCH 57/61] fix(node): slug-lock every peer-ingest writer and drive the real ingest in the purge-race test --- crates/gitlawb-node/src/db/mod.rs | 132 ++++++++++++++++++++++-------- 1 file changed, 98 insertions(+), 34 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 79a3de17..3f1fd4ce 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -945,6 +945,20 @@ async fn lock_repo_slug(conn: &mut sqlx::PgConnection, slug: &str) -> Result<()> Ok(()) } +/// Normalize an `owner/name` slug to the exact string `delete_repo_by_id` builds its +/// advisory-lock key from: `normalize_owner_key(owner)/name`, splitting at the first +/// `/` (owners never contain `/`; DID owners use `:`). Writers that receive the slug +/// as one string (`branch_cids.repo`, `sync_queue.repo`, `received_ref_updates.repo`) +/// must lock this normalized form, not the verbatim input, or a full-DID caller would +/// hash a different key than the purge and slip past the serialization. A string with +/// no `/` is returned as-is; the purge never locks such a key, so it is inert. +fn normalize_repo_slug(repo: &str) -> String { + match repo.split_once('/') { + Some((owner, name)) => format!("{}/{}", normalize_owner_key(owner), name), + None => repo.to_string(), + } +} + #[cfg(test)] mod normalize_owner_key_tests { use super::normalize_owner_key; @@ -1782,6 +1796,15 @@ impl Db { cid: &str, node_did: &str, ) -> Result<()> { + // `branch_cids` is one of the slug-scoped tables `delete_repo_by_id` cascades + // over. Take the shared advisory lock (same normalized slug the purge hashes) + // inside a tx so this row cannot land between the purge's point-in-time + // sibling check and its cascade deletes (which would strand an orphan row or + // half-wipe a live mirror's state). Single lock, nothing else held, so the + // ordering matches every other slug-lock site and cannot deadlock. + let slug = normalize_repo_slug(repo); + let mut tx = self.pool.begin().await?; + lock_repo_slug(&mut tx, &slug).await?; sqlx::query( "INSERT INTO branch_cids (repo, ref_name, sha, cid, node_did, updated_at) VALUES ($1, $2, $3, $4, $5, $6) @@ -1795,8 +1818,9 @@ impl Db { .bind(cid) .bind(node_did) .bind(Utc::now().to_rfc3339()) - .execute(&self.pool) + .execute(&mut *tx) .await?; + tx.commit().await?; Ok(()) } @@ -1845,6 +1869,13 @@ impl Db { new_sha: &str, cid: Option<&str>, ) -> Result<()> { + // `sync_queue` is slug-scoped and cascaded by `delete_repo_by_id`; take the + // shared advisory lock on the same normalized slug so an enqueue cannot + // interleave with the purge's sibling-check-then-cascade (same shape and + // rationale as `upsert_branch_cid`; single lock, no deadlock exposure). + let slug = normalize_repo_slug(repo); + let mut tx = self.pool.begin().await?; + lock_repo_slug(&mut tx, &slug).await?; sqlx::query( "INSERT INTO sync_queue (id, repo, node_did, ref_name, new_sha, cid, status, enqueued_at) VALUES ($1, $2, $3, $4, $5, $6, 'pending', $7) @@ -1857,8 +1888,9 @@ impl Db { .bind(new_sha) .bind(cid) .bind(Utc::now().to_rfc3339()) - .execute(&self.pool) + .execute(&mut *tx) .await?; + tx.commit().await?; Ok(()) } @@ -2537,6 +2569,14 @@ impl Db { impl Db { pub async fn insert_ref_update(&self, update: &ReceivedRefUpdate) -> Result<()> { + // `received_ref_updates` is slug-scoped and cascaded by `delete_repo_by_id`; + // take the shared advisory lock on the same normalized slug so a peer-fed + // insert cannot land inside the purge's sibling-check-then-cascade window + // (same shape and rationale as `upsert_branch_cid`; single lock, no deadlock + // exposure). + let slug = normalize_repo_slug(&update.repo); + let mut tx = self.pool.begin().await?; + lock_repo_slug(&mut tx, &slug).await?; sqlx::query( "INSERT INTO received_ref_updates (id, node_did, pusher_did, repo, ref_name, old_sha, new_sha, timestamp, @@ -2555,8 +2595,9 @@ impl Db { .bind(&update.cert_id) .bind(&update.received_at) .bind(&update.from_peer) - .execute(&self.pool) + .execute(&mut *tx) .await?; + tx.commit().await?; Ok(()) } @@ -3993,15 +4034,18 @@ mod dedup_db_tests { /// AFTER that check but before/around the cascade would leave a half-wipe: the /// mirror's `repos` row survives while its live child rows / bounty are gone. /// - /// Deterministic race: hold a session advisory lock on the same `hashtextextended` - /// slug key the cascade locks on, so the spawned purge BLOCKS at the top of its tx - /// (before the sibling check). While it is parked, land the mirror's `repos` row - /// (what ingestion writes), then release the lock. With the lock in place the purge - /// resumes, its sibling check now SEES the mirror row, and it skips the cascade — - /// both the mirror row and its child rows survive (GREEN). Remove the - /// `lock_repo_slug` call from `delete_repo_by_id` and the purge no longer blocks: it - /// runs its sibling check before the mirror row exists, cascades, and half-wipes the - /// child rows / bounty while the mirror row (inserted afterwards) survives (RED). + /// Deterministic race, with BOTH sides going through the real code: hold a session + /// advisory lock on the same `hashtextextended` slug key both sides hash, spawn the + /// REAL `upsert_mirror_repo` (parks on its own `lock_repo_slug`, first in the wait + /// queue), assert the mirror row has NOT landed while parked, then spawn the purge + /// (parks second) and release. Advisory-lock waiters are granted FIFO, so the + /// ingest commits the mirror row first and the purge's sibling check then SEES it + /// and skips the cascade: mirror row, child rows, and bounty all survive (GREEN). + /// Remove `lock_repo_slug` from `upsert_mirror_repo` and the ingest no longer + /// parks: the mirror row appears while the session lock is still held and the + /// mid-park absence assert fails (RED). Remove it from `delete_repo_by_id` and the + /// purge no longer blocks: it runs its sibling check before the parked ingest has + /// committed, cascades, and half-wipes the child rows / bounty (RED). #[sqlx::test] async fn purge_serializes_with_mirror_ingest_via_advisory_lock(pool: PgPool) { let db = db(pool).await; @@ -4060,40 +4104,60 @@ mod dedup_db_tests { .await .unwrap(); - // Spawn the purge; with the lock present it parks at the top of its tx. - let pool_for_task = db.pool().clone(); - let purge = tokio::spawn(async move { - Db::for_testing(pool_for_task) - .delete_repo_by_id("rid-purge-race-target") + // Spawn the REAL mirror ingest; its own `lock_repo_slug` inside + // `upsert_mirror_repo` parks it on the held session lock, first in the wait + // queue. Bare owner short form, so it normalizes to the identical slug the + // purge locks. + let pool_for_ingest = db.pool().clone(); + let ingest = tokio::spawn(async move { + Db::for_testing(pool_for_ingest) + .upsert_mirror_repo( + "z6MkPurgeRaceMirrorFixtureForLock", + "victim", + "/srv/mirror", + None, + false, + ) .await }); - // Give the purge time to reach the lock wait (GREEN) or, without the lock, to - // run its whole sibling-check-then-cascade (RED) before the mirror row lands. + // Give the ingest time to reach the lock wait, then assert it is actually + // parked: the mirror row must NOT exist while the session lock is held. This + // is the load-bearing check for the ingest side; without `lock_repo_slug` in + // `upsert_mirror_repo` the insert lands during this sleep (RED). tokio::time::sleep(std::time::Duration::from_millis(300)).await; + let landed_early: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM repos WHERE id=$1") + .bind(&slug) + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!( + landed_early, 0, + "the real upsert_mirror_repo must block on the slug advisory lock, not insert past it" + ); - // Mirror ingestion lands the colliding `repos` row (slug-form id, bare owner) — - // exactly what upsert_mirror_repo writes. Raw insert here so it does not contend - // on the advisory lock the test connection is holding. - sqlx::query( - "INSERT INTO repos (id, name, owner_did, description, is_public, default_branch, - created_at, updated_at, disk_path, machine_id, quarantined) - VALUES ($1, 'victim', 'z6MkPurgeRaceMirrorFixtureForLock', 'mirrored from peer', - true, 'main', '2026-02-01T00:00:00Z', '2026-02-01T00:00:00Z', - '/srv/mirror', NULL, false)", - ) - .bind(&slug) - .execute(db.pool()) - .await - .unwrap(); + // Spawn the purge; it parks on the same lock, second in the queue. Without + // `lock_repo_slug` in `delete_repo_by_id` it runs immediately instead: its + // sibling check sees no mirror row (the ingest is still parked), so it + // cascades the child rows and tombstones the bounty (RED below). + let pool_for_purge = db.pool().clone(); + let purge = tokio::spawn(async move { + Db::for_testing(pool_for_purge) + .delete_repo_by_id("rid-purge-race-target") + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; - // Release the lock so the purge can proceed (and observe the mirror row). + // Release the lock. Advisory-lock waiters are woken FIFO: the ingest (queued + // first) commits the mirror row, then the purge acquires the lock and its + // sibling check observes the committed row. sqlx::query("SELECT pg_advisory_unlock(hashtextextended($1, 0))") .bind(&slug) .execute(&mut *lock_conn) .await .unwrap(); + ingest.await.unwrap().unwrap(); let removed = purge.await.unwrap().unwrap(); assert_eq!(removed, 1, "the empty canonical target row is purged"); From 39b002f4fa4e0678f59821569da7321324fe4187 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 22 Jul 2026 16:21:27 -0500 Subject: [PATCH 58/61] fix(node): own cold-download cleanup in a spawned task so handler cancellation cannot resurrect temp dirs --- crates/gitlawb-node/src/git/repo_store.rs | 68 +++++++++++++++- .../src/git/repo_store/download.rs | 81 +++++++++---------- 2 files changed, 105 insertions(+), 44 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 3fdd1004..51905213 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -2734,9 +2734,9 @@ mod tests { // spawn_blocking extraction resurrect the abandoned temp dir after cleanup. // The store's extraction lands well past the download timeout; the timeout // fires, acquire bails, and the extraction then creates ("renames into place") - // temp.path(). The fix ties cleanup to the extraction's completion — a - // detached janitor awaits the download future to settle, then removes the temp - // dir — so no `.tmp-download.*` dir survives. Pre-fix the timeout arm drops the + // temp.path(). The fix ties cleanup to the extraction's completion: a spawned + // task owns the download and the temp guard, so the temp dir is removed only + // once the download settles, and no `.tmp-download.*` dir survives. Pre-fix the timeout arm drops the // download future, the detached extraction recreates temp.path() AFTER the RAII // guard's Drop, and it survives forever — the second poll never sees a clean // parent -> RED. @@ -2787,4 +2787,66 @@ mod tests { "a timed-out cold read must not leave a resurrected .tmp-download.* dir" ); } + + // Cancellation twin of the resurrection test above: the handler future is + // DROPPED mid-download (axum cancels it on client disconnect) instead of + // timing out. Pre-fix, dropping the handler dropped the download future + // while its spawn_blocking extraction kept running; TempDownloadDir::Drop + // removed the (not yet created) temp dir, and the extraction's later + // "rename into place" resurrected it as an orphan that only a later + // same-repo download would sweep -> the cleanup poll never succeeds -> RED. + // Post-fix the spawned task owns the download and the temp guard, keeps + // running across the caller's cancellation, and removes the temp dir once + // the download settles -> GREEN. + #[sqlx::test] + async fn cancelled_cold_read_does_not_resurrect_temp_dir(pool: sqlx::PgPool) { + use std::sync::atomic::Ordering::SeqCst; + let tmp = tempfile::TempDir::new().unwrap(); + let extracted = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let downloads = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let ts = DetachExtractStore { + // Long enough that the abort below lands while the extraction is + // still running, so the cancellation drops the download mid-flight. + extract_delay: std::time::Duration::from_millis(600), + downloads: downloads.clone(), + extracted: extracted.clone(), + }; + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkCancelResurrectTempDirAAAAAAAAAAAAAAAA"; + let name = "cancelresurrect"; + let dir = repo_dir_of(tmp.path(), owner, name); + let parent = dir.parent().unwrap().to_path_buf(); + + // Cold cache miss: the handler enters the download path and its store + // download (the slow extraction) is in flight. + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.load(SeqCst) >= 1).await, + "the reader's download must be in flight" + ); + + // Drop the handler future mid-download (the client-disconnect hazard). + h.abort(); + + // Wait for the underlying extraction to settle (the resurrection moment + // pre-fix; the cleanup moment post-fix). + assert!( + poll_until_true(|| extracted.load(SeqCst) >= 1).await, + "the detached extraction must run to completion" + ); + + // Once the download has settled, no temp dir may survive under the + // parent. Pre-fix the resurrected dir survives and this poll never + // sees a clean parent -> RED. + assert!( + poll_until_true(|| !has_leftover_temp(&parent)).await, + "a cancelled cold read must not leave a resurrected .tmp-download.* dir" + ); + } } diff --git a/crates/gitlawb-node/src/git/repo_store/download.rs b/crates/gitlawb-node/src/git/repo_store/download.rs index efbb01c8..86cf7b96 100644 --- a/crates/gitlawb-node/src/git/repo_store/download.rs +++ b/crates/gitlawb-node/src/git/repo_store/download.rs @@ -194,8 +194,8 @@ impl RepoStore { .to_string_lossy(); // Best-effort sweep of leftover temp siblings from a prior download whose - // RAII guard dropped, or whose extract spawn_blocking completed - // uninterruptibly AFTER that guard's Drop ran (finding 1). Serialized per + // cleanup never ran (a process crash or kill mid-download, where no Drop + // and no owning task survive to remove the temp). Serialized per // repo by the download mutex, so no concurrent same-repo download owns a // live temp here; scoped to THIS repo's prefix, so other repos are // untouched. @@ -228,56 +228,55 @@ impl RepoStore { uuid::Uuid::new_v4() ))); - // Bound the network download: a stalled GET would park this reader (and, - // via the per-repo download mutex, every coalesced reader) indefinitely. - // A timeout takes the SAME cleanup arm as a download error (return Err, - // the temp guard's Drop removes the dir), so `download_published` frees - // the map entry and wakes waiters rather than leaving them parked forever. - // - // The download future is OWNED (Box::pin) rather than borrowed so the - // timeout arm can hand the still-running extraction to a detached janitor - // instead of dropping it (finding F3). `store.download` runs its tar - // extraction in `spawn_blocking`, and a tokio timeout does NOT cancel a - // spawn_blocking task: dropping the download future on timeout detaches - // that task, which then renames its extraction into `temp.path()` AFTER - // this function's `temp` guard Drop has already removed the dir — - // resurrecting an abandoned, fully-populated temp dir that the per-repo - // sweep above only reclaims on a later download of the SAME repo, so an - // abandoned extraction for a repo never re-downloaded accumulates on disk. + // Ownership of the download AND the temp-dir guard moves into a spawned + // task, so timeout and handler cancellation share ONE cleanup path + // (finding F3 and its cancellation twin). `store.download` runs its tar + // extraction in `spawn_blocking`, and neither a tokio timeout nor a + // dropped handler future cancels a spawn_blocking task: pre-fix, + // dropping the download future (a timed-out await, or axum dropping the + // whole handler on client disconnect) detached the extraction, whose + // later rename resurrected `temp.path()` AFTER the guard's Drop had + // removed it, leaving an orphan only a later same-repo download sweeps. + // The spawned task is never dropped mid-flight, so the download future + // always runs to settle. On success it returns the still-armed guard + // through the JoinHandle: the caller consumes it and publishes below. + // If the caller timed out or was cancelled, the JoinHandle is gone, so + // the runtime drops the returned guard when the task settles and its + // Drop removes the temp dir; cleanup is tied to the extraction's + // completion, never racing it (this subsumes the previous detached + // janitor). A download error drops the guard inside the task with the + // same at-settle timing. let store_dl = Arc::clone(store); let (dl_owner, dl_repo, dl_target) = ( owner_slug.to_string(), repo_name.to_string(), temp.path().to_path_buf(), ); - let mut download_fut = - Box::pin(async move { store_dl.download(&dl_owner, &dl_repo, &dl_target).await }); - match tokio::time::timeout(self.release_upload_timeout, &mut download_fut).await { - Ok(Ok(())) => {} - Ok(Err(e)) => return Err(e), + let dl_task = tokio::spawn(async move { + match store_dl.download(&dl_owner, &dl_repo, &dl_target).await { + Ok(()) => Ok(temp), + // `temp` drops here: the download settled, so removal is safe. + Err(e) => Err(e), + } + }); + // Bound the wait, not the work: a stalled GET would park this reader + // (and, via the per-repo download mutex, every coalesced reader) + // indefinitely. On timeout the caller gets the same error as before and + // `download_published` frees the map entry and wakes waiters; the task + // keeps running and cleans up when the download settles (see above). + let temp = match tokio::time::timeout(self.release_upload_timeout, dl_task).await { + Ok(Ok(Ok(temp))) => temp, + Ok(Ok(Err(e))) => return Err(e), + // The task panicked; its unwind dropped the guard, removing the dir. + Ok(Err(join_err)) => { + return Err(anyhow::Error::from(join_err).context("read download task failed")) + } Err(_) => { warn!(repo = %repo_name, timeout_secs = self.release_upload_timeout.as_secs(), "read download timed out — discarding"); - // Do NOT drop the download future here: its spawn_blocking - // extraction may still be running, and dropping detaches it, - // letting its rename resurrect `temp.path()` after cleanup. Tie - // cleanup to the extraction's completion instead — disarm the RAII - // guard and hand the future to a detached janitor that awaits it - // to settle (the spawn_blocking join resolves, so any rename has - // landed) and only THEN removes the temp dir. Waiters are freed - // immediately by the return below; the janitor runs off the - // critical path, bounded by the extraction itself. - let temp_path = temp.path().to_path_buf(); - temp.disarm(); - tokio::spawn(async move { - let _ = download_fut.await; - let _ = - tokio::task::spawn_blocking(move || std::fs::remove_dir_all(&temp_path)) - .await; - }); return Err(anyhow::anyhow!("read download timed out")); } - } + }; let guard = match self.try_lock_repo(owner_did, repo_name).await { Ok(Some(g)) => g, From f1574edff0b4dfa4b4e7cb59e215f51d913cb769 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 22 Jul 2026 16:21:27 -0500 Subject: [PATCH 59/61] refactor(node): delegate issue ref helpers to the store ref primitives --- crates/gitlawb-node/src/git/issues.rs | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/crates/gitlawb-node/src/git/issues.rs b/crates/gitlawb-node/src/git/issues.rs index 31070e4f..9e1277ff 100644 --- a/crates/gitlawb-node/src/git/issues.rs +++ b/crates/gitlawb-node/src/git/issues.rs @@ -90,18 +90,10 @@ pub fn issue_ref_oid(repo_path: &Path, issue_id: &str) -> Result return Ok(None), }; let ref_name = format!("refs/gitlawb/issues/{full_id}"); - let output = Command::new("git") - .args(["rev-parse", "--verify", &ref_name]) - .current_dir(repo_path) - .output() - .context("failed to run git rev-parse")?; - if !output.status.success() { - return Ok(None); - } - let oid = String::from_utf8_lossy(&output.stdout).trim().to_string(); - if oid.is_empty() { - return Ok(None); - } + let oid = match crate::git::store::ref_oid(repo_path, &ref_name)? { + Some(oid) => oid, + None => return Ok(None), + }; Ok(Some((full_id, oid))) } @@ -111,16 +103,7 @@ pub fn issue_ref_oid(repo_path: &Path, issue_id: &str) -> Result Result<()> { let ref_name = format!("refs/gitlawb/issues/{full_id}"); - let output = Command::new("git") - .args(["update-ref", &ref_name, oid]) - .current_dir(repo_path) - .output() - .context("failed to run git update-ref")?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - anyhow::bail!("git update-ref failed: {stderr}"); - } - Ok(()) + crate::git::store::set_ref(repo_path, &ref_name, oid) } /// List all issue refs and return their JSON content. From ff8515eba814d01c8882ddd969a71f3d3115e811 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 22 Jul 2026 16:55:28 -0500 Subject: [PATCH 60/61] fix(node): slug-lock arweave anchor writes; fail closed when the pre-push ref snapshot fails --- crates/gitlawb-node/src/api/repos.rs | 18 ++++++++++++++++-- crates/gitlawb-node/src/db/mod.rs | 10 +++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index a5f71541..7c7597f1 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1036,8 +1036,22 @@ pub async fn git_receive_pack( // a later write refreshes the archive, even though the client got a 5xx. // `.ok()`, not `.unwrap_or_default()`: a listing error must read as // "snapshot unavailable", never as an empty snapshot, or the rollback - // below would delete every ref in the repo. - let pre_push_refs = store::list_refs(&disk_path).ok(); + // below would delete every ref in the repo. Fail closed on a snapshot + // failure: without a restore plan, a failed durable upload would leave + // the pushed refs on the local fast path with no way back (the class + // this rollback exists to close), so refuse the push before mutating. + let pre_push_refs = match store::list_refs(&disk_path) { + Ok(refs) => Some(refs), + Err(e) => { + tracing::error!(repo = %record.name, err = %e, "pre-push ref snapshot failed"); + // release(false): no upload attempted, so the bool is always true. + let _ = guard.release(false).await; + let _ = report_tx.send(Err(AppError::Internal(anyhow::anyhow!( + "cannot snapshot pre-push refs; refusing push: {e}" + )))); + return; + } + }; tracing::debug!(repo = %record.name, path = %disk_path.display(), "running git receive-pack"); let result = smart_http::receive_pack(&disk_path, body, git_timeout).await; diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 44381b45..0b9353de 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -2953,6 +2953,13 @@ impl Db { pub async fn record_arweave_anchor(&self, input: &RecordAnchorInput<'_>) -> Result<()> { let id = Uuid::new_v4().to_string(); let now = Utc::now().to_rfc3339(); + // `arweave_anchors` is slug-scoped and cascaded by `delete_repo_by_id`; + // take the shared advisory lock on the same normalized slug so a post-push + // anchor cannot land inside the purge's sibling-check-then-cascade window + // (same shape as `upsert_branch_cid`; single lock, no deadlock exposure). + let slug = normalize_repo_slug(input.repo); + let mut tx = self.pool.begin().await?; + lock_repo_slug(&mut tx, &slug).await?; sqlx::query( "INSERT INTO arweave_anchors (id, repo, owner_did, ref_name, old_sha, new_sha, cid, irys_tx_id, arweave_url, node_did, anchored_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)", @@ -2968,8 +2975,9 @@ impl Db { .bind(input.arweave_url) .bind(input.node_did) .bind(&now) - .execute(&self.pool) + .execute(&mut *tx) .await?; + tx.commit().await?; Ok(()) } From 93d9a107aa0983d2b87915688e904d2a09afe44e Mon Sep 17 00:00:00 2001 From: t Date: Wed, 22 Jul 2026 17:29:06 -0500 Subject: [PATCH 61/61] fix(node): close review residuals: coalesce cold downloads to one in-flight task, compensate a timed-out upload with the rolled-back tree, aggregate partial ref restores, race-test the anchor lock, fail-closed snapshot test --- crates/gitlawb-node/src/api/repos.rs | 169 ++++++++ crates/gitlawb-node/src/db/mod.rs | 92 ++++- crates/gitlawb-node/src/git/repo_store.rs | 389 +++++++++++++++++- .../src/git/repo_store/download.rs | 180 +++++--- crates/gitlawb-node/src/git/store.rs | 207 +++++++++- 5 files changed, 962 insertions(+), 75 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 7c7597f1..49b4a8e8 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -3702,6 +3702,175 @@ mod tests { ); } + // Fail-closed fence on the pre-push ref snapshot, driven through the real + // handler: when `store::list_refs` on the acquired disk path fails, the + // handler must refuse the push (Internal, 5xx) BEFORE running receive-pack, + // because without a snapshot a failed durable upload has no restore plan. + // The repo on disk is a real bare repo whose config declares + // repositoryformatversion=999, so every git invocation in it fails + // deterministically at repo-setup time (`for-each-ref` included) while the + // directory itself is untouched valid state. No object store is configured, + // so acquire_write's local fast path uses the existing directory as-is and + // never repairs or re-downloads it. + // + // Load-bearing (RED) checks: remove the fence at the `store::list_refs` + // match in `git_receive_pack` (e.g. fall back to `.ok()` and proceed) and + // the failure comes from receive-pack instead, so the "cannot snapshot + // pre-push refs" assertion on the error text fails. The directory-listing + // comparison pins that nothing mutated the repo before the refusal. + #[sqlx::test] + async fn push_fails_closed_when_ref_snapshot_unavailable(pool: sqlx::PgPool) { + use axum::extract::{Path as AxPath, State}; + use axum::response::IntoResponse; + use std::process::{Command, Stdio}; + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + // No object store: acquire_write takes the local fast path and hands the + // on-disk directory to the handler exactly as seeded below. + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + None, + pool.clone(), + ); + + let owner = "z6snapshotfenceowner"; + let name = "snapfence"; + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + + fn git(args: &[&str], dir: &std::path::Path) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + // A scratch repo provides a real oid and a well-formed (empty) pack so + // the request body is a genuine push, not garbage the parser rejects. + let work = tempfile::TempDir::new().unwrap(); + git(&["init", "-q", "-b", "main", "."], work.path()); + git(&["config", "user.email", "t@t"], work.path()); + git(&["config", "user.name", "t"], work.path()); + std::fs::write(work.path().join("f.txt"), "hi").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c"], work.path()); + let oid = git(&["rev-parse", "HEAD"], work.path()); + + // Seed the target: a REAL bare repo, then declare an unsupported + // repository format version. Discovery still finds the repo (HEAD, + // objects/, refs/ all present) but every git command in it dies with + // "expected git repo version <= 1": the deterministic corruption that + // makes `list_refs` fail without any racy filesystem state. + std::fs::create_dir_all(bare.parent().unwrap()).unwrap(); + let out = Command::new("git") + .args(["init", "--bare", "-q", &bare.to_string_lossy()]) + .output() + .unwrap(); + assert!(out.status.success(), "git init --bare failed"); + std::fs::write( + bare.join("config"), + "[core]\n\trepositoryformatversion = 999\n\tbare = true\n", + ) + .unwrap(); + assert!( + store::list_refs(&bare).is_err(), + "precondition: list_refs must fail on the corrupted repo" + ); + + // Recursive sorted listing of the repo dir, to pin "nothing modified". + fn listing(root: &std::path::Path) -> Vec { + fn walk(root: &std::path::Path, dir: &std::path::Path, out: &mut Vec) { + for entry in std::fs::read_dir(dir).unwrap() { + let p = entry.unwrap().path(); + out.push(p.strip_prefix(root).unwrap().to_string_lossy().into_owned()); + if p.is_dir() { + walk(root, &p, out); + } + } + } + let mut out = Vec::new(); + walk(root, root, &mut out); + out.sort(); + out + } + let before = listing(&bare); + + // Minimal push body: create refs/heads/main at the scratch oid plus an + // empty pack (same shape as the rollback test above). + let zero = "0".repeat(40); + let cmd = format!("{zero} {oid} refs/heads/main\0report-status\n"); + let mut body = Vec::new(); + body.extend_from_slice(format!("{:04x}", cmd.len() + 4).as_bytes()); + body.extend_from_slice(cmd.as_bytes()); + body.extend_from_slice(b"0000"); + let pack = Command::new("git") + .args([ + "-C", + &work.path().to_string_lossy(), + "pack-objects", + "--stdout", + "-q", + ]) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(pack.status.success(), "pack-objects failed"); + body.extend_from_slice(&pack.stdout); + let body = Bytes::from(body); + + let resp = super::git_receive_pack( + State(state.clone()), + AxPath((owner.to_string(), format!("{name}.git"))), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + body, + ) + .await; + let err = match resp { + Ok(r) => panic!( + "push must fail closed when the ref snapshot is unavailable, got {}", + r.status() + ), + Err(e) => e, + }; + let msg = err.to_string(); + let status = err.into_response().status(); + assert!( + status.is_server_error(), + "snapshot failure must surface as a 5xx, got {status}" + ); + // This is the fence-specific assert: without the fail-closed branch the + // failure comes from receive-pack ("git error: ...") instead. + assert!( + msg.contains("cannot snapshot pre-push refs"), + "the refusal must come from the snapshot fence, before receive-pack; got: {msg}" + ); + + // Nothing ran against the repo: the directory contents are unchanged + // (no new refs, no objects, no receive-pack side effects). + let after = listing(&bare); + assert_eq!( + after, before, + "the corrupted repo must not be modified by a refused push" + ); + } + // A bare repo at HEAD with one commit on main, for the rollback-decision // unit tests below. Returns the tempdirs (kept alive by the caller), the // bare path, and main's oid. diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 0b9353de..51499ab6 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -4031,7 +4031,7 @@ mod agent_discovery_tests { #[cfg(test)] mod dedup_db_tests { - use super::{normalize_owner_key, BountyRecord, Db, RepoRecord}; + use super::{normalize_owner_key, BountyRecord, Db, RecordAnchorInput, RepoRecord}; use chrono::{DateTime, Utc}; use sqlx::PgPool; @@ -4445,6 +4445,96 @@ mod dedup_db_tests { ); } + /// Sibling of the mirror-ingest race above, for the arweave-anchor writer. + /// `record_arweave_anchor` inserts a slug-scoped child row (`arweave_anchors`) + /// that `delete_repo_by_id`'s cascade wipes, so it must serialize on the same + /// `lock_repo_slug` key or a post-push anchor can land inside the purge's + /// sibling-check-then-cascade window. Deterministic choreography, same as the + /// mirror test: hold a session advisory lock on the exact `hashtextextended` + /// slug key, spawn the REAL `record_arweave_anchor` (its own `lock_repo_slug` + /// parks it), assert mid-park that the anchor row has NOT landed (this is the + /// load-bearing check: remove `lock_repo_slug` from `record_arweave_anchor` + /// and the insert lands during the sleep, RED), then release and assert the + /// row lands. The writer is handed the full-DID `owner/name` form so the test + /// also pins the `normalize_repo_slug` folding to the purge's key. + #[sqlx::test] + async fn purge_serializes_with_arweave_anchor_via_advisory_lock(pool: PgPool) { + let db = db(pool).await; + let owner = "did:key:z6MkAnchorRaceFixtureForSlugLock"; + let target = rec( + "rid-anchor-race-target", + owner, + "victim", + "canonical", + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:00Z", + ); + db.create_repo(&target).await.unwrap(); + let slug = format!("{}/victim", crate::db::normalize_owner_key(owner)); + + // Hold a session-scoped advisory lock on the exact slug key the anchor + // writer's xact lock hashes; they share Postgres' advisory lock space. + let mut lock_conn = db.pool().acquire().await.unwrap(); + sqlx::query("SELECT pg_advisory_lock(hashtextextended($1, 0))") + .bind(&slug) + .execute(&mut *lock_conn) + .await + .unwrap(); + + // Spawn the REAL anchor writer with the full-DID slug form; its + // `normalize_repo_slug` folds it to the identical key the session lock + // holds, so `lock_repo_slug` parks it here. + let pool_for_anchor = db.pool().clone(); + let repo_arg = format!("{owner}/victim"); + let repo_arg_for_task = repo_arg.clone(); + let anchor = tokio::spawn(async move { + Db::for_testing(pool_for_anchor) + .record_arweave_anchor(&RecordAnchorInput { + repo: &repo_arg_for_task, + owner_did: "did:key:z6MkAnchorRaceFixtureForSlugLock", + ref_name: "refs/heads/main", + old_sha: "0", + new_sha: "1", + cid: Some("cid"), + irys_tx_id: "irys-tx-race", + arweave_url: "https://ar/anchor-race", + node_did: "did:key:z6MkNodeFixture", + }) + .await + }); + + // Give the writer time to reach the lock wait, then assert it is actually + // parked: no anchor row while the session lock is held. Without + // `lock_repo_slug` in `record_arweave_anchor` the insert commits during + // this sleep and the assert fails (RED). + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + let landed_early: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM arweave_anchors WHERE repo=$1") + .bind(&repo_arg) + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!( + landed_early, 0, + "record_arweave_anchor must block on the slug advisory lock, not insert past it" + ); + + // Release; the parked writer acquires the lock and commits. + sqlx::query("SELECT pg_advisory_unlock(hashtextextended($1, 0))") + .bind(&slug) + .execute(&mut *lock_conn) + .await + .unwrap(); + anchor.await.unwrap().unwrap(); + + let landed: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM arweave_anchors WHERE repo=$1") + .bind(&repo_arg) + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!(landed, 1, "the anchor lands once the lock is released"); + } + /// U8 (F): bounties key on (repo_owner, repo_name), not an immutable repo id, /// and `delete_repo_by_id` deliberately does not delete them (financial record). /// Without tombstoning, a purged repo's OPEN bounty stays claimable and silently diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 51905213..8b9c540f 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -807,6 +807,12 @@ impl RepoWriteGuard { // caller surfaces as a FAILED push so the client re-pushes rather than // trusting a commit that never reached durable storage. (P1 data-loss.) let mut upload_ok = true; + // Whether the failed upload TIMED OUT (as opposed to erroring). On the + // error arm the PUT definitively failed; on the timeout arm the dropped + // upload future may have already transmitted the full body, so the store + // can still commit the POST-state archive; that arm alone needs the + // compensating upload after cleanup below. + let mut upload_timed_out = false; // Upload to Tigris only on success. Bound the upload so a stalled PUT // cannot pin the guard's advisory-lock connection indefinitely — on @@ -829,6 +835,7 @@ impl RepoWriteGuard { } Err(_) => { upload_ok = false; + upload_timed_out = true; warn!(repo = %self.repo_name, timeout_secs = self.release_upload_timeout.as_secs(), "tigris upload timed out after write — releasing the advisory lock without a completed upload"); } @@ -844,6 +851,41 @@ impl RepoWriteGuard { // back, which a later download would resurrect. if !upload_ok { cleanup(&self.local_path); + + // TIMEOUT arm only: the dropped upload future may have fully + // transmitted the PUT body with only the response in flight, so the + // store can commit the POST-state archive even though `cleanup` + // just rolled the local refs back, and a later cold download would + // resurrect the rolled-back write. Attempt ONE bounded compensating + // upload of the now-rolled-back tree: it starts strictly after the + // rollback, so if it lands it commits after the orphan PUT's + // near-immediate commit and the store converges to the rolled-back + // state either way. This at most doubles the bounded lock hold on + // this rare arm; the advisory lock must still be held during the + // compensation, and it is, since this runs before the unlock below. + // `upload_ok` stays false regardless: the client was already told + // the write failed; the compensation only converges durable state. + // The error arm needs none of this: its PUT definitively failed. + if upload_timed_out { + if let Some(ref tigris) = self.object_store { + match tokio::time::timeout( + self.release_upload_timeout, + tigris.upload(&self.owner_slug, &self.repo_name, &self.local_path), + ) + .await + { + Ok(Ok(())) => {} + Ok(Err(e)) => { + warn!(repo = %self.repo_name, err = %e, + "compensating upload after timed-out release upload failed: store may hold a write that was reported failed until the next successful upload"); + } + Err(_) => { + warn!(repo = %self.repo_name, timeout_secs = self.release_upload_timeout.as_secs(), + "compensating upload after timed-out release upload timed out: store may hold a write that was reported failed until the next successful upload"); + } + } + } + } } // Unlock on the SAME connection that took the lock, then TAKE the @@ -1929,6 +1971,146 @@ mod tests { ); } + // ObjectStore double for the timed-out-release compensation tests: `upload()` + // records its call count and, at entry, whether the caller's rollback marker + // file exists in the repo dir (so a test can prove WHICH state each upload + // observed). When `err_first` is set the first call fails immediately (the + // Err arm); otherwise the first call parks forever past the release timeout + // (the timeout arm) and every later call succeeds instantly. + struct CompensationProbeStore { + err_first: bool, + calls: std::sync::Arc, + marker_at_upload: std::sync::Arc>>, + } + + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for CompensationProbeStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(false) + } + async fn upload(&self, _o: &str, _r: &str, p: &std::path::Path) -> anyhow::Result<()> { + let n = self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.marker_at_upload + .lock() + .unwrap() + .push(p.join("rollback-marker").exists()); + if n == 0 { + if self.err_first { + anyhow::bail!("simulated PUT failure"); + } + // Park forever; the release timeout is what must unblock it. + std::future::pending::<()>().await; + } + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, _p: &std::path::Path) -> anyhow::Result<()> { + unreachable!("exists() is always false, so no download should run"); + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + // A release upload that TIMES OUT may have fully transmitted its PUT body + // (only the response in flight), so the store can commit the POST-state + // archive while cleanup rolls the local tree back, and a later cold download + // would resurrect the rolled-back write. release_with_failure_cleanup must + // therefore attempt ONE bounded compensating upload of the rolled-back tree + // after cleanup, on the timeout arm only. Pre-fix (no compensation) upload + // runs exactly once -> the call-count == 2 assert is RED. + #[sqlx::test] + async fn timed_out_release_upload_compensates_with_rolled_back_tree(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let marker_at_upload = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let ts = CompensationProbeStore { + err_first: false, + calls: calls.clone(), + marker_at_upload: marker_at_upload.clone(), + }; + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ) + .with_release_upload_timeout(std::time::Duration::from_millis(300)); + let owner = "did:key:z6MkTimeoutCompensateAAAAAAAAAAAAAAAAAAAAAA"; + let name = "compensate"; + + // A real write under the guard, so the first upload has a tree to PUT. + let guard = store.acquire_write(owner, name).await.unwrap(); + crate::git::store::init_bare(guard.path()).unwrap(); + + let cleanup_ran = std::sync::atomic::AtomicBool::new(false); + let ok = guard + .release_with_failure_cleanup(true, |p| { + // Models the rollback: mutate a marker the double snapshots at + // upload entry, so the compensating upload's observed state is + // provably POST-cleanup. + std::fs::write(p.join("rollback-marker"), b"rolled back").unwrap(); + cleanup_ran.store(true, std::sync::atomic::Ordering::SeqCst); + }) + .await; + + assert!(!ok, "a timed-out upload must still be reported as failed"); + assert!( + cleanup_ran.load(std::sync::atomic::Ordering::SeqCst), + "cleanup must run on the timeout arm" + ); + assert_eq!( + calls.load(std::sync::atomic::Ordering::SeqCst), + 2, + "the timeout arm must attempt exactly one compensating upload after cleanup" + ); + assert_eq!( + *marker_at_upload.lock().unwrap(), + vec![false, true], + "the compensating upload must observe the POST-cleanup (rolled-back) tree" + ); + } + + // The Err arm must NOT compensate: an erroring PUT definitively failed, so + // there is no orphan archive to converge over. Exactly one upload call. + #[sqlx::test] + async fn errored_release_upload_does_not_compensate(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let ts = CompensationProbeStore { + err_first: true, + calls: calls.clone(), + marker_at_upload: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), + }; + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ) + .with_release_upload_timeout(std::time::Duration::from_millis(300)); + let owner = "did:key:z6MkErrNoCompensateAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "nocompensate"; + + let guard = store.acquire_write(owner, name).await.unwrap(); + crate::git::store::init_bare(guard.path()).unwrap(); + + let cleanup_ran = std::sync::atomic::AtomicBool::new(false); + let ok = guard + .release_with_failure_cleanup(true, |_| { + cleanup_ran.store(true, std::sync::atomic::Ordering::SeqCst) + }) + .await; + + assert!(!ok, "an erroring upload must be reported as failed"); + assert!( + cleanup_ran.load(std::sync::atomic::Ordering::SeqCst), + "cleanup must run on the error arm" + ); + assert_eq!( + calls.load(std::sync::atomic::Ordering::SeqCst), + 1, + "the error arm must not attempt a compensating upload" + ); + } + // FINDING 3 / R2 / INV-22: acquire_write's under-lock freshness DOWNLOAD must // be bounded, not just release()'s upload. acquire_write holds the write // advisory lock on a pinned lock-pool connection, then does store.exists() + @@ -2633,12 +2815,16 @@ mod tests { // A cold-read holder parks mid-download, then its handler future is dropped // (axum cancels it on client disconnect). The per-repo map entry must be - // removed anyway — the pre-fix code removed it only at explicit return points, - // which a cancelled future skips, so a permissionless caller could fan cold - // GETs across many public repos and disconnect to grow the map. The RAII - // DownloadEntryGuard's Drop closes that gap. This reads the private - // download_locks map directly (the test module is a child of RepoStore's - // module). Pre-fix: after abort the entry LEAKS -> still present -> RED. + // removed once the in-flight download settles. The original pre-fix code + // removed it only at explicit return points, which a cancelled future + // skips, so a permissionless caller could fan cold GETs across many public + // repos and disconnect to grow the map. The RAII DownloadEntryGuard's Drop + // closes that gap; the guard now travels with the spawned download task, + // so the entry deliberately STAYS registered while the download is in + // flight (that is what keeps cancel-then-retry from duplicating the + // download) and is pruned at settle. This reads the private download_locks + // map directly (the test module is a child of RepoStore's module). + // Original bug: after abort the entry leaked forever -> never freed -> RED. #[sqlx::test] async fn cancelled_cold_read_frees_the_map_entry(pool: sqlx::PgPool) { let tmp = tempfile::TempDir::new().unwrap(); @@ -2672,12 +2858,26 @@ mod tests { // Cancel the handler future mid-download (the client-disconnect hazard). h.abort(); + assert!( + h.await.unwrap_err().is_cancelled(), + "the holder future must be cancelled, not settled" + ); + + // The spawned download task owns the coordination, so the entry stays + // registered while the download is still in flight; a reader arriving + // now queues on it instead of starting a duplicate download. + assert!( + store.download_locks.lock().unwrap().contains_key(&map_key), + "the entry must stay registered while the cancelled holder's download is in flight" + ); - // The RAII guard's Drop must remove the entry despite the cancellation. + // Let the parked download settle; the task's returned guards then drop + // in the runtime and prune the entry. + gate.notify_one(); let freed = poll_until_true(|| store.download_locks.lock().unwrap().is_empty()).await; assert!( freed, - "a cold-read future cancelled mid-download must still free its map entry" + "the map entry must be freed once the cancelled holder's download settles" ); } @@ -2849,4 +3049,177 @@ mod tests { "a cancelled cold read must not leave a resurrected .tmp-download.* dir" ); } + + // Timeout waves must coalesce on the ONE in-flight download. Pre-fix, a + // holder whose download timed out released the per-repo mutex and pruned + // the map entry while its spawned download task was still running; the + // next reader of the same repo then became a fresh holder and started a + // second full store.download, so N timeout waves meant N concurrent + // downloads of the same object. Post-fix the spawned download task itself + // owns `held` and the entry guard until it settles, so the second reader + // queues on the same entry, its bounded wait elapses, and it reports the + // same timeout error WITHOUT invoking store.download again. RED recipe: in + // download.rs, revert the ownership move (make the spawned task return + // only `temp` and keep `held`/`entry_guard` in the caller frame, the + // pre-fix unwind): the holder's timeout then frees the lock and entry + // while the download still runs, the second reader acquires them and + // starts download #2, and the downloads == 1 assert fails. + #[sqlx::test] + async fn timed_out_cold_reads_coalesce_on_the_inflight_download(pool: sqlx::PgPool) { + use std::sync::atomic::Ordering::SeqCst; + let tmp = tempfile::TempDir::new().unwrap(); + let extracted = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let downloads = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let ts = DetachExtractStore { + // The download settles well after BOTH callers' 150ms timeouts. + extract_delay: std::time::Duration::from_millis(600), + downloads: downloads.clone(), + extracted: extracted.clone(), + }; + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ) + .with_release_upload_timeout(std::time::Duration::from_millis(150)); + let owner = "did:key:z6MkTimeoutCoalesceAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "timeoutcoalesce"; + let dir = repo_dir_of(tmp.path(), owner, name); + let parent = dir.parent().unwrap().to_path_buf(); + + // First cold reader: its download stalls and its 150ms timeout fires. + let store1 = store.clone(); + let (o1, n1) = (owner.to_string(), name.to_string()); + let h1 = tokio::spawn(async move { store1.acquire(&o1, &n1).await }); + assert!( + poll_until_true(|| downloads.load(SeqCst) == 1).await, + "the first reader's download must be in flight" + ); + // Start the second reader measurably after the first, so that pre-fix + // (RED) the lock the first holder frees at ~150ms is available well + // before the second reader's own 150ms wait elapses. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h2 = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + + let res1 = h1.await.unwrap(); + let res2 = h2.await.unwrap(); + assert!( + res1.is_err(), + "the stalled holder must report its timeout error" + ); + let err2 = res2.expect_err("the coalesced reader must also time out, not succeed"); + assert!( + format!("{err2:#}").contains("read download timed out"), + "the coalesced reader reports the same timeout error, got: {err2:#}" + ); + assert_eq!( + downloads.load(SeqCst), + 1, + "both timed-out cold reads must share the ONE in-flight download" + ); + + // Once the in-flight download settles, the task-owned coordination must + // be released (map entry pruned) and the temp dir must be cleaned. + assert!( + poll_until_true(|| extracted.load(SeqCst) >= 1).await, + "the detached extraction must run to completion" + ); + assert!( + poll_until_true(|| store.download_locks.lock().unwrap().is_empty()).await, + "the map entry must be pruned once the download settles" + ); + assert!( + poll_until_true(|| !has_leftover_temp(&parent)).await, + "the settled download's temp dir must be cleaned up" + ); + } + + // Cancellation twin of the coalescing test above: the first cold reader is + // ABORTED mid-download (axum drops the handler on client disconnect) + // instead of timing out. Pre-fix (guards in the caller frame), the abort + // dropped `held` and the entry guard immediately while the spawned + // download task kept running, so a retry arriving right after the + // disconnect became a fresh holder and started a second full + // store.download. Post-fix the task owns the coordination, so the retry + // queues on the in-flight download, its bounded wait elapses, and it + // reports the timeout error WITHOUT a second download; the entry is + // pruned once the task settles. RED recipe: same revert as above (task + // returns only `temp`, guards stay caller-side): the abort then frees + // lock and entry at once, the retry starts download #2, and the + // downloads == 1 assert fails. + #[sqlx::test] + async fn cancelled_cold_read_coalesces_next_reader_on_the_inflight_download( + pool: sqlx::PgPool, + ) { + use std::sync::atomic::Ordering::SeqCst; + let tmp = tempfile::TempDir::new().unwrap(); + let extracted = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let downloads = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let ts = DetachExtractStore { + // The download settles well after the retry's 150ms bounded wait. + extract_delay: std::time::Duration::from_millis(600), + downloads: downloads.clone(), + extracted: extracted.clone(), + }; + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ) + .with_release_upload_timeout(std::time::Duration::from_millis(150)); + let owner = "did:key:z6MkCancelCoalesceAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "cancelcoalesce"; + let dir = repo_dir_of(tmp.path(), owner, name); + let parent = dir.parent().unwrap().to_path_buf(); + + // First cold reader: download in flight, then the handler is dropped. + let store1 = store.clone(); + let (o1, n1) = (owner.to_string(), name.to_string()); + let h1 = tokio::spawn(async move { store1.acquire(&o1, &n1).await }); + assert!( + poll_until_true(|| downloads.load(SeqCst) == 1).await, + "the first reader's download must be in flight" + ); + h1.abort(); + assert!( + h1.await.unwrap_err().is_cancelled(), + "the first reader must be cancelled, not settled" + ); + + // Retry after the disconnect, well before the download settles at + // ~600ms: it must queue on the in-flight download, not start its own. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let res2 = tokio::spawn(async move { store2.acquire(&o2, &n2).await }) + .await + .unwrap(); + let err2 = res2.expect_err("the retry must hit its bounded wait, not succeed"); + assert!( + format!("{err2:#}").contains("read download timed out"), + "the retry reports the bounded-wait timeout error, got: {err2:#}" + ); + assert_eq!( + downloads.load(SeqCst), + 1, + "a retry after a cancelled cold read must share the ONE in-flight download" + ); + + // At settle, the task-owned coordination unwinds: entry pruned, temp + // dir cleaned. + assert!( + poll_until_true(|| extracted.load(SeqCst) >= 1).await, + "the detached extraction must run to completion" + ); + assert!( + poll_until_true(|| store.download_locks.lock().unwrap().is_empty()).await, + "the map entry must be pruned once the cancelled holder's download settles" + ); + assert!( + poll_until_true(|| !has_leftover_temp(&parent)).await, + "the settled download's temp dir must be cleaned up" + ); + } } diff --git a/crates/gitlawb-node/src/git/repo_store/download.rs b/crates/gitlawb-node/src/git/repo_store/download.rs index 86cf7b96..8543b11a 100644 --- a/crates/gitlawb-node/src/git/repo_store/download.rs +++ b/crates/gitlawb-node/src/git/repo_store/download.rs @@ -59,11 +59,14 @@ impl Drop for TempDownloadDir { /// RAII removal of a download-coordination map entry (finding 4). Its `Drop` /// removes the entry from `download_locks` whenever the map still holds THIS /// Arc (ptr_eq-guarded so a newer entry inserted after an earlier removal is -/// never clobbered). Removing on every exit — normal return, `?`, or a handler -/// future cancelled mid cold-read (axum drops it on client disconnect) — is -/// what keeps the map from growing per arbitrary requested name: the pre-fix -/// explicit removals ran only at the labelled return points, which a cancelled -/// future skipped, leaking the entry. The outer map is a std::sync::Mutex so +/// never clobbered). Removal is guaranteed on every path (a normal return or +/// `?` drops it in the holder; once a download is in flight it travels with +/// the spawned task and drops at settle, covering holder timeout and a +/// handler future cancelled mid cold-read, which axum triggers on client +/// disconnect), which is what keeps the map from growing per arbitrary +/// requested name: the pre-fix explicit removals ran only at the labelled +/// return points, which a cancelled future skipped, leaking the entry. The +/// outer map is a std::sync::Mutex so /// this Drop can prune it synchronously (Drop cannot be async); it is only ever /// held briefly for a get/insert/remove, never across an await. struct DownloadEntryGuard { @@ -111,24 +114,35 @@ impl RepoStore { .or_insert_with(|| Arc::new(Mutex::new(()))), ) }; - // RAII: remove the map entry on EVERY exit of this function — normal - // return, `?`, or handler cancellation (finding 4). Its Drop is ptr_eq- - // guarded, so it never clobbers a newer entry, and it subsumes the - // pre-fix explicit removals (which a cancelled future skipped, leaking - // the entry). Declared here so it outlives `held` below and prunes the - // map only after the inner per-repo lock has been released. - let _entry_guard = DownloadEntryGuard { - locks: Arc::clone(&self.download_locks), - key: map_key.clone(), - entry: Arc::clone(&entry), - }; - let held = match entry.try_lock() { + let held = match Arc::clone(&entry).try_lock_owned() { Ok(g) => g, Err(_) => { - let g = entry.lock().await; + // Bounded coalescing wait: once a holder's download is in + // flight, the spawned task owns this lock (and the map entry) + // until it settles, whatever happens to the holder (timeout or + // cancellation), so an unbounded lock().await here would park + // this reader for as long as a stall lasts. On elapse, report + // the same timeout error the holder would. Do NOT touch the + // map entry on this path: pruning it while the download is in + // flight would let the next arrival start a duplicate download. + let waited = tokio::time::timeout( + self.release_upload_timeout, + Arc::clone(&entry).lock_owned(), + ) + .await; + let g = match waited { + Ok(g) => g, + Err(_) => { + if local_path.exists() { + // A concurrent reader published while we waited. + return Ok(DownloadOutcome::Published); + } + return Err(anyhow::anyhow!("read download timed out")); + } + }; if local_path.exists() { // A concurrent reader published while we waited — serve it. - // `_entry_guard`'s Drop removes the map entry on return. + // The holder's own guard prunes the map entry on its exit. return Ok(DownloadOutcome::Published); } // The holder degraded without publishing; this reader takes over. @@ -145,20 +159,33 @@ impl RepoStore { g } }; - let outcome = self - .download_and_publish( - owner_did, - repo_name, - owner_slug, - local_path, - store, - local_existed_at_entry, - ) - .await; - drop(held); - outcome - // `_entry_guard`'s Drop removes the map entry here (after `held` and on - // every early return above), including when this future is cancelled. + // RAII: remove the map entry when this holder's work is done (finding + // 4). Created only once THIS reader holds the per-repo lock, so a + // parked or timed-out waiter never prunes the entry out from under a + // live holder; its Drop is ptr_eq-guarded, so it never clobbers a + // newer entry. Ownership moves into download_and_publish and from + // there into the spawned download task itself, so on EVERY settle path + // (success, download error, holder timeout, holder cancellation) the + // entry stays registered and the lock stays held until the in-flight + // download settles: waiters queue on the ONE download instead of each + // timeout or cancel wave starting a fresh full download of the same + // object. + let entry_guard = DownloadEntryGuard { + locks: Arc::clone(&self.download_locks), + key: map_key, + entry: Arc::clone(&entry), + }; + self.download_and_publish( + owner_did, + repo_name, + owner_slug, + local_path, + store, + local_existed_at_entry, + entry_guard, + held, + ) + .await } /// The holder's half of the coordinated download: fetch and extract the @@ -177,6 +204,15 @@ impl RepoStore { /// the lock, before the swap, we detect a writer that published a fresher /// copy during our unlocked download window and serve theirs instead of /// clobbering it with our now-stale temp. + /// + /// `entry_guard` and `held` are this holder's per-repo coordination (the + /// download_locks map entry and the locked per-repo mutex). Both move into + /// the spawned download task, which returns them through its JoinHandle on + /// success; on any other settle path (download error, task panic, holder + /// timeout, holder cancellation) they drop when the task settles, so + /// waiters keep queueing on the SAME in-flight download until it settles + /// rather than each starting a duplicate one. + #[allow(clippy::too_many_arguments)] async fn download_and_publish( &self, owner_did: &str, @@ -185,6 +221,8 @@ impl RepoStore { local_path: &Path, store: &Arc, local_existed_at_entry: bool, + entry_guard: DownloadEntryGuard, + held: tokio::sync::OwnedMutexGuard<()>, ) -> Result { let parent = local_path.parent().context("repo path has no parent")?; std::fs::create_dir_all(parent).context("creating repo parent dir")?; @@ -252,31 +290,67 @@ impl RepoStore { repo_name.to_string(), temp.path().to_path_buf(), ); + // The per-repo coordination (`held`, `entry_guard`) moves into the task + // alongside the temp guard: whatever happens to THIS caller (timeout + // below, or a handler future cancelled by axum on client disconnect), + // the lock stays held and the map entry stays registered until the + // in-flight download settles, so waiters and new arrivals queue on the + // ONE download instead of starting duplicates. On success the tuple + // returns through the JoinHandle: the caller publishes under `held` as + // before. If the caller timed out or was cancelled, the runtime drops + // the returned tuple when the task settles, in field order: `temp` + // (dir removed), then `held` (lock released), then `entry_guard` (map + // entry pruned), the same unlock-before-prune order as a normal + // return. let dl_task = tokio::spawn(async move { + // The guards travel as one `(held, entry_guard)` tuple so that + // wherever it drops (here on error, in the runtime at settle after + // a caller timeout or cancellation, or in the caller after a + // normal publish), its field order gives unlock-before-prune. match store_dl.download(&dl_owner, &dl_repo, &dl_target).await { - Ok(()) => Ok(temp), - // `temp` drops here: the download settled, so removal is safe. - Err(e) => Err(e), + Ok(()) => Ok((temp, (held, entry_guard))), + // The download settled, so cleanup is safe. Explicit drops pin + // the order (async-block captures have no guaranteed one): + // temp removed, lock released, map entry pruned last. + Err(e) => { + drop(temp); + drop(held); + drop(entry_guard); + Err(e) + } } }); // Bound the wait, not the work: a stalled GET would park this reader - // (and, via the per-repo download mutex, every coalesced reader) - // indefinitely. On timeout the caller gets the same error as before and - // `download_published` frees the map entry and wakes waiters; the task - // keeps running and cleans up when the download settles (see above). - let temp = match tokio::time::timeout(self.release_upload_timeout, dl_task).await { - Ok(Ok(Ok(temp))) => temp, - Ok(Ok(Err(e))) => return Err(e), - // The task panicked; its unwind dropped the guard, removing the dir. - Ok(Err(join_err)) => { - return Err(anyhow::Error::from(join_err).context("read download task failed")) - } - Err(_) => { - warn!(repo = %repo_name, timeout_secs = self.release_upload_timeout.as_secs(), - "read download timed out — discarding"); - return Err(anyhow::anyhow!("read download timed out")); - } - }; + // indefinitely. On timeout the caller gets the same error as before, + // but the coordination does NOT unwind: it lives in the task (see + // above) and is released only at settle. Waiters therefore stay queued + // on the SAME in-flight download (each bounded by its own timeout in + // download_published) instead of each timeout wave starting a fresh + // full download of the same object; the task keeps running and cleans + // up its temp when the download settles. + // `_coordination` is `(held, entry_guard)`, kept alive to the end of + // this function exactly as the separate guards were before: the + // publish below still runs with the per-repo lock held, and the map + // entry is pruned only on exit (unlock first, prune second, per the + // tuple's field order). + let (temp, _coordination) = + match tokio::time::timeout(self.release_upload_timeout, dl_task).await { + Ok(Ok(Ok(returned))) => returned, + Ok(Ok(Err(e))) => return Err(e), + // The task panicked; its unwind dropped the guards, removing + // the dir, releasing the lock, and pruning the map entry. + Ok(Err(join_err)) => { + return Err(anyhow::Error::from(join_err).context("read download task failed")) + } + Err(_) => { + warn!(repo = %repo_name, timeout_secs = self.release_upload_timeout.as_secs(), + "read download timed out — discarding"); + // Dropping the JoinHandle detaches the task; the runtime + // frees temp dir, lock, and map entry at settle (see the + // spawn comment above). + return Err(anyhow::anyhow!("read download timed out")); + } + }; let guard = match self.try_lock_repo(owner_did, repo_name).await { Ok(Some(g)) => g, diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 64a6b7b4..09904aea 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -120,21 +120,36 @@ pub fn restore_refs(repo_path: &Path, snapshot: &[(String, String)]) -> Result<( } // Delete refs that exist now but were not in the snapshot (created by the - // write being rolled back). + // write being rolled back). A failure to LIST the current refs is collected + // like any per-ref failure, never returned early: the snapshot-reset half + // above already ran, and bailing here would discard its collected errors. let snapshot_names: HashSet<&str> = snapshot.iter().map(|(r, _)| r.as_str()).collect(); - let current = list_refs(repo_path)?; - for (ref_name, _) in ¤t { - if !snapshot_names.contains(ref_name.as_str()) { - let output = Command::new("git") - .args(["update-ref", "-d", ref_name]) - .current_dir(repo_path) - .output() - .context("failed to run git update-ref -d")?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - errors.push(format!("{ref_name} (delete): {stderr}")); + match list_refs(repo_path) { + Ok(current) => { + for (ref_name, _) in ¤t { + if !snapshot_names.contains(ref_name.as_str()) { + match Command::new("git") + .args(["update-ref", "-d", ref_name]) + .current_dir(repo_path) + .output() + { + Ok(output) if output.status.success() => {} + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + errors.push(format!("{ref_name} (delete): {stderr}")); + } + Err(e) => { + errors.push(format!( + "{ref_name} (delete): failed to run git update-ref -d: {e}" + )); + } + } + } } } + Err(e) => { + errors.push(format!("(extra-ref sweep) listing current refs: {e}")); + } } if !errors.is_empty() { @@ -539,10 +554,176 @@ pub fn repo_disk_path(repos_dir: &Path, owner_did: &str, repo_name: &str) -> Pat #[cfg(test)] mod tests { - use super::branch_diff_names; + use super::{branch_diff_names, list_refs, restore_refs}; use std::path::Path; use std::process::Command; + fn run_git(dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + /// Bare repo with main at c2 and keeper at c2, whose "pre-push" state was + /// main and keeper both at c1. Returns (workdir, baredir, bare_path, c1, c2). + fn advanced_bare_repo() -> ( + tempfile::TempDir, + tempfile::TempDir, + std::path::PathBuf, + String, + String, + ) { + let work = tempfile::TempDir::new().unwrap(); + run_git(work.path(), &["init", "-q", "-b", "main", "."]); + run_git(work.path(), &["config", "user.email", "t@t"]); + run_git(work.path(), &["config", "user.name", "t"]); + std::fs::write(work.path().join("f.txt"), "one").unwrap(); + run_git(work.path(), &["add", "f.txt"]); + run_git(work.path(), &["commit", "-qm", "c1"]); + let c1 = run_git(work.path(), &["rev-parse", "HEAD"]); + std::fs::write(work.path().join("f.txt"), "two").unwrap(); + run_git(work.path(), &["add", "f.txt"]); + run_git(work.path(), &["commit", "-qm", "c2"]); + let c2 = run_git(work.path(), &["rev-parse", "HEAD"]); + + let dir = tempfile::TempDir::new().unwrap(); + let bare = dir.path().join("repo.git"); + let out = Command::new("git") + .args([ + "clone", + "--bare", + "-q", + &work.path().to_string_lossy(), + &bare.to_string_lossy(), + ]) + .output() + .unwrap(); + assert!(out.status.success(), "clone --bare failed"); + // "The push" advanced main to c2 (already there from the clone), moved + // keeper c1 -> c2, and created feature at c2. + run_git(&bare, &["update-ref", "refs/heads/keeper", &c2]); + run_git(&bare, &["update-ref", "refs/heads/feature", &c2]); + (work, dir, bare, c1, c2) + } + + fn sorted_refs(bare: &Path) -> Vec<(String, String)> { + let mut refs = list_refs(bare).unwrap(); + refs.sort(); + refs + } + + /// A single unrestorable snapshot entry (invalid oid, `set_ref` fails) must + /// not abort the restore: every OTHER snapshot ref is still reset and the + /// push-created extra ref is still deleted, and the fn returns Err carrying + /// the failure. RED with the reset loop reverted to `set_ref(..)?`: the + /// broken entry is first in the snapshot, so an early return leaves keeper + /// at c2 and feature alive. + #[test] + fn restore_refs_partial_failure_still_resets_rest_and_deletes_extras() { + let (_work, _dir, bare, c1, c2) = advanced_bare_repo(); + + let snapshot = vec![ + ( + "refs/heads/broken".to_string(), + "not-a-valid-oid".to_string(), + ), + ("refs/heads/main".to_string(), c1.clone()), + ("refs/heads/keeper".to_string(), c1.clone()), + ]; + let err = restore_refs(&bare, &snapshot).unwrap_err(); + let msg = format!("{err:#}"); + assert!( + msg.contains("refs/heads/broken"), + "the aggregated error names the failed ref: {msg}" + ); + + let refs = sorted_refs(&bare); + assert_eq!( + refs, + vec![ + ("refs/heads/keeper".to_string(), c1.clone()), + ("refs/heads/main".to_string(), c1.clone()), + ], + "keeper and main are reset to c1 and feature is deleted despite the \ + broken entry (c2 was {c2})" + ); + } + + /// Happy path unchanged by the hardening: a fully valid snapshot restores + /// exactly (refs rewound, created ref deleted) and returns Ok. + #[test] + fn restore_refs_happy_path_restores_snapshot_exactly() { + let (_work, _dir, bare, c1, _c2) = advanced_bare_repo(); + + let snapshot = vec![ + ("refs/heads/main".to_string(), c1.clone()), + ("refs/heads/keeper".to_string(), c1.clone()), + ]; + restore_refs(&bare, &snapshot).unwrap(); + + let refs = sorted_refs(&bare); + assert_eq!( + refs, + vec![ + ("refs/heads/keeper".to_string(), c1.clone()), + ("refs/heads/main".to_string(), c1), + ], + "restore must reproduce the snapshot exactly" + ); + } + + /// When the internal `list_refs` (extra-ref sweep) fails, the snapshot-reset + /// half must still have run and its collected failures must surface in the + /// aggregated error, not be discarded by an early `?` return. The repo is a + /// real bare repo declaring repositoryformatversion=999, so both `set_ref` + /// and `list_refs` fail deterministically. RED with the sweep reverted to + /// `let current = list_refs(repo_path)?;`: the raw for-each-ref error + /// propagates without the "failed to restore" aggregate or the ref name. + #[test] + fn restore_refs_aggregates_when_internal_list_refs_fails() { + let dir = tempfile::TempDir::new().unwrap(); + let bare = dir.path().join("repo.git"); + let out = Command::new("git") + .args(["init", "--bare", "-q", &bare.to_string_lossy()]) + .output() + .unwrap(); + assert!(out.status.success(), "git init --bare failed"); + std::fs::write( + bare.join("config"), + "[core]\n\trepositoryformatversion = 999\n\tbare = true\n", + ) + .unwrap(); + assert!( + list_refs(&bare).is_err(), + "precondition: list_refs must fail on the corrupted repo" + ); + + let snapshot = vec![("refs/heads/main".to_string(), "a".repeat(40))]; + let err = restore_refs(&bare, &snapshot).unwrap_err(); + let msg = format!("{err:#}"); + assert!( + msg.contains("failed to restore"), + "must return the aggregated error, not the raw list_refs error: {msg}" + ); + assert!( + msg.contains("refs/heads/main"), + "the set_ref failure from the reset half must survive the sweep \ + failure: {msg}" + ); + assert!( + msg.contains("listing current refs"), + "the sweep failure itself is also collected: {msg}" + ); + } + #[test] fn branch_diff_names_lists_changed_paths() { let td = tempfile::TempDir::new().unwrap();